mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
feat(config): support per-model reasoning_effort overrides
Add agent.reasoning_overrides dict to config.yaml. Users can now set
a reasoning_effort per model, overriding the global agent.reasoning_effort.
Example:
agent:
reasoning_effort: "medium" # global default
reasoning_overrides:
"openrouter/anthropic/claude-opus-4.5": "xhigh"
"openai/gpt-5": "low"
"claude-sonnet-4.6": "high" # bare model name also works
The helper is spelling-tolerant: override keys match regardless of
provider prefix or dots-vs-dashes normalization, so users can write
keys in any sensible form and they'll match.
Resolution priority:
1. Session-scoped /reasoning --session override (gateway only; unchanged)
2. Per-model override from agent.reasoning_overrides (spelling-tolerant)
3. Global agent.reasoning_effort (existing)
4. Provider default (unchanged)
Wired into:
- CLI startup (cli.py)
- Messaging gateway agent construction (gateway/run.py)
- Desktop/TUI _load_reasoning_config (tui_gateway/server.py)
- Cron job scheduler (cron/scheduler.py)
- /model mid-session switch (agent/agent_runtime_helpers.py)
+ _primary_runtime now tracks reasoning_config for correct fallback recovery
- Fallback activation (agent/chat_completion_helpers.py::try_activate_fallback)
+ Re-resolves reasoning_config for the fallback model (best-effort)
Closes #21256 (per-model reasoning_effort defaults).
Note: no hermes config set agent.reasoning_overrides.<model> support;
users edit the YAML directly. _set_nested splits on "." and would
corrupt model keys containing version dots.
This commit is contained in:
parent
1f41bdbecd
commit
d9cdb81923
17 changed files with 1321 additions and 18 deletions
|
|
@ -1305,6 +1305,13 @@ def restore_primary_runtime(agent) -> bool:
|
|||
primary_provider or "?",
|
||||
)
|
||||
|
||||
# ── Restore reasoning_config if it was saved ──
|
||||
# switch_model saves reasoning_config in _primary_runtime. If the
|
||||
# snapshot predates that (older sessions), keep the current value.
|
||||
saved_reasoning = rt.get("reasoning_config")
|
||||
if saved_reasoning is not None:
|
||||
agent.reasoning_config = dict(saved_reasoning)
|
||||
|
||||
# ── Reset fallback chain for the new turn ──
|
||||
agent._fallback_activated = False
|
||||
agent._fallback_index = 0
|
||||
|
|
@ -2065,6 +2072,45 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
api_mode=agent.api_mode,
|
||||
)
|
||||
|
||||
# ── Re-resolve reasoning_config from per-model override ──
|
||||
# The new model may have a different reasoning_effort override. Re-read
|
||||
# config so the override takes effect immediately on /model switch.
|
||||
# Try both agent.model (normalized, e.g. "claude-opus-4-5") AND the raw
|
||||
# config default (user's original spelling, e.g. "claude-opus-4.5") so
|
||||
# override keys match regardless of how downstream consumers normalized
|
||||
# the input. See plan FINDING #7 + session follow-up.
|
||||
try:
|
||||
from hermes_constants import (
|
||||
parse_reasoning_effort,
|
||||
resolve_per_model_reasoning_effort,
|
||||
)
|
||||
from hermes_cli.config import load_config as _sm_load_config
|
||||
|
||||
_reasoning_cfg = _sm_load_config() or {}
|
||||
_sm_overrides = (_reasoning_cfg.get("agent") or {}).get("reasoning_overrides", {}) or {}
|
||||
# Try the normalized agent.model first, then the raw config default
|
||||
_sm_raw_model_default = str((_reasoning_cfg.get("model") or {}).get("default", "") or "").strip()
|
||||
_sm_per_model = None
|
||||
for _candidate in (agent.model, _sm_raw_model_default):
|
||||
if _candidate:
|
||||
_sm_per_model = resolve_per_model_reasoning_effort(_candidate, _sm_overrides)
|
||||
if _sm_per_model is not None:
|
||||
break
|
||||
if _sm_per_model is not None:
|
||||
agent.reasoning_config = _sm_per_model
|
||||
logger.info(
|
||||
"switch_model: reasoning_config resolved to per-model override for %s: %s",
|
||||
agent.model, _sm_per_model,
|
||||
)
|
||||
else:
|
||||
# Raw value — a YAML boolean False means thinking disabled,
|
||||
# see parse_reasoning_effort. Do NOT str()/strip() coerce.
|
||||
_sm_global = (_reasoning_cfg.get("agent") or {}).get("reasoning_effort", "")
|
||||
agent.reasoning_config = parse_reasoning_effort(_sm_global)
|
||||
logger.info("switch_model: reasoning_config resolved to global effort: %s", _sm_global or "(none)")
|
||||
except Exception as _reasoning_err:
|
||||
logger.debug("switch_model: could not re-resolve reasoning_config: %s", _reasoning_err)
|
||||
|
||||
# ── Invalidate cached system prompt so it rebuilds next turn ──
|
||||
agent._cached_system_prompt = None
|
||||
|
||||
|
|
@ -2087,6 +2133,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
"client_kwargs": dict(agent._client_kwargs),
|
||||
"use_prompt_caching": agent._use_prompt_caching,
|
||||
"use_native_cache_layout": agent._use_native_cache_layout,
|
||||
"reasoning_config": dict(agent.reasoning_config) if getattr(agent, "reasoning_config", None) else None,
|
||||
"compressor_model": getattr(_cc, "model", agent.model) if _cc else agent.model,
|
||||
"compressor_base_url": getattr(_cc, "base_url", agent.base_url) if _cc else agent.base_url,
|
||||
"compressor_api_key": getattr(_cc, "api_key", "") if _cc else "",
|
||||
|
|
|
|||
|
|
@ -1635,6 +1635,45 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
api_mode=agent.api_mode,
|
||||
)
|
||||
|
||||
# Re-resolve reasoning_config for the new fallback model (Closes #21256).
|
||||
# Per-model override (if any) takes precedence, else global reasoning_effort.
|
||||
# Wrapped in try/except because config load failure must not kill the swap.
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort
|
||||
|
||||
_fb_cfg = load_config() or {}
|
||||
_fb_agent_cfg = _fb_cfg.get("agent", {}) or {}
|
||||
_fb_overrides = _fb_agent_cfg.get("reasoning_overrides", {}) or {}
|
||||
_fb_per_model = resolve_per_model_reasoning_effort(agent.model, _fb_overrides)
|
||||
if _fb_per_model is not None:
|
||||
agent.reasoning_config = _fb_per_model
|
||||
logger.info(
|
||||
"Fallback %s: reasoning_config resolved to per-model override: %s",
|
||||
agent.model, _fb_per_model,
|
||||
)
|
||||
else:
|
||||
# Raw value — a YAML boolean False means thinking disabled,
|
||||
# see parse_reasoning_effort. Do NOT coerce with ``or ""``.
|
||||
_fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "")
|
||||
agent.reasoning_config = parse_reasoning_effort(_fb_global_effort)
|
||||
if agent.reasoning_config:
|
||||
logger.info(
|
||||
"Fallback %s: reasoning_config resolved to global effort: %s",
|
||||
agent.model, _fb_global_effort,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Fallback %s: reasoning_config resolved to None (disabled or default)",
|
||||
agent.model,
|
||||
)
|
||||
except Exception as _reasoning_err:
|
||||
logger.debug(
|
||||
"Failed to resolve reasoning_config for fallback %s; keeping current: %s",
|
||||
agent.model, _reasoning_err,
|
||||
)
|
||||
# Keep whatever reasoning_config was active — don't break the fallback swap.
|
||||
|
||||
# Keep the prompt's self-identity in sync with the model actually
|
||||
# answering, so "what model are you?" doesn't report the primary.
|
||||
rewrite_prompt_model_identity(agent, fb_model, fb_provider)
|
||||
|
|
|
|||
|
|
@ -742,6 +742,19 @@ agent:
|
|||
# Options: "xhigh" (max), "high", "medium", "low", "minimal", "none" (disable)
|
||||
reasoning_effort: "medium"
|
||||
|
||||
# Per-model reasoning effort overrides (optional dict)
|
||||
# Key: any sensible model spelling works (exact, dots↔dashes interchangeable,
|
||||
# provider prefix optional). First match wins.
|
||||
# Value: reasoning effort level (same options as reasoning_effort)
|
||||
# Override the global reasoning_effort for that specific model.
|
||||
# NOTE: no `hermes config set` support for this key -- edit YAML directly.
|
||||
# reasoning_overrides:
|
||||
# "openrouter/anthropic/claude-opus-4.5": "xhigh"
|
||||
# "openai/gpt-5": "low"
|
||||
# "claude-opus-4.6": "high" # bare model name also works
|
||||
# "deepseek/deepseek-v4-pro": "xhigh" # dots and dashes are interchangeable
|
||||
reasoning_overrides: {}
|
||||
|
||||
# Predefined personalities (use with /personality command)
|
||||
personalities:
|
||||
helpful: "You are a helpful, friendly AI assistant."
|
||||
|
|
|
|||
14
cli.py
14
cli.py
|
|
@ -3917,8 +3917,18 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin):
|
|||
)
|
||||
|
||||
# Reasoning config (OpenRouter reasoning effort level)
|
||||
self.reasoning_config = _parse_reasoning_config(
|
||||
CLI_CONFIG["agent"].get("reasoning_effort", "")
|
||||
# Per-model override takes precedence over global effort (Closes #21256).
|
||||
_reasoning_overrides = CLI_CONFIG["agent"].get("reasoning_overrides", {}) or {}
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
_per_model_reasoning = resolve_per_model_reasoning_effort(
|
||||
self.model, _reasoning_overrides
|
||||
)
|
||||
self.reasoning_config = (
|
||||
_per_model_reasoning
|
||||
if _per_model_reasoning is not None
|
||||
else _parse_reasoning_config(
|
||||
CLI_CONFIG["agent"].get("reasoning_effort", "")
|
||||
)
|
||||
)
|
||||
self.service_tier = _parse_service_tier_config(
|
||||
CLI_CONFIG["agent"].get("service_tier", "")
|
||||
|
|
|
|||
|
|
@ -2988,12 +2988,25 @@ def run_job(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# Reasoning config from config.yaml (raw value — a YAML boolean False
|
||||
# means thinking disabled, see parse_reasoning_effort)
|
||||
from hermes_constants import parse_reasoning_effort
|
||||
reasoning_config = parse_reasoning_effort(
|
||||
_cfg.get("agent", {}).get("reasoning_effort", "")
|
||||
# Reasoning config from config.yaml (per-model override > global)
|
||||
from hermes_constants import (
|
||||
parse_reasoning_effort,
|
||||
resolve_per_model_reasoning_effort,
|
||||
)
|
||||
_cron_model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {}
|
||||
_cron_model = str(
|
||||
_cron_model_cfg.get("default", "") or _cron_model_cfg.get("model", "") or ""
|
||||
).strip()
|
||||
_cron_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {}
|
||||
_cron_per_model = resolve_per_model_reasoning_effort(_cron_model, _cron_overrides)
|
||||
if _cron_per_model is not None:
|
||||
reasoning_config = _cron_per_model
|
||||
else:
|
||||
# Raw value — a YAML boolean False means thinking disabled,
|
||||
# see parse_reasoning_effort. Do NOT str()/strip() coerce.
|
||||
reasoning_config = parse_reasoning_effort(
|
||||
_cfg.get("agent", {}).get("reasoning_effort", "")
|
||||
)
|
||||
|
||||
# Prefill messages from env or config.yaml. The top-level
|
||||
# prefill_messages_file key is canonical; agent.prefill_messages_file is
|
||||
|
|
|
|||
101
docs/PER_MODEL_REASONING.md
Normal file
101
docs/PER_MODEL_REASONING.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# Hermes Agent Configuration Guide
|
||||
|
||||
## Per-Model Reasoning Effort Overrides
|
||||
|
||||
You can configure different reasoning effort levels for different models. This allows you to set `high` effort for complex reasoning models like `claude-opus-4.5` while keeping `medium` for faster models like `gemini-flash`.
|
||||
|
||||
### Configuration
|
||||
|
||||
Edit your `config.yaml` (typically at `~/.hermes/config.yaml`):
|
||||
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_overrides:
|
||||
claude-opus-4.5: high
|
||||
gemini-flash: medium
|
||||
gpt-4.5: high
|
||||
```
|
||||
|
||||
### Key Matching
|
||||
|
||||
The model name matching is **spelling-tolerant**. All of these variations will match:
|
||||
- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5`
|
||||
- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5`
|
||||
- With or without provider prefixes
|
||||
|
||||
Exact matches take precedence over variants.
|
||||
|
||||
### Resolution Order
|
||||
|
||||
When determining reasoning effort for a model, Hermes checks in this order:
|
||||
|
||||
1. **Session override**: `/reasoning high` (current session only)
|
||||
2. **Per-model override**: `agent.reasoning_overrides.<model>` from config.yaml
|
||||
3. **Global default**: `agent.reasoning_effort` from config.yaml
|
||||
|
||||
### How It Works
|
||||
|
||||
The override applies automatically in these scenarios:
|
||||
|
||||
- **CLI startup**: Uses the override for the configured default model
|
||||
- **Gateway messaging**: Each gateway session uses the override for its model
|
||||
- **Desktop/TUI**: Uses the override for the configured model
|
||||
- **Model switching**: When you switch models, the reasoning effort updates to the new model's override
|
||||
- **Fallback activation**: When the primary model fails and Hermes falls back to a secondary model, it uses that fallback model's override
|
||||
- **Reasoning recovery**: When the primary model recovers after a fallback, the original model's override is restored
|
||||
|
||||
### Examples
|
||||
|
||||
#### Example 1: High effort for Opus, medium for others
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_overrides:
|
||||
claude-opus-4.5: high
|
||||
```
|
||||
|
||||
#### Example 2: Different efforts per model
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_overrides:
|
||||
claude-opus-4.5: high
|
||||
gemini-2.0-flash: low
|
||||
gpt-4.5: high
|
||||
o3-mini: medium
|
||||
```
|
||||
|
||||
#### Example 3: With provider prefixes
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_overrides:
|
||||
anthropic/claude-opus-4.5: high
|
||||
google/gemini-2.0-flash: low
|
||||
```
|
||||
|
||||
All of these are equivalent — the provider prefix is optional.
|
||||
|
||||
### Disabling Reasoning for Specific Models
|
||||
|
||||
Set the override to `none` to disable reasoning for a specific model:
|
||||
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_overrides:
|
||||
gemini-flash: none
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
**Override not taking effect?**
|
||||
- Check the exact model name in your config with `/model`
|
||||
- Verify the override is under `agent.reasoning_overrides` (not `agent.reasoning_effort`)
|
||||
- Restart the gateway or CLI session after editing config.yaml
|
||||
- Check logs for parsing errors
|
||||
|
||||
**Override applies but reasoning doesn't work?**
|
||||
- Not all models support reasoning (e.g., `gemini-flash` has limited support)
|
||||
- Check the model's documentation for reasoning capability
|
||||
- Use a model that explicitly supports extended thinking
|
||||
|
||||
**Session override not respecting per-model override?**
|
||||
- Session overrides take precedence (by design)
|
||||
- Clear the session override with `/reasoning default` to return to the per-model override
|
||||
|
|
@ -4852,17 +4852,33 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
|
||||
@staticmethod
|
||||
def _load_reasoning_config() -> dict | None:
|
||||
"""Load reasoning effort from config.yaml.
|
||||
"""Load reasoning effort from config.yaml, respecting per-model overrides.
|
||||
|
||||
Reads agent.reasoning_effort from config.yaml. Valid: "none",
|
||||
"minimal", "low", "medium", "high", "xhigh", "max", "ultra". Returns None to use
|
||||
default (medium).
|
||||
|
||||
Per-model overrides (agent.reasoning_overrides) take precedence
|
||||
over the global value when the current model matches a key
|
||||
(spelling-tolerant). Closes #21256.
|
||||
"""
|
||||
from hermes_constants import parse_reasoning_effort
|
||||
from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort
|
||||
cfg = _load_gateway_runtime_config()
|
||||
# Keep the raw value — coercing with ``or ""`` turns a YAML boolean
|
||||
# False (``reasoning_effort: false``/``off``/``no``) into "", silently
|
||||
# re-enabling thinking for users who explicitly disabled it.
|
||||
# Per-model override first
|
||||
model_cfg = cfg.get("model") or {}
|
||||
model = str(
|
||||
(model_cfg.get("default", "") if isinstance(model_cfg, dict) else "")
|
||||
or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "")
|
||||
or ""
|
||||
).strip()
|
||||
overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {}
|
||||
per_model = resolve_per_model_reasoning_effort(model, overrides)
|
||||
if per_model is not None:
|
||||
return per_model
|
||||
# Global fallback — keep the raw value; coercing with ``or ""`` turns
|
||||
# a YAML boolean False (``reasoning_effort: false``/``off``/``no``)
|
||||
# into "", silently re-enabling thinking for users who explicitly
|
||||
# disabled it.
|
||||
effort = cfg_get(cfg, "agent", "reasoning_effort", default="")
|
||||
result = parse_reasoning_effort(effort)
|
||||
if effort and str(effort).strip() and result is None:
|
||||
|
|
|
|||
|
|
@ -1158,8 +1158,15 @@ DEFAULT_CONFIG = {
|
|||
# only controls how inbound user images are presented.
|
||||
"image_input_mode": "auto",
|
||||
"disabled_toolsets": [],
|
||||
|
||||
# Per-model reasoning effort overrides (spelling-tolerant).
|
||||
# Dict mapping model names (any reasonable spelling) to effort levels.
|
||||
# Takes precedence over agent.reasoning_effort when the current model
|
||||
# matches a key in this dict.
|
||||
# Edit directly in config.yaml (no CLI support due to dots in keys).
|
||||
"reasoning_overrides": {},
|
||||
},
|
||||
|
||||
|
||||
"terminal": {
|
||||
"backend": "local",
|
||||
"modal_mode": "auto",
|
||||
|
|
|
|||
|
|
@ -823,6 +823,131 @@ def parse_reasoning_effort(effort) -> dict | None:
|
|||
return None
|
||||
|
||||
|
||||
def _canonical_model_variants(model: str) -> list[str]:
|
||||
"""Generate bounded spelling variants for tolerant override matching.
|
||||
|
||||
Model names mix two types of separators:
|
||||
- **Word separators**: dashes between words (``claude-opus``)
|
||||
- **Version separators**: dots or dashes between version digits (``4.5``, ``4-5``)
|
||||
|
||||
The tricky case is that ``.`` appears in BOTH roles (word sep in some
|
||||
spellings, version sep in others), so a blanket ``.replace('.', '-')``
|
||||
is lossy — it collapses version dots into dashes and no later step
|
||||
recovers the canonical form (``claude-opus-4.5``).
|
||||
|
||||
Strategy: generate a small set of base forms, then apply version-dot
|
||||
recovery to EACH of them. This ensures symmetry:
|
||||
``claude-opus-4.5``, ``claude-opus-4-5``, and ``claude-opus.4.5`` all
|
||||
produce the same variant set.
|
||||
|
||||
Steps:
|
||||
1. Exact input
|
||||
2. Dots/dashes cross-substitution on the entire string
|
||||
3. Version-dot recovery applied to ALL derivatives
|
||||
4. Strip provider/aggregator prefix → bare model variants
|
||||
5. Apply version-dot recovery to bare derivatives
|
||||
6. Prepend known provider/aggregator prefixes
|
||||
|
||||
Duplicates removed in insertion order (exact always wins).
|
||||
"""
|
||||
import re
|
||||
|
||||
# Version-dot regexes — digit-separator-digit interconversion
|
||||
_dash_to_dot = lambda s: re.sub(r'(\d)-(\d)', r'\1.\2', s)
|
||||
_dot_to_dash = lambda s: re.sub(r'(\d)\.(\d)', r'\1-\2', s)
|
||||
|
||||
seen = set()
|
||||
variants = []
|
||||
|
||||
def _add(v):
|
||||
if v and v not in seen:
|
||||
seen.add(v)
|
||||
variants.append(v)
|
||||
|
||||
def _add_with_derivatives(s):
|
||||
"""Add s plus its dots↔dashes and version-dot derivatives."""
|
||||
_add(s)
|
||||
all_dashed = s.replace('.', '-')
|
||||
_add(all_dashed)
|
||||
all_dotted = s.replace('-', '.')
|
||||
_add(all_dotted)
|
||||
# Version-dot recovery on each base form
|
||||
_add(_dash_to_dot(s))
|
||||
_add(_dot_to_dash(s))
|
||||
_add(_dash_to_dot(all_dashed))
|
||||
_add(_dot_to_dash(all_dotted))
|
||||
|
||||
# 1-3. Base variants for the full string
|
||||
_add_with_derivatives(model)
|
||||
|
||||
# Split by / to handle provider prefix
|
||||
parts = model.split('/')
|
||||
|
||||
# 4. Bare model variants (strip provider/aggregator prefix)
|
||||
if len(parts) >= 2:
|
||||
bare = parts[-1]
|
||||
_add_with_derivatives(bare)
|
||||
|
||||
# Strip aggregator only (3+ parts)
|
||||
# e.g. "openrouter/anthropic/claude-opus-4.5" → "anthropic/claude-opus-4.5"
|
||||
if len(parts) >= 3:
|
||||
_add_with_derivatives('/'.join(parts[1:]))
|
||||
|
||||
# 5. Prepend known provider prefixes to bare variants
|
||||
known_providers = (
|
||||
'anthropic', 'openai', 'google', 'openrouter', 'groq', 'mistral',
|
||||
'xai', 'cohere', 'perplexity', 'together', 'fireworks', 'deepseek',
|
||||
)
|
||||
bare_variants = [v for v in variants if '/' not in v]
|
||||
for v in bare_variants:
|
||||
for provider in known_providers:
|
||||
_add(f"{provider}/{v}")
|
||||
|
||||
# Prepend aggregator to single-slash variants
|
||||
single_slash_variants = [v for v in variants if v.count('/') == 1]
|
||||
known_aggregators = ('openrouter', 'opencode', 'fireworks', 'groq', 'together')
|
||||
for v in single_slash_variants:
|
||||
for agg in known_aggregators:
|
||||
_add(f"{agg}/{v}")
|
||||
|
||||
return variants
|
||||
|
||||
|
||||
def resolve_per_model_reasoning_effort(model: str, overrides: dict | None) -> dict | None:
|
||||
"""Lookup a per-model reasoning_effort override with spelling-tolerance.
|
||||
|
||||
Args:
|
||||
model: The model string (any spelling — exact, normalized, bare,
|
||||
with provider prefix, etc.)
|
||||
overrides: The dict of per-model overrides from
|
||||
agent.reasoning_overrides in config.yaml. Keys can be
|
||||
any sensible spelling of the model name.
|
||||
|
||||
Returns:
|
||||
The parsed reasoning_config dict if a match is found,
|
||||
None otherwise (caller should fall back to global reasoning_effort).
|
||||
|
||||
Resolution order:
|
||||
1. Exact match
|
||||
2. Dots ↔ dashes variants
|
||||
3. Strip provider prefix (bare model name only)
|
||||
4. Strip aggregator prefix (middle segment only)
|
||||
5. Prepend known aggregator prefixes to bare/single-slash variants
|
||||
|
||||
First non-None parse_reasoning_effort result wins.
|
||||
"""
|
||||
if not overrides or not isinstance(overrides, dict) or not model:
|
||||
return None
|
||||
|
||||
for variant in _canonical_model_variants(model):
|
||||
if variant in overrides:
|
||||
result = parse_reasoning_effort(overrides[variant])
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_termux() -> bool:
|
||||
"""Return True when running inside a Termux (Android) environment.
|
||||
|
||||
|
|
|
|||
106
tests/cron/test_reasoning_config_per_model.py
Normal file
106
tests/cron/test_reasoning_config_per_model.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Tests for per-model reasoning_effort override in cron scheduler."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestCronPerModelReasoningConfig:
|
||||
"""Test cron scheduler respects per-model reasoning overrides.
|
||||
|
||||
Rather than spinning up a full CronScheduler (heavy), we verify the
|
||||
resolution logic by testing the helper directly against a config dict
|
||||
shaped the same way the scheduler reads it.
|
||||
"""
|
||||
|
||||
def test_per_model_override_resolves_for_cron_model(self):
|
||||
"""The spelling-tolerant helper resolves the cron config's model."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
|
||||
# Simulate cron scheduler config shape
|
||||
_cfg = {
|
||||
"model": {"default": "anthropic/claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
_model_cfg = _cfg.get("model", {})
|
||||
_model = str(_model_cfg.get("default", "") or "").strip()
|
||||
_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {}
|
||||
|
||||
result = resolve_per_model_reasoning_effort(_model, _overrides)
|
||||
assert result is not None
|
||||
assert result["effort"] == "xhigh"
|
||||
|
||||
def test_cron_falls_back_to_global_when_no_override(self):
|
||||
"""When no per-model override matches, global effort is used."""
|
||||
from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort
|
||||
|
||||
_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "low",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
_model = _cfg["model"]["default"]
|
||||
_overrides = _cfg["agent"]["reasoning_overrides"]
|
||||
|
||||
per_model = resolve_per_model_reasoning_effort(_model, _overrides)
|
||||
assert per_model is None # no match
|
||||
|
||||
# Scheduler falls back to global
|
||||
effort = _cfg["agent"]["reasoning_effort"]
|
||||
result = parse_reasoning_effort(effort)
|
||||
assert result is not None
|
||||
assert result["effort"] == "low"
|
||||
|
||||
def test_cron_handles_missing_model_key(self):
|
||||
"""Works when config has no model.default."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
|
||||
_cfg = {
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {"claude-opus-4.5": "high"},
|
||||
},
|
||||
}
|
||||
_model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {}
|
||||
_model = str(_model_cfg.get("default", "") or _model_cfg.get("model", "") or "").strip()
|
||||
_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {}
|
||||
|
||||
# Empty model → resolve returns None → scheduler uses global
|
||||
result = resolve_per_model_reasoning_effort(_model, _overrides)
|
||||
assert result is None
|
||||
|
||||
def test_global_fallback_with_yaml_false(self):
|
||||
"""YAML boolean False must reach parse_reasoning_effort uncoerced.
|
||||
|
||||
Regression: str(... or "").strip() turned False into "", silently
|
||||
re-enabling thinking. The raw value must pass through so
|
||||
parse_reasoning_effort(False) returns {'enabled': False}.
|
||||
"""
|
||||
from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort
|
||||
|
||||
_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": False, # YAML boolean, not string
|
||||
"reasoning_overrides": {"claude-opus-4.5": "xhigh"},
|
||||
},
|
||||
}
|
||||
_model = _cfg["model"]["default"]
|
||||
_overrides = _cfg["agent"]["reasoning_overrides"]
|
||||
|
||||
per_model = resolve_per_model_reasoning_effort(_model, _overrides)
|
||||
assert per_model is None # no match
|
||||
|
||||
# Scheduler global fallback — raw value, no coercion
|
||||
result = parse_reasoning_effort(
|
||||
_cfg.get("agent", {}).get("reasoning_effort", "")
|
||||
)
|
||||
assert result is not None
|
||||
assert result.get("enabled") is False
|
||||
111
tests/gateway/test_reasoning_config_per_model.py
Normal file
111
tests/gateway/test_reasoning_config_per_model.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
"""Tests for per-model reasoning_effort override in gateway _load_reasoning_config."""
|
||||
|
||||
import pytest
|
||||
|
||||
import gateway.run as gateway_run
|
||||
|
||||
|
||||
class TestGatewayPerModelReasoningConfig:
|
||||
"""Test GatewayRunner._load_reasoning_config respects per-model overrides."""
|
||||
|
||||
def test_per_model_override_takes_precedence(self, monkeypatch):
|
||||
"""Per-model override wins over global reasoning_effort."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "anthropic/claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["enabled"] is True
|
||||
assert result["effort"] == "xhigh"
|
||||
|
||||
def test_global_fallback_when_no_override(self, monkeypatch):
|
||||
"""Global reasoning_effort applies when no per-model override matches."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "high",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "high"
|
||||
|
||||
def test_spelling_tolerant_match_in_gateway(self, monkeypatch):
|
||||
"""Override matches even with different spelling (dots vs dashes)."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "claude-opus-4-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"claude-opus-4.5": "xhigh", # key has dots, model has dashes
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "xhigh"
|
||||
|
||||
def test_no_overrides_dict(self, monkeypatch):
|
||||
"""Works fine when reasoning_overrides key is absent."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "low",
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "low"
|
||||
|
||||
def test_empty_overrides(self, monkeypatch):
|
||||
"""Empty overrides dict falls back to global."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "medium"
|
||||
|
||||
def test_global_fallback_with_yaml_false(self, monkeypatch):
|
||||
"""YAML boolean False must reach parse_reasoning_effort uncoerced.
|
||||
|
||||
Regression: str(... or "").strip() turned False into "", silently
|
||||
re-enabling thinking. The raw value must pass through so
|
||||
parse_reasoning_effort(False) returns {'enabled': False}.
|
||||
"""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": False, # YAML boolean, not string
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result.get("enabled") is False
|
||||
144
tests/run_agent/test_fallback_reasoning_override.py
Normal file
144
tests/run_agent/test_fallback_reasoning_override.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
"""Tests for per-model reasoning_effort override during fallback activation.
|
||||
|
||||
Tests that try_activate_fallback re-resolves reasoning_config when
|
||||
swapping to a fallback model, so per-model overrides are honored even
|
||||
during error recovery.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestFallbackReasoningOverride:
|
||||
"""Test try_activate_fallback re-resolves reasoning_config."""
|
||||
|
||||
def test_fallback_re_resolves_reasoning_config(self):
|
||||
"""When fallback activates, reasoning_config should be re-resolved.
|
||||
|
||||
We test the resolution logic directly rather than spinning up a
|
||||
full try_activate_fallback (which requires extensive agent setup).
|
||||
The production code calls resolve_per_model_reasoning_effort with
|
||||
the fallback model string — we verify that works correctly.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
|
||||
# Simulate: primary was gemini-flash (medium), fallback to claude-opus-4.5 (xhigh)
|
||||
overrides = {
|
||||
"claude-opus-4.5": "xhigh",
|
||||
"gemini-flash": "medium",
|
||||
}
|
||||
|
||||
# Fallback model lookup
|
||||
fb_result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides)
|
||||
assert fb_result is not None
|
||||
assert fb_result["effort"] == "xhigh"
|
||||
|
||||
# Primary model lookup (for comparison)
|
||||
primary_result = resolve_per_model_reasoning_effort("gemini-flash", overrides)
|
||||
assert primary_result is not None
|
||||
assert primary_result["effort"] == "medium"
|
||||
|
||||
# The key point: fallback result differs from primary
|
||||
assert fb_result["effort"] != primary_result["effort"]
|
||||
|
||||
def test_fallback_to_model_without_override_uses_global(self):
|
||||
"""Fallback to a model with no override should resolve to None (→ global)."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
|
||||
overrides = {"claude-opus-4.5": "xhigh"}
|
||||
|
||||
# Fallback to gpt-5 which has no override
|
||||
result = resolve_per_model_reasoning_effort("gpt-5", overrides)
|
||||
assert result is None # caller falls back to global
|
||||
|
||||
def test_fallback_with_normalized_model_name(self):
|
||||
"""Fallback model name may be normalized (dots→dashes); override should still match."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
|
||||
# User wrote key with dots, but normalize_model_for_provider converts to dashes
|
||||
overrides = {"claude-sonnet-4.6": "high"}
|
||||
|
||||
result = resolve_per_model_reasoning_effort("claude-sonnet-4-6", overrides)
|
||||
assert result is not None
|
||||
assert result["effort"] == "high"
|
||||
|
||||
def test_fallback_recovery_restores_primary_reasoning(self):
|
||||
"""After fallback + restore_primary_runtime, reasoning_config returns to primary's value.
|
||||
|
||||
This tests the integration of Task 6 (_primary_runtime snapshot) with
|
||||
Task 6b (fallback re-resolution). The full cycle:
|
||||
1. Primary model = gemini-flash, reasoning = medium
|
||||
2. /model switch → _primary_runtime captures reasoning_config
|
||||
3. Fallback activates → reasoning re-resolved for fallback model
|
||||
4. restore_primary_runtime → reasoning_config restored from snapshot
|
||||
"""
|
||||
from agent.agent_runtime_helpers import restore_primary_runtime
|
||||
|
||||
agent = MagicMock()
|
||||
# Simulate: _primary_runtime was captured during /model switch
|
||||
agent._primary_runtime = {
|
||||
"model": "gemini-flash",
|
||||
"provider": "google",
|
||||
"base_url": "",
|
||||
"api_mode": "openai",
|
||||
"api_key": "key",
|
||||
"client_kwargs": {},
|
||||
"use_prompt_caching": False,
|
||||
"use_native_cache_layout": False,
|
||||
"reasoning_config": {"enabled": True, "effort": "medium"},
|
||||
"compressor_model": "gemini-flash",
|
||||
"compressor_base_url": "",
|
||||
"compressor_api_key": "",
|
||||
"compressor_provider": "",
|
||||
"compressor_context_length": 0,
|
||||
"compressor_api_mode": "",
|
||||
"compressor_threshold_tokens": 0,
|
||||
}
|
||||
agent._fallback_activated = True
|
||||
agent._fallback_index = 0
|
||||
agent._fallback_chain = []
|
||||
agent._fallback_model = None
|
||||
agent._transport_cache = {}
|
||||
agent._config_context_length = None
|
||||
agent._rate_limited_until = 0
|
||||
# During fallback, reasoning was changed to xhigh (fallback model's override)
|
||||
agent.model = "claude-opus-4.5"
|
||||
agent.provider = "anthropic"
|
||||
agent.reasoning_config = {"enabled": True, "effort": "xhigh"}
|
||||
agent.context_compressor = MagicMock()
|
||||
agent.base_url = ""
|
||||
agent._anthropic_prompt_cache_policy = MagicMock(return_value=(False, False))
|
||||
agent._create_openai_client = MagicMock(return_value=MagicMock())
|
||||
agent._ensure_lmstudio_runtime_loaded = MagicMock()
|
||||
|
||||
result = restore_primary_runtime(agent)
|
||||
assert result is True
|
||||
# reasoning_config should be restored to primary's value (medium)
|
||||
assert agent.reasoning_config == {"enabled": True, "effort": "medium"}
|
||||
|
||||
def test_fallback_global_fallback_with_yaml_false(self):
|
||||
"""Fallback global fallback must not coerce YAML boolean False.
|
||||
|
||||
Regression: ``or ""`` turned False into "", silently re-enabling
|
||||
thinking. The raw value must pass through so
|
||||
parse_reasoning_effort(False) returns {'enabled': False}.
|
||||
|
||||
The production code in try_activate_fallback does:
|
||||
_fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "")
|
||||
agent.reasoning_config = parse_reasoning_effort(_fb_global_effort)
|
||||
We verify that passing the raw False (not coerced "") produces
|
||||
the disabled config.
|
||||
"""
|
||||
from hermes_constants import parse_reasoning_effort
|
||||
|
||||
# Simulate: no per-model override matches, global is YAML False
|
||||
_fb_agent_cfg = {"reasoning_effort": False}
|
||||
|
||||
# This is the exact line from try_activate_fallback's else branch.
|
||||
# The bug was: _fb_global_effort = _fb_agent_cfg.get(...) or ""
|
||||
# which turned False into "". The fix passes the raw value.
|
||||
_fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "")
|
||||
result = parse_reasoning_effort(_fb_global_effort)
|
||||
|
||||
assert result is not None
|
||||
assert result.get("enabled") is False
|
||||
220
tests/run_agent/test_switch_model_reasoning_override.py
Normal file
220
tests/run_agent/test_switch_model_reasoning_override.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
"""Tests for per-model reasoning_effort override during /model switch.
|
||||
|
||||
Tests that switch_model:
|
||||
1. Re-resolves reasoning_config when switching to a model with an override
|
||||
2. Falls back to global when switching to a model without an override
|
||||
3. Saves reasoning_config into _primary_runtime for fallback recovery
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestSwitchModelReasoningOverride:
|
||||
"""Test switch_model re-resolves reasoning_config on model switch."""
|
||||
|
||||
def _make_fake_agent(self, model="gpt-5", provider="openai"):
|
||||
"""Create a minimal fake agent for switch_model testing."""
|
||||
agent = MagicMock()
|
||||
agent.model = model
|
||||
agent.provider = provider
|
||||
agent.base_url = "https://api.openai.com/v1"
|
||||
agent.api_mode = "openai"
|
||||
agent.api_key = "test-key"
|
||||
agent._client_kwargs = {"api_key": "test-key", "base_url": "https://api.openai.com/v1"}
|
||||
agent._use_prompt_caching = False
|
||||
agent._use_native_cache_layout = False
|
||||
agent.reasoning_config = {"enabled": True, "effort": "medium"}
|
||||
agent._fallback_activated = False
|
||||
agent._fallback_index = 0
|
||||
agent._fallback_chain = []
|
||||
agent._fallback_model = None
|
||||
agent._config_context_length = None
|
||||
agent._transport_cache = {}
|
||||
agent.context_compressor = None
|
||||
agent._cached_system_prompt = None
|
||||
agent._anthropic_api_key = ""
|
||||
agent._anthropic_base_url = None
|
||||
agent._is_anthropic_oauth = False
|
||||
agent._anthropic_prompt_cache_policy = MagicMock(
|
||||
return_value=(False, False)
|
||||
)
|
||||
agent._ensure_lmstudio_runtime_loaded = MagicMock()
|
||||
agent._create_openai_client = MagicMock(return_value=MagicMock())
|
||||
return agent
|
||||
|
||||
def test_primary_runtime_includes_reasoning_config(self):
|
||||
"""After switch_model, _primary_runtime should contain reasoning_config key."""
|
||||
from agent.agent_runtime_helpers import switch_model
|
||||
|
||||
agent = self._make_fake_agent()
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
with patch("hermes_cli.config.load_config", return_value=fake_cfg):
|
||||
try:
|
||||
switch_model(
|
||||
agent,
|
||||
new_model="claude-opus-4.5",
|
||||
new_provider="anthropic",
|
||||
base_url="https://api.anthropic.com",
|
||||
api_mode="anthropic_messages",
|
||||
)
|
||||
except Exception:
|
||||
# Client creation may fail in test env; check _primary_runtime was set
|
||||
pass
|
||||
|
||||
assert hasattr(agent, "_primary_runtime")
|
||||
assert "reasoning_config" in agent._primary_runtime
|
||||
|
||||
def test_reasoning_config_resolves_to_override_on_switch(self):
|
||||
"""switch_model should resolve reasoning_config to per-model override."""
|
||||
from agent.agent_runtime_helpers import switch_model
|
||||
|
||||
agent = self._make_fake_agent()
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
with patch("hermes_cli.config.load_config", return_value=fake_cfg):
|
||||
try:
|
||||
switch_model(
|
||||
agent,
|
||||
new_model="claude-opus-4.5",
|
||||
new_provider="anthropic",
|
||||
base_url="https://api.anthropic.com",
|
||||
api_mode="anthropic_messages",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# reasoning_config should be updated to xhigh
|
||||
assert agent.reasoning_config is not None
|
||||
assert agent.reasoning_config.get("effort") == "xhigh"
|
||||
|
||||
def test_reasoning_config_falls_back_to_global(self):
|
||||
"""switch_model should fall back to global when no override for new model."""
|
||||
from agent.agent_runtime_helpers import switch_model
|
||||
|
||||
agent = self._make_fake_agent()
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "low",
|
||||
"reasoning_overrides": {
|
||||
"claude-opus-4.5": "xhigh", # override for different model
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
with patch("hermes_cli.config.load_config", return_value=fake_cfg):
|
||||
try:
|
||||
switch_model(
|
||||
agent,
|
||||
new_model="gpt-5",
|
||||
new_provider="openai",
|
||||
api_mode="openai",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# No override for gpt-5 → should fall back to global "low"
|
||||
assert agent.reasoning_config is not None
|
||||
assert agent.reasoning_config.get("effort") == "low"
|
||||
|
||||
def test_restore_primary_runtime_restores_reasoning(self):
|
||||
"""restore_primary_runtime should restore reasoning_config from snapshot."""
|
||||
from agent.agent_runtime_helpers import restore_primary_runtime
|
||||
|
||||
agent = MagicMock()
|
||||
agent._primary_runtime = {
|
||||
"model": "claude-opus-4.5",
|
||||
"provider": "anthropic",
|
||||
"base_url": "https://api.anthropic.com",
|
||||
"api_mode": "anthropic_messages",
|
||||
"api_key": "key",
|
||||
"client_kwargs": {},
|
||||
"use_prompt_caching": True,
|
||||
"use_native_cache_layout": False,
|
||||
"reasoning_config": {"enabled": True, "effort": "xhigh"},
|
||||
"compressor_model": "claude-opus-4.5",
|
||||
"compressor_base_url": "",
|
||||
"compressor_api_key": "",
|
||||
"compressor_provider": "",
|
||||
"compressor_context_length": 0,
|
||||
"compressor_api_mode": "",
|
||||
"compressor_threshold_tokens": 0,
|
||||
"anthropic_api_key": "key",
|
||||
"anthropic_base_url": "https://api.anthropic.com",
|
||||
"is_anthropic_oauth": False,
|
||||
}
|
||||
agent._fallback_activated = True
|
||||
agent._fallback_index = 0
|
||||
agent._fallback_chain = []
|
||||
agent._fallback_model = None
|
||||
agent._transport_cache = {}
|
||||
agent._config_context_length = None
|
||||
agent._rate_limited_until = 0
|
||||
agent.model = "fallback-model"
|
||||
agent.provider = "openai"
|
||||
agent.reasoning_config = {"enabled": True, "effort": "medium"}
|
||||
agent.context_compressor = MagicMock()
|
||||
agent.base_url = ""
|
||||
# Mock the methods restore_primary_runtime calls
|
||||
agent._anthropic_prompt_cache_policy = MagicMock(return_value=(True, False))
|
||||
agent._create_openai_client = MagicMock(return_value=MagicMock())
|
||||
agent._ensure_lmstudio_runtime_loaded = MagicMock()
|
||||
|
||||
result = restore_primary_runtime(agent)
|
||||
assert result is True
|
||||
assert agent.reasoning_config == {"enabled": True, "effort": "xhigh"}
|
||||
|
||||
def test_switch_model_global_fallback_with_yaml_false(self):
|
||||
"""switch_model global fallback must not coerce YAML boolean False.
|
||||
|
||||
Regression: str(... or "").strip() turned False into "", silently
|
||||
re-enabling thinking. The raw value must pass through so
|
||||
parse_reasoning_effort(False) returns {'enabled': False}.
|
||||
"""
|
||||
from agent.agent_runtime_helpers import switch_model
|
||||
|
||||
agent = self._make_fake_agent()
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": False, # YAML boolean, not string
|
||||
"reasoning_overrides": {},
|
||||
},
|
||||
}
|
||||
|
||||
with patch("hermes_cli.config.load_config", return_value=fake_cfg):
|
||||
try:
|
||||
switch_model(
|
||||
agent,
|
||||
new_model="gpt-5",
|
||||
new_provider="openai",
|
||||
api_mode="openai",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# No override for gpt-5 → global fallback with raw False
|
||||
assert agent.reasoning_config is not None
|
||||
assert agent.reasoning_config.get("enabled") is False
|
||||
|
|
@ -490,6 +490,206 @@ class TestParseReasoningEffort:
|
|||
assert documented.issubset(set(VALID_REASONING_EFFORTS))
|
||||
|
||||
|
||||
class TestResolvePerModelReasoningEffort:
|
||||
"""Tests for resolve_per_model_reasoning_effort() — spelling-tolerant
|
||||
per-model override lookup from agent.reasoning_overrides dict.
|
||||
|
||||
Contract: the override key the user writes in config.yaml should match
|
||||
regardless of how downstream consumers normalize the model string.
|
||||
normalize_model_for_provider() converts dots to dashes and
|
||||
adds/strips provider prefixes. Our resolver tolerates these
|
||||
variations so the user's intent ("this model always gets xhigh")
|
||||
is honored no matter which code path feeds the model string.
|
||||
"""
|
||||
|
||||
def test_exact_match(self):
|
||||
"""Exact model string match returns the parsed override."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "xhigh"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "xhigh"}
|
||||
|
||||
def test_none_when_no_matching_key(self):
|
||||
"""Model not in overrides returns None (caller falls back to global)."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "xhigh"}
|
||||
assert resolve_per_model_reasoning_effort("gpt-5", overrides) is None
|
||||
|
||||
def test_none_value_returns_disabled(self):
|
||||
"""Override set to 'none' returns {'enabled': False}."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "none"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": False}
|
||||
|
||||
def test_invalid_value_returns_none(self):
|
||||
"""Override with invalid effort falls back to None (global)."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "banana"}
|
||||
assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None
|
||||
|
||||
def test_none_or_empty_overrides_returns_none(self):
|
||||
"""None or empty overrides dict returns None."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
assert resolve_per_model_reasoning_effort("claude-opus-4.5", None) is None
|
||||
assert resolve_per_model_reasoning_effort("claude-opus-4.5", {}) is None
|
||||
|
||||
def test_empty_model_returns_none(self):
|
||||
"""Empty model string returns None."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
assert resolve_per_model_reasoning_effort("", {"gpt-5": "low"}) is None
|
||||
|
||||
# --- Spelling tolerance layer ---
|
||||
|
||||
def test_dots_to_dashes_variant(self):
|
||||
"""User wrote key with dots; input comes in normalized with dashes.
|
||||
|
||||
normalize_model_for_provider converts claude-opus-4.5 → claude-opus-4-5
|
||||
for the anthropic provider. The user's override key should still match.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "xhigh"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides)
|
||||
assert result == {"enabled": True, "effort": "xhigh"}
|
||||
|
||||
def test_dashes_to_dots_variant(self):
|
||||
"""User wrote key with dashes; input comes in with dots."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4-5": "high"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_strip_provider_prefix(self):
|
||||
"""User wrote key WITH provider prefix; input comes in bare.
|
||||
|
||||
E.g. user config: model.default: claude-opus-4.5 (no prefix),
|
||||
but override key: anthropic/claude-opus-4.5.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"anthropic/claude-opus-4.5": "high"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_prepend_provider_prefix(self):
|
||||
"""User wrote key bare; input comes in WITH provider prefix.
|
||||
|
||||
E.g. user config: model.default: anthropic/claude-opus-4.5,
|
||||
but override key: claude-opus-4.5 (no prefix).
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "high"}
|
||||
result = resolve_per_model_reasoning_effort("anthropic/claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_aggregator_prefix_stripping(self):
|
||||
"""openrouter/anthropic/claude-opus-4.5 should match key anthropic/claude-opus-4.5.
|
||||
|
||||
Aggregator providers (openrouter) prepend their own name,
|
||||
creating a triple-prefix. The resolver strips the aggregator
|
||||
layer to find the user's two-segment key.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"anthropic/claude-opus-4.5": "xhigh"}
|
||||
result = resolve_per_model_reasoning_effort("openrouter/anthropic/claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "xhigh"}
|
||||
|
||||
def test_exact_match_wins_over_variant(self):
|
||||
"""Ambiguity resolution: exact match takes priority over a variant.
|
||||
|
||||
If both 'claude-opus-4.5' (exact) and 'claude-opus-4-5' (dashes
|
||||
variant) are keys, the exact input matches the exact key first.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "high", "claude-opus-4-5": "xhigh"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides)
|
||||
assert result == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_none_when_no_variant_matches(self):
|
||||
"""All variants exhausted without a match returns None."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"gpt-5": "low"}
|
||||
assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None
|
||||
|
||||
def test_all_dotted_input_matches_canonical_key(self):
|
||||
"""Regression: all-dotted input (claude-opus.4.5) must match
|
||||
canonical key (claude-opus-4.5).
|
||||
|
||||
This was a real bug found by delegate review: the old
|
||||
all_dashed = model.replace('.', '-') collapsed version dots,
|
||||
making the canonical form unreachable from all-dotted input.
|
||||
"""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"claude-opus-4.5": "xhigh"}
|
||||
result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides)
|
||||
assert result is not None
|
||||
assert result["effort"] == "xhigh"
|
||||
|
||||
def test_different_models_do_not_match(self):
|
||||
"""No false positives: gemini-2.0-flash must not match gemini-flash."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
overrides = {"gemini-flash": "low"}
|
||||
assert resolve_per_model_reasoning_effort("gemini-2.0-flash", overrides) is None
|
||||
|
||||
|
||||
class TestReasoningOverridesDefaultConfig:
|
||||
"""Tests for the agent.reasoning_overrides default config key (Task 2)."""
|
||||
|
||||
def test_default_config_has_reasoning_overrides_key(self):
|
||||
"""DEFAULT_CONFIG['agent'] contains 'reasoning_overrides' as an empty dict."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
assert "reasoning_overrides" in DEFAULT_CONFIG["agent"]
|
||||
assert DEFAULT_CONFIG["agent"]["reasoning_overrides"] == {}
|
||||
|
||||
def test_config_version_bumped(self):
|
||||
"""Config version was bumped to signal the schema change for reasoning_overrides."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
assert DEFAULT_CONFIG.get("_config_version") >= 31
|
||||
|
||||
def test_load_config_preserves_user_reasoning_overrides(self, tmp_path, monkeypatch):
|
||||
"""User-added reasoning_overrides are preserved through load_config()."""
|
||||
import yaml
|
||||
from hermes_cli.config import load_config, get_config_path
|
||||
|
||||
user_config = {
|
||||
"agent": {
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4-5": "high",
|
||||
"openrouter/anthropic/claude-sonnet-4-6": "low",
|
||||
}
|
||||
}
|
||||
}
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(yaml.safe_dump(user_config))
|
||||
|
||||
# load_config() reads from get_config_path() — patch its global reference
|
||||
monkeypatch.setitem(
|
||||
load_config.__globals__, "get_config_path", lambda: config_path
|
||||
)
|
||||
|
||||
loaded = load_config()
|
||||
assert loaded["agent"]["reasoning_overrides"] == {
|
||||
"anthropic/claude-opus-4-5": "high",
|
||||
"openrouter/anthropic/claude-sonnet-4-6": "low",
|
||||
}
|
||||
|
||||
def test_spelling_tolerant_lookup_works_with_user_config(self):
|
||||
"""resolve_per_model_reasoning_effort works with user-added overrides."""
|
||||
from hermes_constants import resolve_per_model_reasoning_effort
|
||||
# User config with one override, query uses different spelling
|
||||
overrides = {
|
||||
"anthropic/claude-opus-4.5": "xhigh", # user wrote with dots
|
||||
}
|
||||
# Lookup with different spelling (bare, dashes) — should still match
|
||||
result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides)
|
||||
assert result == {"enabled": True, "effort": "xhigh"}
|
||||
|
||||
# Another override, bare key
|
||||
overrides2 = {"gpt-5": "low"}
|
||||
# Lookup with provider prefix — should match
|
||||
result2 = resolve_per_model_reasoning_effort("openai/gpt-5", overrides2)
|
||||
assert result2 == {"enabled": True, "effort": "low"}
|
||||
|
||||
|
||||
class TestSecureParentDir:
|
||||
"""Tests for secure_parent_dir() — prevents chmod on / or top-level dirs."""
|
||||
|
||||
|
|
|
|||
100
tests/tui_gateway/test_reasoning_config_per_model.py
Normal file
100
tests/tui_gateway/test_reasoning_config_per_model.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
"""Tests for per-model reasoning_effort override in TUI gateway _load_reasoning_config."""
|
||||
|
||||
import pytest
|
||||
|
||||
import tui_gateway.server as tui_server
|
||||
|
||||
|
||||
class TestTUIPerModelReasoningConfig:
|
||||
"""Test tui_gateway _load_reasoning_config respects per-model overrides."""
|
||||
|
||||
def test_per_model_override_takes_precedence(self, monkeypatch):
|
||||
"""Per-model override wins over global reasoning_effort."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "anthropic/claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg)
|
||||
|
||||
result = tui_server._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["enabled"] is True
|
||||
assert result["effort"] == "xhigh"
|
||||
|
||||
def test_global_fallback_when_no_override(self, monkeypatch):
|
||||
"""Global reasoning_effort applies when no per-model override matches."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "high",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "xhigh",
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg)
|
||||
|
||||
result = tui_server._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "high"
|
||||
|
||||
def test_spelling_tolerant_match(self, monkeypatch):
|
||||
"""Override matches even with different spelling (provider prefix)."""
|
||||
fake_cfg = {
|
||||
"model": {"default": "claude-opus-4.5"},
|
||||
"agent": {
|
||||
"reasoning_effort": "medium",
|
||||
"reasoning_overrides": {
|
||||
"anthropic/claude-opus-4.5": "high", # key has prefix, model doesn't
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg)
|
||||
|
||||
result = tui_server._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result["effort"] == "high"
|
||||
|
||||
def test_parity_with_gateway_loader(self, monkeypatch):
|
||||
"""TUI and gateway loaders return identical results for same config."""
|
||||
import gateway.run as gateway_run
|
||||
|
||||
fake_cfg = {
|
||||
"model": {"default": "openrouter/anthropic/claude-sonnet-4.6"},
|
||||
"agent": {
|
||||
"reasoning_effort": "low",
|
||||
"reasoning_overrides": {
|
||||
"claude-sonnet-4.6": "high",
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg)
|
||||
monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg)
|
||||
|
||||
tui_result = tui_server._load_reasoning_config()
|
||||
gw_result = gateway_run.GatewayRunner._load_reasoning_config()
|
||||
assert tui_result == gw_result
|
||||
|
||||
def test_global_fallback_with_yaml_false(self, monkeypatch):
|
||||
"""YAML boolean False must reach parse_reasoning_effort uncoerced.
|
||||
|
||||
Regression: str(... or "").strip() turned False into "", silently
|
||||
re-enabling thinking. The raw value must pass through so
|
||||
parse_reasoning_effort(False) returns {'enabled': False}.
|
||||
"""
|
||||
fake_cfg = {
|
||||
"model": {"default": "gpt-5"},
|
||||
"agent": {
|
||||
"reasoning_effort": False, # YAML boolean, not string
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg)
|
||||
|
||||
result = tui_server._load_reasoning_config()
|
||||
assert result is not None
|
||||
assert result.get("enabled") is False
|
||||
|
|
@ -2563,13 +2563,33 @@ def _display_mouse_tracking(display: dict) -> str:
|
|||
|
||||
|
||||
def _load_reasoning_config() -> dict | None:
|
||||
from hermes_constants import parse_reasoning_effort
|
||||
"""Load reasoning effort from config.yaml, respecting per-model overrides.
|
||||
|
||||
# Pass the raw value through — ``or ""`` would coerce a YAML boolean
|
||||
# False (``reasoning_effort: false``/``off``/``no``) to "", silently
|
||||
# re-enabling thinking for users who explicitly turned it off.
|
||||
Per-model overrides (agent.reasoning_overrides) take precedence
|
||||
over the global value when the current model matches a key
|
||||
(spelling-tolerant). Closes #21256.
|
||||
"""
|
||||
from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort
|
||||
|
||||
cfg = _load_cfg()
|
||||
|
||||
# Per-model override first
|
||||
model_cfg = cfg.get("model") or {}
|
||||
model = str(
|
||||
(model_cfg.get("default", "") if isinstance(model_cfg, dict) else "")
|
||||
or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "")
|
||||
or ""
|
||||
).strip()
|
||||
overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {}
|
||||
per_model = resolve_per_model_reasoning_effort(model, overrides)
|
||||
if per_model is not None:
|
||||
return per_model
|
||||
|
||||
# Global fallback — pass the raw value through; ``or ""`` would coerce
|
||||
# a YAML boolean False (``reasoning_effort: false``/``off``/``no``) to
|
||||
# "", silently re-enabling thinking for users who explicitly turned it off.
|
||||
return parse_reasoning_effort(
|
||||
(_load_cfg().get("agent") or {}).get("reasoning_effort", "")
|
||||
(cfg.get("agent") or {}).get("reasoning_effort", "")
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1301,6 +1301,37 @@ You can also change the reasoning effort at runtime with the `/reasoning` comman
|
|||
/reasoning hide # Hide model thinking
|
||||
```
|
||||
|
||||
#### Per-Model Reasoning Overrides
|
||||
|
||||
You can set different reasoning effort levels for different models. This is useful when you want high reasoning for complex models but medium for faster ones:
|
||||
|
||||
```yaml
|
||||
agent:
|
||||
reasoning_effort: "medium" # global default
|
||||
reasoning_overrides:
|
||||
"openrouter/anthropic/claude-opus-4.5": "xhigh"
|
||||
"openai/gpt-5": "low"
|
||||
"claude-sonnet-4.6": "high" # bare model name also works
|
||||
```
|
||||
|
||||
The key matching is **spelling-tolerant** — any reasonable spelling will match:
|
||||
- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` (dots and dashes are interchangeable)
|
||||
- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` (provider prefix optional)
|
||||
- Exact matches take precedence over variants
|
||||
|
||||
:::note
|
||||
There is no `hermes config set` support for `reasoning_overrides` keys — edit the YAML file directly. This is because model names often contain dots (e.g. `claude-opus-4.5`), which conflict with the CLI's dotted-key syntax.
|
||||
:::
|
||||
|
||||
**Resolution priority:**
|
||||
|
||||
1. Session-scoped `/reasoning --session` override (gateway only)
|
||||
2. Per-model override from `agent.reasoning_overrides` (spelling-tolerant)
|
||||
3. Global `agent.reasoning_effort`
|
||||
4. Provider default
|
||||
|
||||
The override applies automatically everywhere: CLI startup, messaging gateway, Desktop/TUI, cron jobs, `/model` mid-session switches, and fallback model activation.
|
||||
|
||||
## Tool-Use Enforcement
|
||||
|
||||
Some models occasionally describe intended actions as text instead of making tool calls ("I would run the tests..." instead of actually calling the terminal). Tool-use enforcement injects system prompt guidance that steers the model back to actually calling tools.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue