mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
8394b5ddd2
Previously /fast only supported gpt-5.4 and forced a provider switch to openai-codex. Now supports all 13 models from OpenAI's Priority Processing pricing table (gpt-5.4, gpt-5.4-mini, gpt-5.2, gpt-5.1, gpt-5, gpt-5-mini, gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o3, o4-mini). Key changes: - Replaced _FAST_MODE_BACKEND_CONFIG with _PRIORITY_PROCESSING_MODELS frozenset - Removed provider-forcing logic — service_tier is now injected into whatever API path the user is already on (Codex Responses, Chat Completions, or OpenRouter passthrough) - Added request_overrides support to chat_completions path in run_agent.py - Updated messaging from 'Codex inference tier' to 'Priority Processing' - Expanded test coverage for all supported models
256 lines
9.4 KiB
Python
256 lines
9.4 KiB
Python
"""Tests for the /fast CLI command and service-tier config handling."""
|
|
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
|
|
def _import_cli():
|
|
import hermes_cli.config as config_mod
|
|
|
|
if not hasattr(config_mod, "save_env_value_secure"):
|
|
config_mod.save_env_value_secure = lambda key, value: {
|
|
"success": True,
|
|
"stored_as": key,
|
|
"validated": False,
|
|
}
|
|
|
|
import cli as cli_mod
|
|
|
|
return cli_mod
|
|
|
|
|
|
class TestParseServiceTierConfig(unittest.TestCase):
|
|
def _parse(self, raw):
|
|
cli_mod = _import_cli()
|
|
return cli_mod._parse_service_tier_config(raw)
|
|
|
|
def test_fast_maps_to_priority(self):
|
|
self.assertEqual(self._parse("fast"), "priority")
|
|
self.assertEqual(self._parse("priority"), "priority")
|
|
|
|
def test_normal_disables_service_tier(self):
|
|
self.assertIsNone(self._parse("normal"))
|
|
self.assertIsNone(self._parse("off"))
|
|
self.assertIsNone(self._parse(""))
|
|
|
|
|
|
class TestHandleFastCommand(unittest.TestCase):
|
|
def _make_cli(self, service_tier=None):
|
|
return SimpleNamespace(
|
|
service_tier=service_tier,
|
|
provider="openai-codex",
|
|
requested_provider="openai-codex",
|
|
model="gpt-5.4",
|
|
_fast_command_available=lambda: True,
|
|
agent=MagicMock(),
|
|
)
|
|
|
|
def test_no_args_shows_status(self):
|
|
cli_mod = _import_cli()
|
|
stub = self._make_cli(service_tier=None)
|
|
with (
|
|
patch.object(cli_mod, "_cprint") as mock_cprint,
|
|
patch.object(cli_mod, "save_config_value") as mock_save,
|
|
):
|
|
cli_mod.HermesCLI._handle_fast_command(stub, "/fast")
|
|
|
|
# Bare /fast shows status, does not change config
|
|
mock_save.assert_not_called()
|
|
# Should have printed the status line
|
|
printed = " ".join(str(c) for c in mock_cprint.call_args_list)
|
|
self.assertIn("normal", printed)
|
|
|
|
def test_no_args_shows_fast_when_enabled(self):
|
|
cli_mod = _import_cli()
|
|
stub = self._make_cli(service_tier="priority")
|
|
with (
|
|
patch.object(cli_mod, "_cprint") as mock_cprint,
|
|
patch.object(cli_mod, "save_config_value") as mock_save,
|
|
):
|
|
cli_mod.HermesCLI._handle_fast_command(stub, "/fast")
|
|
|
|
mock_save.assert_not_called()
|
|
printed = " ".join(str(c) for c in mock_cprint.call_args_list)
|
|
self.assertIn("fast", printed)
|
|
|
|
def test_normal_argument_clears_service_tier(self):
|
|
cli_mod = _import_cli()
|
|
stub = self._make_cli(service_tier="priority")
|
|
with (
|
|
patch.object(cli_mod, "_cprint"),
|
|
patch.object(cli_mod, "save_config_value", return_value=True) as mock_save,
|
|
):
|
|
cli_mod.HermesCLI._handle_fast_command(stub, "/fast normal")
|
|
|
|
mock_save.assert_called_once_with("agent.service_tier", "normal")
|
|
self.assertIsNone(stub.service_tier)
|
|
self.assertIsNone(stub.agent)
|
|
|
|
def test_unsupported_model_does_not_expose_fast(self):
|
|
cli_mod = _import_cli()
|
|
stub = SimpleNamespace(
|
|
service_tier=None,
|
|
provider="openai-codex",
|
|
requested_provider="openai-codex",
|
|
model="gpt-5.3-codex",
|
|
_fast_command_available=lambda: False,
|
|
agent=MagicMock(),
|
|
)
|
|
|
|
with (
|
|
patch.object(cli_mod, "_cprint") as mock_cprint,
|
|
patch.object(cli_mod, "save_config_value") as mock_save,
|
|
):
|
|
cli_mod.HermesCLI._handle_fast_command(stub, "/fast")
|
|
|
|
mock_save.assert_not_called()
|
|
self.assertTrue(mock_cprint.called)
|
|
|
|
|
|
class TestPriorityProcessingModels(unittest.TestCase):
|
|
"""Verify the expanded Priority Processing model registry."""
|
|
|
|
def test_all_documented_models_supported(self):
|
|
from hermes_cli.models import model_supports_fast_mode
|
|
|
|
# All models from OpenAI's Priority Processing pricing table
|
|
supported = [
|
|
"gpt-5.4", "gpt-5.4-mini", "gpt-5.2",
|
|
"gpt-5.1", "gpt-5", "gpt-5-mini",
|
|
"gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano",
|
|
"gpt-4o", "gpt-4o-mini",
|
|
"o3", "o4-mini",
|
|
]
|
|
for model in supported:
|
|
assert model_supports_fast_mode(model), f"{model} should support fast mode"
|
|
|
|
def test_vendor_prefix_stripped(self):
|
|
from hermes_cli.models import model_supports_fast_mode
|
|
|
|
assert model_supports_fast_mode("openai/gpt-5.4") is True
|
|
assert model_supports_fast_mode("openai/gpt-4.1") is True
|
|
assert model_supports_fast_mode("openai/o3") is True
|
|
|
|
def test_non_priority_models_rejected(self):
|
|
from hermes_cli.models import model_supports_fast_mode
|
|
|
|
assert model_supports_fast_mode("gpt-5.3-codex") is False
|
|
assert model_supports_fast_mode("claude-sonnet-4") is False
|
|
assert model_supports_fast_mode("") is False
|
|
assert model_supports_fast_mode(None) is False
|
|
|
|
def test_resolve_overrides_returns_service_tier(self):
|
|
from hermes_cli.models import resolve_fast_mode_overrides
|
|
|
|
result = resolve_fast_mode_overrides("gpt-5.4")
|
|
assert result == {"service_tier": "priority"}
|
|
|
|
result = resolve_fast_mode_overrides("gpt-4.1")
|
|
assert result == {"service_tier": "priority"}
|
|
|
|
def test_resolve_overrides_none_for_unsupported(self):
|
|
from hermes_cli.models import resolve_fast_mode_overrides
|
|
|
|
assert resolve_fast_mode_overrides("gpt-5.3-codex") is None
|
|
assert resolve_fast_mode_overrides("claude-sonnet-4") is None
|
|
|
|
|
|
class TestFastModeRouting(unittest.TestCase):
|
|
def test_fast_command_exposed_for_model_even_when_provider_is_auto(self):
|
|
cli_mod = _import_cli()
|
|
stub = SimpleNamespace(provider="auto", requested_provider="auto", model="gpt-5.4", agent=None)
|
|
|
|
assert cli_mod.HermesCLI._fast_command_available(stub) is True
|
|
|
|
def test_fast_command_exposed_for_non_codex_models(self):
|
|
cli_mod = _import_cli()
|
|
stub = SimpleNamespace(provider="openai", requested_provider="openai", model="gpt-4.1", agent=None)
|
|
assert cli_mod.HermesCLI._fast_command_available(stub) is True
|
|
|
|
stub = SimpleNamespace(provider="openrouter", requested_provider="openrouter", model="o3", agent=None)
|
|
assert cli_mod.HermesCLI._fast_command_available(stub) is True
|
|
|
|
def test_turn_route_injects_overrides_without_provider_switch(self):
|
|
"""Fast mode should add request_overrides but NOT change the provider/runtime."""
|
|
cli_mod = _import_cli()
|
|
stub = SimpleNamespace(
|
|
model="gpt-5.4",
|
|
api_key="primary-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
provider="openrouter",
|
|
api_mode="chat_completions",
|
|
acp_command=None,
|
|
acp_args=[],
|
|
_credential_pool=None,
|
|
_smart_model_routing={},
|
|
service_tier="priority",
|
|
)
|
|
|
|
original_runtime = {
|
|
"api_key": "***",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"provider": "openrouter",
|
|
"api_mode": "chat_completions",
|
|
"command": None,
|
|
"args": [],
|
|
"credential_pool": None,
|
|
}
|
|
|
|
with patch("agent.smart_model_routing.resolve_turn_route", return_value={
|
|
"model": "gpt-5.4",
|
|
"runtime": dict(original_runtime),
|
|
"label": None,
|
|
"signature": ("gpt-5.4", "openrouter", "https://openrouter.ai/api/v1", "chat_completions", None, ()),
|
|
}):
|
|
route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi")
|
|
|
|
# Provider should NOT have changed
|
|
assert route["runtime"]["provider"] == "openrouter"
|
|
assert route["runtime"]["api_mode"] == "chat_completions"
|
|
# But request_overrides should be set
|
|
assert route["request_overrides"] == {"service_tier": "priority"}
|
|
|
|
def test_turn_route_keeps_primary_runtime_when_model_has_no_fast_backend(self):
|
|
cli_mod = _import_cli()
|
|
stub = SimpleNamespace(
|
|
model="gpt-5.3-codex",
|
|
api_key="primary-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
provider="openrouter",
|
|
api_mode="chat_completions",
|
|
acp_command=None,
|
|
acp_args=[],
|
|
_credential_pool=None,
|
|
_smart_model_routing={},
|
|
service_tier="priority",
|
|
)
|
|
|
|
primary_route = {
|
|
"model": "gpt-5.3-codex",
|
|
"runtime": {
|
|
"api_key": "***",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"provider": "openrouter",
|
|
"api_mode": "chat_completions",
|
|
"command": None,
|
|
"args": [],
|
|
"credential_pool": None,
|
|
},
|
|
"label": None,
|
|
"signature": ("gpt-5.3-codex", "openrouter", "https://openrouter.ai/api/v1", "chat_completions", None, ()),
|
|
}
|
|
with patch("agent.smart_model_routing.resolve_turn_route", return_value=primary_route):
|
|
route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi")
|
|
|
|
assert route["runtime"]["provider"] == "openrouter"
|
|
assert route.get("request_overrides") is None
|
|
|
|
|
|
class TestConfigDefault(unittest.TestCase):
|
|
def test_default_config_has_service_tier(self):
|
|
from hermes_cli.config import DEFAULT_CONFIG
|
|
|
|
agent = DEFAULT_CONFIG.get("agent", {})
|
|
self.assertIn("service_tier", agent)
|
|
self.assertEqual(agent["service_tier"], "")
|