mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-18 04:41:56 +00:00
feat(openrouter): wire Pareto Code router with min_coding_score knob (#22838)
Pick openrouter/pareto-code as your model and OpenRouter auto-routes each
request to the cheapest model meeting your coding-quality bar (ranked by
Artificial Analysis). The new openrouter.min_coding_score config key (0.0-1.0,
default 0.65) tunes the floor.
- hermes_cli/models.py: add openrouter/pareto-code to OPENROUTER_MODELS so
it shows up in the picker with a description
- hermes_cli/config.py: add openrouter.min_coding_score (default 0.65 — lands
on a mid-tier coder on the current Pareto frontier)
- plugins/model-providers/openrouter: emit extra_body.plugins =
[{id: pareto-router, min_coding_score: X}] when model is openrouter/pareto-code
AND the score is a valid float in [0.0, 1.0]
- agent/transports/chat_completions.py: same emission on the legacy flag
path (when no provider profile is loaded)
- run_agent.py: openrouter_min_coding_score kwarg + storage; plumbed into
both build_kwargs() invocations and the context-summary extra_body path
- cli.py: read openrouter.min_coding_score once at init, validate float in
[0,1], pass to AIAgent constructions (CLI + background-task paths)
- cron/scheduler.py, batch_runner.py, tools/delegate_tool.py,
tui_gateway/server.py: propagate the kwarg (mirrors providers_order
plumbing — subagents inherit, cron/batch read from config)
- tests: profile-level + transport-level coverage of the model gating,
unset/empty/out-of-range handling, and the legacy flag path
- docs: new 'OpenRouter Pareto Code Router' section in providers.md
Verified end-to-end against api.openrouter.ai: at score=0.65 we land on a
mid-tier coder, at omission we get the strongest. Score is silently dropped
on any model other than openrouter/pareto-code, so it's safe to leave set.
This commit is contained in:
parent
b349ae1e4c
commit
c7f0aab949
14 changed files with 222 additions and 0 deletions
|
|
@ -323,6 +323,21 @@ class ChatCompletionsTransport(ProviderTransport):
|
||||||
if provider_prefs and is_openrouter:
|
if provider_prefs and is_openrouter:
|
||||||
extra_body["provider"] = provider_prefs
|
extra_body["provider"] = provider_prefs
|
||||||
|
|
||||||
|
# Pareto Code router plugin — model-gated. Same shape as the
|
||||||
|
# profile path in plugins/model-providers/openrouter/__init__.py;
|
||||||
|
# this branch only runs when the OpenRouter profile isn't loaded.
|
||||||
|
if is_openrouter and model == "openrouter/pareto-code":
|
||||||
|
_pareto_score = params.get("openrouter_min_coding_score")
|
||||||
|
if _pareto_score is not None and _pareto_score != "":
|
||||||
|
try:
|
||||||
|
_pareto_score_f = float(_pareto_score)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
_pareto_score_f = None
|
||||||
|
if _pareto_score_f is not None and 0.0 <= _pareto_score_f <= 1.0:
|
||||||
|
extra_body["plugins"] = [
|
||||||
|
{"id": "pareto-router", "min_coding_score": _pareto_score_f}
|
||||||
|
]
|
||||||
|
|
||||||
# Kimi extra_body.thinking
|
# Kimi extra_body.thinking
|
||||||
if is_kimi:
|
if is_kimi:
|
||||||
_kimi_thinking_enabled = True
|
_kimi_thinking_enabled = True
|
||||||
|
|
@ -463,6 +478,7 @@ class ChatCompletionsTransport(ProviderTransport):
|
||||||
model=model,
|
model=model,
|
||||||
base_url=params.get("base_url"),
|
base_url=params.get("base_url"),
|
||||||
reasoning_config=reasoning_config,
|
reasoning_config=reasoning_config,
|
||||||
|
openrouter_min_coding_score=params.get("openrouter_min_coding_score"),
|
||||||
)
|
)
|
||||||
if profile_body:
|
if profile_body:
|
||||||
extra_body.update(profile_body)
|
extra_body.update(profile_body)
|
||||||
|
|
|
||||||
|
|
@ -337,6 +337,7 @@ def _process_single_prompt(
|
||||||
providers_ignored=config.get("providers_ignored"),
|
providers_ignored=config.get("providers_ignored"),
|
||||||
providers_order=config.get("providers_order"),
|
providers_order=config.get("providers_order"),
|
||||||
provider_sort=config.get("provider_sort"),
|
provider_sort=config.get("provider_sort"),
|
||||||
|
openrouter_min_coding_score=config.get("openrouter_min_coding_score"),
|
||||||
max_tokens=config.get("max_tokens"),
|
max_tokens=config.get("max_tokens"),
|
||||||
reasoning_config=config.get("reasoning_config"),
|
reasoning_config=config.get("reasoning_config"),
|
||||||
prefill_messages=config.get("prefill_messages"),
|
prefill_messages=config.get("prefill_messages"),
|
||||||
|
|
@ -546,6 +547,7 @@ class BatchRunner:
|
||||||
providers_ignored: List[str] = None,
|
providers_ignored: List[str] = None,
|
||||||
providers_order: List[str] = None,
|
providers_order: List[str] = None,
|
||||||
provider_sort: str = None,
|
provider_sort: str = None,
|
||||||
|
openrouter_min_coding_score: Optional[float] = None,
|
||||||
max_tokens: int = None,
|
max_tokens: int = None,
|
||||||
reasoning_config: Dict[str, Any] = None,
|
reasoning_config: Dict[str, Any] = None,
|
||||||
prefill_messages: List[Dict[str, Any]] = None,
|
prefill_messages: List[Dict[str, Any]] = None,
|
||||||
|
|
@ -595,6 +597,7 @@ class BatchRunner:
|
||||||
self.providers_ignored = providers_ignored
|
self.providers_ignored = providers_ignored
|
||||||
self.providers_order = providers_order
|
self.providers_order = providers_order
|
||||||
self.provider_sort = provider_sort
|
self.provider_sort = provider_sort
|
||||||
|
self.openrouter_min_coding_score = openrouter_min_coding_score
|
||||||
self.max_tokens = max_tokens
|
self.max_tokens = max_tokens
|
||||||
self.reasoning_config = reasoning_config
|
self.reasoning_config = reasoning_config
|
||||||
self.prefill_messages = prefill_messages
|
self.prefill_messages = prefill_messages
|
||||||
|
|
@ -873,6 +876,7 @@ class BatchRunner:
|
||||||
"providers_ignored": self.providers_ignored,
|
"providers_ignored": self.providers_ignored,
|
||||||
"providers_order": self.providers_order,
|
"providers_order": self.providers_order,
|
||||||
"provider_sort": self.provider_sort,
|
"provider_sort": self.provider_sort,
|
||||||
|
"openrouter_min_coding_score": self.openrouter_min_coding_score,
|
||||||
"max_tokens": self.max_tokens,
|
"max_tokens": self.max_tokens,
|
||||||
"reasoning_config": self.reasoning_config,
|
"reasoning_config": self.reasoning_config,
|
||||||
"prefill_messages": self.prefill_messages,
|
"prefill_messages": self.prefill_messages,
|
||||||
|
|
|
||||||
16
cli.py
16
cli.py
|
|
@ -2473,6 +2473,20 @@ class HermesCLI:
|
||||||
self._providers_order = pr.get("order")
|
self._providers_order = pr.get("order")
|
||||||
self._provider_require_params = pr.get("require_parameters", False)
|
self._provider_require_params = pr.get("require_parameters", False)
|
||||||
self._provider_data_collection = pr.get("data_collection")
|
self._provider_data_collection = pr.get("data_collection")
|
||||||
|
|
||||||
|
# OpenRouter Pareto Code router knob — coding-score floor (0.0-1.0).
|
||||||
|
# Only applied when model.model == "openrouter/pareto-code".
|
||||||
|
# Empty string / None / out-of-range = unset (let OR pick strongest coder).
|
||||||
|
_or_cfg = CLI_CONFIG.get("openrouter", {}) or {}
|
||||||
|
_raw_score = _or_cfg.get("min_coding_score")
|
||||||
|
self._openrouter_min_coding_score: Optional[float] = None
|
||||||
|
if _raw_score not in (None, ""):
|
||||||
|
try:
|
||||||
|
_f = float(_raw_score)
|
||||||
|
if 0.0 <= _f <= 1.0:
|
||||||
|
self._openrouter_min_coding_score = _f
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
# Fallback provider chain — tried in order when primary fails after retries.
|
# Fallback provider chain — tried in order when primary fails after retries.
|
||||||
# Supports new list format (fallback_providers) and legacy single-dict (fallback_model).
|
# Supports new list format (fallback_providers) and legacy single-dict (fallback_model).
|
||||||
|
|
@ -4031,6 +4045,7 @@ class HermesCLI:
|
||||||
provider_sort=self._provider_sort,
|
provider_sort=self._provider_sort,
|
||||||
provider_require_parameters=self._provider_require_params,
|
provider_require_parameters=self._provider_require_params,
|
||||||
provider_data_collection=self._provider_data_collection,
|
provider_data_collection=self._provider_data_collection,
|
||||||
|
openrouter_min_coding_score=self._openrouter_min_coding_score,
|
||||||
session_id=self.session_id,
|
session_id=self.session_id,
|
||||||
platform="cli",
|
platform="cli",
|
||||||
session_db=self._session_db,
|
session_db=self._session_db,
|
||||||
|
|
@ -7249,6 +7264,7 @@ class HermesCLI:
|
||||||
provider_sort=self._provider_sort,
|
provider_sort=self._provider_sort,
|
||||||
provider_require_parameters=self._provider_require_params,
|
provider_require_parameters=self._provider_require_params,
|
||||||
provider_data_collection=self._provider_data_collection,
|
provider_data_collection=self._provider_data_collection,
|
||||||
|
openrouter_min_coding_score=self._openrouter_min_coding_score,
|
||||||
fallback_model=self._fallback_model,
|
fallback_model=self._fallback_model,
|
||||||
)
|
)
|
||||||
# Silence raw spinner; route thinking through TUI widget when no foreground agent is active.
|
# Silence raw spinner; route thinking through TUI widget when no foreground agent is active.
|
||||||
|
|
|
||||||
|
|
@ -1439,6 +1439,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||||
providers_ignored=pr.get("ignore"),
|
providers_ignored=pr.get("ignore"),
|
||||||
providers_order=pr.get("order"),
|
providers_order=pr.get("order"),
|
||||||
provider_sort=pr.get("sort"),
|
provider_sort=pr.get("sort"),
|
||||||
|
openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
|
||||||
enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
|
enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
|
||||||
disabled_toolsets=["cronjob", "messaging", "clarify"],
|
disabled_toolsets=["cronjob", "messaging", "clarify"],
|
||||||
quiet_mode=True,
|
quiet_mode=True,
|
||||||
|
|
|
||||||
|
|
@ -691,9 +691,18 @@ DEFAULT_CONFIG = {
|
||||||
# See: https://openrouter.ai/docs/guides/features/response-caching
|
# See: https://openrouter.ai/docs/guides/features/response-caching
|
||||||
# response_cache_ttl: how long cached responses remain valid, in seconds (1-86400).
|
# response_cache_ttl: how long cached responses remain valid, in seconds (1-86400).
|
||||||
# Default 300 (5 minutes). Only used when response_cache is enabled.
|
# Default 300 (5 minutes). Only used when response_cache is enabled.
|
||||||
|
# min_coding_score: knob for the openrouter/pareto-code router (0.0-1.0).
|
||||||
|
# Only applied when model.model is "openrouter/pareto-code". Higher
|
||||||
|
# values route to stronger (more expensive) coders; lower values open
|
||||||
|
# up cheaper, faster options. Default 0.65 lands on the mid-tier
|
||||||
|
# coder on the current Pareto frontier. Empty string = let OpenRouter
|
||||||
|
# pick the strongest available coder (router's documented default
|
||||||
|
# when the plugins block is omitted).
|
||||||
|
# See: https://openrouter.ai/docs/guides/routing/routers/pareto-router
|
||||||
"openrouter": {
|
"openrouter": {
|
||||||
"response_cache": True,
|
"response_cache": True,
|
||||||
"response_cache_ttl": 300,
|
"response_cache_ttl": 300,
|
||||||
|
"min_coding_score": 0.65,
|
||||||
},
|
},
|
||||||
|
|
||||||
# AWS Bedrock provider configuration.
|
# AWS Bedrock provider configuration.
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ COPILOT_REASONING_EFFORTS_O_SERIES = ["low", "medium", "high"]
|
||||||
# (model_id, display description shown in menus)
|
# (model_id, display description shown in menus)
|
||||||
OPENROUTER_MODELS: list[tuple[str, str]] = [
|
OPENROUTER_MODELS: list[tuple[str, str]] = [
|
||||||
("moonshotai/kimi-k2.6", "recommended"),
|
("moonshotai/kimi-k2.6", "recommended"),
|
||||||
|
("openrouter/pareto-code", "auto-routes to cheapest coder meeting openrouter.min_coding_score"),
|
||||||
("anthropic/claude-opus-4.7", ""),
|
("anthropic/claude-opus-4.7", ""),
|
||||||
("anthropic/claude-opus-4.6", ""),
|
("anthropic/claude-opus-4.6", ""),
|
||||||
("anthropic/claude-sonnet-4.6", ""),
|
("anthropic/claude-sonnet-4.6", ""),
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,23 @@ class OpenRouterProfile(ProviderProfile):
|
||||||
prefs = context.get("provider_preferences")
|
prefs = context.get("provider_preferences")
|
||||||
if prefs:
|
if prefs:
|
||||||
body["provider"] = prefs
|
body["provider"] = prefs
|
||||||
|
|
||||||
|
# Pareto Code router — model-gated. The plugins block is only
|
||||||
|
# meaningful for openrouter/pareto-code; sending it on any other
|
||||||
|
# model has no documented effect and would be confusing in logs.
|
||||||
|
# See: https://openrouter.ai/docs/guides/routing/routers/pareto-router
|
||||||
|
model = (context.get("model") or "")
|
||||||
|
if model == "openrouter/pareto-code":
|
||||||
|
score = context.get("openrouter_min_coding_score")
|
||||||
|
if score is not None and score != "":
|
||||||
|
try:
|
||||||
|
score_f = float(score)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
score_f = None
|
||||||
|
if score_f is not None and 0.0 <= score_f <= 1.0:
|
||||||
|
body["plugins"] = [
|
||||||
|
{"id": "pareto-router", "min_coding_score": score_f}
|
||||||
|
]
|
||||||
return body
|
return body
|
||||||
|
|
||||||
def build_api_kwargs_extras(
|
def build_api_kwargs_extras(
|
||||||
|
|
|
||||||
28
run_agent.py
28
run_agent.py
|
|
@ -1075,6 +1075,7 @@ class AIAgent:
|
||||||
provider_sort: str = None,
|
provider_sort: str = None,
|
||||||
provider_require_parameters: bool = False,
|
provider_require_parameters: bool = False,
|
||||||
provider_data_collection: str = None,
|
provider_data_collection: str = None,
|
||||||
|
openrouter_min_coding_score: Optional[float] = None,
|
||||||
session_id: str = None,
|
session_id: str = None,
|
||||||
tool_progress_callback: callable = None,
|
tool_progress_callback: callable = None,
|
||||||
tool_start_callback: callable = None,
|
tool_start_callback: callable = None,
|
||||||
|
|
@ -1137,6 +1138,9 @@ class AIAgent:
|
||||||
providers_ignored (List[str]): OpenRouter providers to ignore (optional)
|
providers_ignored (List[str]): OpenRouter providers to ignore (optional)
|
||||||
providers_order (List[str]): OpenRouter providers to try in order (optional)
|
providers_order (List[str]): OpenRouter providers to try in order (optional)
|
||||||
provider_sort (str): Sort providers by price/throughput/latency (optional)
|
provider_sort (str): Sort providers by price/throughput/latency (optional)
|
||||||
|
openrouter_min_coding_score (float): Coding-score floor (0.0-1.0) for the
|
||||||
|
openrouter/pareto-code router. Only applied when model == "openrouter/pareto-code".
|
||||||
|
None or empty = let OpenRouter pick the strongest available coder.
|
||||||
session_id (str): Pre-generated session ID for logging (optional, auto-generated if not provided)
|
session_id (str): Pre-generated session ID for logging (optional, auto-generated if not provided)
|
||||||
tool_progress_callback (callable): Callback function(tool_name, args_preview) for progress notifications
|
tool_progress_callback (callable): Callback function(tool_name, args_preview) for progress notifications
|
||||||
clarify_callback (callable): Callback function(question, choices) -> str for interactive user questions.
|
clarify_callback (callable): Callback function(question, choices) -> str for interactive user questions.
|
||||||
|
|
@ -1356,6 +1360,7 @@ class AIAgent:
|
||||||
self.provider_sort = provider_sort
|
self.provider_sort = provider_sort
|
||||||
self.provider_require_parameters = provider_require_parameters
|
self.provider_require_parameters = provider_require_parameters
|
||||||
self.provider_data_collection = provider_data_collection
|
self.provider_data_collection = provider_data_collection
|
||||||
|
self.openrouter_min_coding_score = openrouter_min_coding_score
|
||||||
|
|
||||||
# Store toolset filtering options
|
# Store toolset filtering options
|
||||||
self.enabled_toolsets = enabled_toolsets
|
self.enabled_toolsets = enabled_toolsets
|
||||||
|
|
@ -9029,6 +9034,7 @@ class AIAgent:
|
||||||
ollama_num_ctx=self._ollama_num_ctx,
|
ollama_num_ctx=self._ollama_num_ctx,
|
||||||
# Context forwarded to profile hooks:
|
# Context forwarded to profile hooks:
|
||||||
provider_preferences=_prefs or None,
|
provider_preferences=_prefs or None,
|
||||||
|
openrouter_min_coding_score=self.openrouter_min_coding_score,
|
||||||
anthropic_max_output=_ant_max,
|
anthropic_max_output=_ant_max,
|
||||||
supports_reasoning=self._supports_reasoning_extra_body(),
|
supports_reasoning=self._supports_reasoning_extra_body(),
|
||||||
qwen_session_metadata=_qwen_meta,
|
qwen_session_metadata=_qwen_meta,
|
||||||
|
|
@ -9068,6 +9074,7 @@ class AIAgent:
|
||||||
is_custom_provider=self.provider == "custom",
|
is_custom_provider=self.provider == "custom",
|
||||||
ollama_num_ctx=self._ollama_num_ctx,
|
ollama_num_ctx=self._ollama_num_ctx,
|
||||||
provider_preferences=_prefs or None,
|
provider_preferences=_prefs or None,
|
||||||
|
openrouter_min_coding_score=self.openrouter_min_coding_score,
|
||||||
qwen_prepare_fn=self._qwen_prepare_chat_messages if _is_qwen else None,
|
qwen_prepare_fn=self._qwen_prepare_chat_messages if _is_qwen else None,
|
||||||
qwen_prepare_inplace_fn=self._qwen_prepare_chat_messages_inplace if _is_qwen else None,
|
qwen_prepare_inplace_fn=self._qwen_prepare_chat_messages_inplace if _is_qwen else None,
|
||||||
qwen_session_metadata=_qwen_meta,
|
qwen_session_metadata=_qwen_meta,
|
||||||
|
|
@ -10974,6 +10981,27 @@ class AIAgent:
|
||||||
):
|
):
|
||||||
summary_extra_body["provider"] = provider_preferences
|
summary_extra_body["provider"] = provider_preferences
|
||||||
|
|
||||||
|
# Pareto Code router plugin — model-gated. Same shape as
|
||||||
|
# the main-loop emission so summary calls on
|
||||||
|
# openrouter/pareto-code respect the user's coding-score floor.
|
||||||
|
if (
|
||||||
|
self.model == "openrouter/pareto-code"
|
||||||
|
and (
|
||||||
|
(self.provider or "").strip().lower() == "openrouter"
|
||||||
|
or self._is_openrouter_url()
|
||||||
|
)
|
||||||
|
and self.openrouter_min_coding_score is not None
|
||||||
|
and self.openrouter_min_coding_score != ""
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
_ps = float(self.openrouter_min_coding_score)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
_ps = None
|
||||||
|
if _ps is not None and 0.0 <= _ps <= 1.0:
|
||||||
|
summary_extra_body["plugins"] = [
|
||||||
|
{"id": "pareto-router", "min_coding_score": _ps}
|
||||||
|
]
|
||||||
|
|
||||||
if summary_extra_body:
|
if summary_extra_body:
|
||||||
summary_kwargs["extra_body"] = summary_extra_body
|
summary_kwargs["extra_body"] = summary_extra_body
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -83,6 +83,69 @@ class TestChatCompletionsBuildKwargs:
|
||||||
)
|
)
|
||||||
assert kw["extra_body"]["provider"] == {"only": ["openai"]}
|
assert kw["extra_body"]["provider"] == {"only": ["openai"]}
|
||||||
|
|
||||||
|
def test_openrouter_pareto_min_coding_score(self, transport):
|
||||||
|
"""Profile path: model=openrouter/pareto-code + score → plugins block."""
|
||||||
|
from providers import get_provider_profile
|
||||||
|
profile = get_provider_profile("openrouter")
|
||||||
|
msgs = [{"role": "user", "content": "Hi"}]
|
||||||
|
kw = transport.build_kwargs(
|
||||||
|
model="openrouter/pareto-code", messages=msgs,
|
||||||
|
provider_profile=profile,
|
||||||
|
openrouter_min_coding_score=0.65,
|
||||||
|
)
|
||||||
|
assert kw["extra_body"]["plugins"] == [
|
||||||
|
{"id": "pareto-router", "min_coding_score": 0.65}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_openrouter_pareto_score_ignored_for_other_models(self, transport):
|
||||||
|
"""Score must not be emitted for any model other than openrouter/pareto-code."""
|
||||||
|
from providers import get_provider_profile
|
||||||
|
profile = get_provider_profile("openrouter")
|
||||||
|
msgs = [{"role": "user", "content": "Hi"}]
|
||||||
|
kw = transport.build_kwargs(
|
||||||
|
model="anthropic/claude-sonnet-4.6", messages=msgs,
|
||||||
|
provider_profile=profile,
|
||||||
|
openrouter_min_coding_score=0.65,
|
||||||
|
)
|
||||||
|
assert "plugins" not in (kw.get("extra_body") or {})
|
||||||
|
|
||||||
|
def test_openrouter_pareto_score_omitted_when_unset(self, transport):
|
||||||
|
"""No score → no plugins block (router uses its omission default = strongest coder)."""
|
||||||
|
from providers import get_provider_profile
|
||||||
|
profile = get_provider_profile("openrouter")
|
||||||
|
msgs = [{"role": "user", "content": "Hi"}]
|
||||||
|
kw = transport.build_kwargs(
|
||||||
|
model="openrouter/pareto-code", messages=msgs,
|
||||||
|
provider_profile=profile,
|
||||||
|
openrouter_min_coding_score=None,
|
||||||
|
)
|
||||||
|
assert "plugins" not in (kw.get("extra_body") or {})
|
||||||
|
|
||||||
|
def test_openrouter_pareto_score_out_of_range_dropped(self, transport):
|
||||||
|
"""Out-of-range scores must be silently dropped, not forwarded."""
|
||||||
|
from providers import get_provider_profile
|
||||||
|
profile = get_provider_profile("openrouter")
|
||||||
|
msgs = [{"role": "user", "content": "Hi"}]
|
||||||
|
for bad in (1.5, -0.1, "not-a-number"):
|
||||||
|
kw = transport.build_kwargs(
|
||||||
|
model="openrouter/pareto-code", messages=msgs,
|
||||||
|
provider_profile=profile,
|
||||||
|
openrouter_min_coding_score=bad,
|
||||||
|
)
|
||||||
|
assert "plugins" not in (kw.get("extra_body") or {}), f"bad={bad!r}"
|
||||||
|
|
||||||
|
def test_openrouter_pareto_legacy_path(self, transport):
|
||||||
|
"""Legacy flag path (no profile loaded) must also emit the plugins block."""
|
||||||
|
msgs = [{"role": "user", "content": "Hi"}]
|
||||||
|
kw = transport.build_kwargs(
|
||||||
|
model="openrouter/pareto-code", messages=msgs,
|
||||||
|
is_openrouter=True,
|
||||||
|
openrouter_min_coding_score=0.8,
|
||||||
|
)
|
||||||
|
assert kw["extra_body"]["plugins"] == [
|
||||||
|
{"id": "pareto-router", "min_coding_score": 0.8}
|
||||||
|
]
|
||||||
|
|
||||||
def test_nous_tags(self, transport):
|
def test_nous_tags(self, transport):
|
||||||
from providers import get_provider_profile
|
from providers import get_provider_profile
|
||||||
profile = get_provider_profile("nous")
|
profile = get_provider_profile("nous")
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,7 @@ def _make_background_cli_stub():
|
||||||
cli._provider_sort = None
|
cli._provider_sort = None
|
||||||
cli._provider_require_params = None
|
cli._provider_require_params = None
|
||||||
cli._provider_data_collection = None
|
cli._provider_data_collection = None
|
||||||
|
cli._openrouter_min_coding_score = None
|
||||||
cli._fallback_model = None
|
cli._fallback_model = None
|
||||||
cli._agent_running = False
|
cli._agent_running = False
|
||||||
cli._spinner_text = ""
|
cli._spinner_text = ""
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,46 @@ class TestOpenRouterProfile:
|
||||||
body = p.build_extra_body()
|
body = p.build_extra_body()
|
||||||
assert body == {}
|
assert body == {}
|
||||||
|
|
||||||
|
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
|
||||||
|
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
|
||||||
|
p = get_provider_profile("openrouter")
|
||||||
|
body = p.build_extra_body(
|
||||||
|
model="openrouter/pareto-code",
|
||||||
|
openrouter_min_coding_score=0.65,
|
||||||
|
)
|
||||||
|
assert body["plugins"] == [
|
||||||
|
{"id": "pareto-router", "min_coding_score": 0.65}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_pareto_score_ignored_for_other_models(self):
|
||||||
|
"""Score has no effect on any other model — plugins block must not appear."""
|
||||||
|
p = get_provider_profile("openrouter")
|
||||||
|
body = p.build_extra_body(
|
||||||
|
model="anthropic/claude-sonnet-4.6",
|
||||||
|
openrouter_min_coding_score=0.65,
|
||||||
|
)
|
||||||
|
assert "plugins" not in body
|
||||||
|
|
||||||
|
def test_pareto_score_unset_omits_plugins(self):
|
||||||
|
"""Empty/None score → no plugins block (router uses its omission default)."""
|
||||||
|
p = get_provider_profile("openrouter")
|
||||||
|
for unset in (None, ""):
|
||||||
|
body = p.build_extra_body(
|
||||||
|
model="openrouter/pareto-code",
|
||||||
|
openrouter_min_coding_score=unset,
|
||||||
|
)
|
||||||
|
assert "plugins" not in body, f"unset={unset!r}"
|
||||||
|
|
||||||
|
def test_pareto_score_out_of_range_dropped(self):
|
||||||
|
"""Invalid scores are silently dropped — never forwarded to OR."""
|
||||||
|
p = get_provider_profile("openrouter")
|
||||||
|
for bad in (1.5, -0.1, "not-a-number"):
|
||||||
|
body = p.build_extra_body(
|
||||||
|
model="openrouter/pareto-code",
|
||||||
|
openrouter_min_coding_score=bad,
|
||||||
|
)
|
||||||
|
assert "plugins" not in body, f"bad={bad!r}"
|
||||||
|
|
||||||
def test_reasoning_full_config(self):
|
def test_reasoning_full_config(self):
|
||||||
p = get_provider_profile("openrouter")
|
p = get_provider_profile("openrouter")
|
||||||
eb, _ = p.build_api_kwargs_extras(
|
eb, _ = p.build_api_kwargs_extras(
|
||||||
|
|
|
||||||
|
|
@ -1077,11 +1077,15 @@ def _build_child_agent(
|
||||||
child_providers_ignored = getattr(parent_agent, "providers_ignored", None)
|
child_providers_ignored = getattr(parent_agent, "providers_ignored", None)
|
||||||
child_providers_order = getattr(parent_agent, "providers_order", None)
|
child_providers_order = getattr(parent_agent, "providers_order", None)
|
||||||
child_provider_sort = getattr(parent_agent, "provider_sort", None)
|
child_provider_sort = getattr(parent_agent, "provider_sort", None)
|
||||||
|
child_openrouter_min_coding_score = getattr(parent_agent, "openrouter_min_coding_score", None)
|
||||||
if override_provider:
|
if override_provider:
|
||||||
child_providers_allowed = None
|
child_providers_allowed = None
|
||||||
child_providers_ignored = None
|
child_providers_ignored = None
|
||||||
child_providers_order = None
|
child_providers_order = None
|
||||||
child_provider_sort = None
|
child_provider_sort = None
|
||||||
|
# Note: openrouter_min_coding_score is model-gated (only emitted on
|
||||||
|
# openrouter/pareto-code), so we keep it inherited even when the
|
||||||
|
# provider is overridden — it's a no-op on any other model.
|
||||||
|
|
||||||
child = AIAgent(
|
child = AIAgent(
|
||||||
base_url=effective_base_url,
|
base_url=effective_base_url,
|
||||||
|
|
@ -1111,6 +1115,7 @@ def _build_child_agent(
|
||||||
providers_ignored=child_providers_ignored,
|
providers_ignored=child_providers_ignored,
|
||||||
providers_order=child_providers_order,
|
providers_order=child_providers_order,
|
||||||
provider_sort=child_provider_sort,
|
provider_sort=child_provider_sort,
|
||||||
|
openrouter_min_coding_score=child_openrouter_min_coding_score,
|
||||||
tool_progress_callback=child_progress_cb,
|
tool_progress_callback=child_progress_cb,
|
||||||
iteration_budget=None, # fresh budget per subagent
|
iteration_budget=None, # fresh budget per subagent
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1817,6 +1817,7 @@ def _background_agent_kwargs(agent, task_id: str) -> dict:
|
||||||
agent, "provider_require_parameters", False
|
agent, "provider_require_parameters", False
|
||||||
),
|
),
|
||||||
"provider_data_collection": getattr(agent, "provider_data_collection", None),
|
"provider_data_collection": getattr(agent, "provider_data_collection", None),
|
||||||
|
"openrouter_min_coding_score": getattr(agent, "openrouter_min_coding_score", None),
|
||||||
"session_id": task_id,
|
"session_id": task_id,
|
||||||
"reasoning_config": getattr(agent, "reasoning_config", None)
|
"reasoning_config": getattr(agent, "reasoning_config", None)
|
||||||
or _load_reasoning_config(),
|
or _load_reasoning_config(),
|
||||||
|
|
|
||||||
|
|
@ -1372,6 +1372,26 @@ provider_routing:
|
||||||
|
|
||||||
**Shortcuts:** Append `:nitro` to any model name for throughput sorting (e.g., `anthropic/claude-sonnet-4:nitro`), or `:floor` for price sorting.
|
**Shortcuts:** Append `:nitro` to any model name for throughput sorting (e.g., `anthropic/claude-sonnet-4:nitro`), or `:floor` for price sorting.
|
||||||
|
|
||||||
|
## OpenRouter Pareto Code Router
|
||||||
|
|
||||||
|
OpenRouter ships an experimental coding-model router at `openrouter/pareto-code` that auto-routes requests to the cheapest model meeting a coding-quality bar (ranked by [Artificial Analysis](https://artificialanalysis.ai/)). Pick this model and tune the `min_coding_score` knob in `~/.hermes/config.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
model:
|
||||||
|
provider: openrouter
|
||||||
|
model: openrouter/pareto-code
|
||||||
|
|
||||||
|
openrouter:
|
||||||
|
min_coding_score: 0.65 # 0.0–1.0; higher = stronger (more expensive) coders. Default 0.65.
|
||||||
|
```
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- `min_coding_score` is **only** sent when `model.model` is `openrouter/pareto-code`. On any other model the value is a no-op.
|
||||||
|
- Set to empty string (or remove the line) to let OpenRouter pick the strongest available coder — its documented behavior when the plugins block is omitted.
|
||||||
|
- Selection is deterministic per score on a given day, but the actual model chosen can shift as the Pareto frontier moves (new models, benchmark updates).
|
||||||
|
- See OpenRouter's [Pareto Router docs](https://openrouter.ai/docs/guides/routing/routers/pareto-router) for the full router behavior.
|
||||||
|
|
||||||
## Fallback Model
|
## Fallback Model
|
||||||
|
|
||||||
Configure a backup provider:model that Hermes switches to automatically when your primary model fails (rate limits, server errors, auth failures):
|
Configure a backup provider:model that Hermes switches to automatically when your primary model fails (rate limits, server errors, auth failures):
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue