mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-05-18 04:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

feat(openrouter): wire Pareto Code router with min_coding_score knob (#22838)

Browse source 
Pick openrouter/pareto-code as your model and OpenRouter auto-routes each
request to the cheapest model meeting your coding-quality bar (ranked by
Artificial Analysis). The new openrouter.min_coding_score config key (0.0-1.0,
default 0.65) tunes the floor.

- hermes_cli/models.py: add openrouter/pareto-code to OPENROUTER_MODELS so
  it shows up in the picker with a description
- hermes_cli/config.py: add openrouter.min_coding_score (default 0.65 — lands
  on a mid-tier coder on the current Pareto frontier)
- plugins/model-providers/openrouter: emit extra_body.plugins =
  [{id: pareto-router, min_coding_score: X}] when model is openrouter/pareto-code
  AND the score is a valid float in [0.0, 1.0]
- agent/transports/chat_completions.py: same emission on the legacy flag
  path (when no provider profile is loaded)
- run_agent.py: openrouter_min_coding_score kwarg + storage; plumbed into
  both build_kwargs() invocations and the context-summary extra_body path
- cli.py: read openrouter.min_coding_score once at init, validate float in
  [0,1], pass to AIAgent constructions (CLI + background-task paths)
- cron/scheduler.py, batch_runner.py, tools/delegate_tool.py,
  tui_gateway/server.py: propagate the kwarg (mirrors providers_order
  plumbing — subagents inherit, cron/batch read from config)
- tests: profile-level + transport-level coverage of the model gating,
  unset/empty/out-of-range handling, and the legacy flag path
- docs: new 'OpenRouter Pareto Code Router' section in providers.md

Verified end-to-end against api.openrouter.ai: at score=0.65 we land on a
mid-tier coder, at omission we get the strongest. Score is silently dropped
on any model other than openrouter/pareto-code, so it's safe to leave set.
This commit is contained in:
Teknium 2026-05-09 14:47:00 -07:00 • committed by GitHub
parent b349ae1e4c
commit c7f0aab949
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 222 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

16
agent/transports/chat_completions.py
View file

@ -323,6 +323,21 @@ class ChatCompletionsTransport(ProviderTransport):
if provider_prefs and is_openrouter:
extra_body["provider"] = provider_prefs
# Pareto Code router plugin — model-gated. Same shape as the
# profile path in plugins/model-providers/openrouter/__init__.py;
# this branch only runs when the OpenRouter profile isn't loaded.
if is_openrouter and model == "openrouter/pareto-code":
_pareto_score = params.get("openrouter_min_coding_score")
if _pareto_score is not None and _pareto_score != "":
try:
_pareto_score_f = float(_pareto_score)
except (TypeError, ValueError):
_pareto_score_f = None
if _pareto_score_f is not None and 0.0 <= _pareto_score_f <= 1.0:
extra_body["plugins"] = [
{"id": "pareto-router", "min_coding_score": _pareto_score_f}
]
# Kimi extra_body.thinking
if is_kimi:
_kimi_thinking_enabled = True
@ -463,6 +478,7 @@ class ChatCompletionsTransport(ProviderTransport):
model=model,
base_url=params.get("base_url"),
reasoning_config=reasoning_config,
openrouter_min_coding_score=params.get("openrouter_min_coding_score"),
)
if profile_body:
extra_body.update(profile_body)

4
batch_runner.py
View file

@ -337,6 +337,7 @@ def _process_single_prompt(
providers_ignored=config.get("providers_ignored"),
providers_order=config.get("providers_order"),
provider_sort=config.get("provider_sort"),
openrouter_min_coding_score=config.get("openrouter_min_coding_score"),
max_tokens=config.get("max_tokens"),
reasoning_config=config.get("reasoning_config"),
prefill_messages=config.get("prefill_messages"),
@ -546,6 +547,7 @@ class BatchRunner:
providers_ignored: List[str] = None,
providers_order: List[str] = None,
provider_sort: str = None,
openrouter_min_coding_score: Optional[float] = None,
max_tokens: int = None,
reasoning_config: Dict[str, Any] = None,
prefill_messages: List[Dict[str, Any]] = None,
@ -595,6 +597,7 @@ class BatchRunner:
self.providers_ignored = providers_ignored
self.providers_order = providers_order
self.provider_sort = provider_sort
self.openrouter_min_coding_score = openrouter_min_coding_score
self.max_tokens = max_tokens
self.reasoning_config = reasoning_config
self.prefill_messages = prefill_messages
@ -873,6 +876,7 @@ class BatchRunner:
"providers_ignored": self.providers_ignored,
"providers_order": self.providers_order,
"provider_sort": self.provider_sort,
"openrouter_min_coding_score": self.openrouter_min_coding_score,
"max_tokens": self.max_tokens,
"reasoning_config": self.reasoning_config,
"prefill_messages": self.prefill_messages,

16
cli.py
View file

@ -2473,6 +2473,20 @@ class HermesCLI:
self._providers_order = pr.get("order")
self._provider_require_params = pr.get("require_parameters", False)
self._provider_data_collection = pr.get("data_collection")
# OpenRouter Pareto Code router knob — coding-score floor (0.0-1.0).
# Only applied when model.model == "openrouter/pareto-code".
# Empty string / None / out-of-range = unset (let OR pick strongest coder).
_or_cfg = CLI_CONFIG.get("openrouter", {}) or {}
_raw_score = _or_cfg.get("min_coding_score")
self._openrouter_min_coding_score: Optional[float] = None
if _raw_score not in (None, ""):
try:
_f = float(_raw_score)
if 0.0 <= _f <= 1.0:
self._openrouter_min_coding_score = _f
except (TypeError, ValueError):
pass
# Fallback provider chain — tried in order when primary fails after retries.
# Supports new list format (fallback_providers) and legacy single-dict (fallback_model).
@ -4031,6 +4045,7 @@ class HermesCLI:
provider_sort=self._provider_sort,
provider_require_parameters=self._provider_require_params,
provider_data_collection=self._provider_data_collection,
openrouter_min_coding_score=self._openrouter_min_coding_score,
session_id=self.session_id,
platform="cli",
session_db=self._session_db,
@ -7249,6 +7264,7 @@ class HermesCLI:
provider_sort=self._provider_sort,
provider_require_parameters=self._provider_require_params,
provider_data_collection=self._provider_data_collection,
openrouter_min_coding_score=self._openrouter_min_coding_score,
fallback_model=self._fallback_model,
)
# Silence raw spinner; route thinking through TUI widget when no foreground agent is active.

1
cron/scheduler.py
View file

@ -1439,6 +1439,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
providers_ignored=pr.get("ignore"),
providers_order=pr.get("order"),
provider_sort=pr.get("sort"),
openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
disabled_toolsets=["cronjob", "messaging", "clarify"],
quiet_mode=True,

9
hermes_cli/config.py
View file

@ -691,9 +691,18 @@ DEFAULT_CONFIG = {
# See: https://openrouter.ai/docs/guides/features/response-caching
# response_cache_ttl: how long cached responses remain valid, in seconds (1-86400).
# Default 300 (5 minutes). Only used when response_cache is enabled.
# min_coding_score: knob for the openrouter/pareto-code router (0.0-1.0).
# Only applied when model.model is "openrouter/pareto-code". Higher
# values route to stronger (more expensive) coders; lower values open
# up cheaper, faster options. Default 0.65 lands on the mid-tier
# coder on the current Pareto frontier. Empty string = let OpenRouter
# pick the strongest available coder (router's documented default
# when the plugins block is omitted).
# See: https://openrouter.ai/docs/guides/routing/routers/pareto-router
"openrouter": {
"response_cache": True,
"response_cache_ttl": 300,
"min_coding_score": 0.65,
},
# AWS Bedrock provider configuration.

1
hermes_cli/models.py
View file

@ -33,6 +33,7 @@ COPILOT_REASONING_EFFORTS_O_SERIES = ["low", "medium", "high"]
# (model_id, display description shown in menus)
OPENROUTER_MODELS: list[tuple[str, str]] = [
("moonshotai/kimi-k2.6", "recommended"),
("openrouter/pareto-code", "auto-routes to cheapest coder meeting openrouter.min_coding_score"),
("anthropic/claude-opus-4.7", ""),
("anthropic/claude-opus-4.6", ""),
("anthropic/claude-sonnet-4.6", ""),

17
plugins/model-providers/openrouter/__init__.py
View file

@ -46,6 +46,23 @@ class OpenRouterProfile(ProviderProfile):
prefs = context.get("provider_preferences")
if prefs:
body["provider"] = prefs
# Pareto Code router — model-gated. The plugins block is only
# meaningful for openrouter/pareto-code; sending it on any other
# model has no documented effect and would be confusing in logs.
# See: https://openrouter.ai/docs/guides/routing/routers/pareto-router
model = (context.get("model") or "")
if model == "openrouter/pareto-code":
score = context.get("openrouter_min_coding_score")
if score is not None and score != "":
try:
score_f = float(score)
except (TypeError, ValueError):
score_f = None
if score_f is not None and 0.0 <= score_f <= 1.0:
body["plugins"] = [
{"id": "pareto-router", "min_coding_score": score_f}
]
return body
def build_api_kwargs_extras(

28
run_agent.py
View file

@ -1075,6 +1075,7 @@ class AIAgent:
provider_sort: str = None,
provider_require_parameters: bool = False,
provider_data_collection: str = None,
openrouter_min_coding_score: Optional[float] = None,
session_id: str = None,
tool_progress_callback: callable = None,
tool_start_callback: callable = None,
@ -1137,6 +1138,9 @@ class AIAgent:
providers_ignored (List[str]): OpenRouter providers to ignore (optional)
providers_order (List[str]): OpenRouter providers to try in order (optional)
provider_sort (str): Sort providers by price/throughput/latency (optional)
openrouter_min_coding_score (float): Coding-score floor (0.0-1.0) for the
openrouter/pareto-code router. Only applied when model == "openrouter/pareto-code".
None or empty = let OpenRouter pick the strongest available coder.
session_id (str): Pre-generated session ID for logging (optional, auto-generated if not provided)
tool_progress_callback (callable): Callback function(tool_name, args_preview) for progress notifications
clarify_callback (callable): Callback function(question, choices) -> str for interactive user questions.
@ -1356,6 +1360,7 @@ class AIAgent:
self.provider_sort = provider_sort
self.provider_require_parameters = provider_require_parameters
self.provider_data_collection = provider_data_collection
self.openrouter_min_coding_score = openrouter_min_coding_score
# Store toolset filtering options
self.enabled_toolsets = enabled_toolsets
@ -9029,6 +9034,7 @@ class AIAgent:
ollama_num_ctx=self._ollama_num_ctx,
# Context forwarded to profile hooks:
provider_preferences=_prefs or None,
openrouter_min_coding_score=self.openrouter_min_coding_score,
anthropic_max_output=_ant_max,
supports_reasoning=self._supports_reasoning_extra_body(),
qwen_session_metadata=_qwen_meta,
@ -9068,6 +9074,7 @@ class AIAgent:
is_custom_provider=self.provider == "custom",
ollama_num_ctx=self._ollama_num_ctx,
provider_preferences=_prefs or None,
openrouter_min_coding_score=self.openrouter_min_coding_score,
qwen_prepare_fn=self._qwen_prepare_chat_messages if _is_qwen else None,
qwen_prepare_inplace_fn=self._qwen_prepare_chat_messages_inplace if _is_qwen else None,
qwen_session_metadata=_qwen_meta,
@ -10974,6 +10981,27 @@ class AIAgent:
):
summary_extra_body["provider"] = provider_preferences
# Pareto Code router plugin — model-gated. Same shape as
# the main-loop emission so summary calls on
# openrouter/pareto-code respect the user's coding-score floor.
if (
self.model == "openrouter/pareto-code"
and (
(self.provider or "").strip().lower() == "openrouter"
or self._is_openrouter_url()
)
and self.openrouter_min_coding_score is not None
and self.openrouter_min_coding_score != ""
):
try:
_ps = float(self.openrouter_min_coding_score)
except (TypeError, ValueError):
_ps = None
if _ps is not None and 0.0 <= _ps <= 1.0:
summary_extra_body["plugins"] = [
{"id": "pareto-router", "min_coding_score": _ps}
]
if summary_extra_body:
summary_kwargs["extra_body"] = summary_extra_body

63
tests/agent/transports/test_chat_completions.py
View file

@ -83,6 +83,69 @@ class TestChatCompletionsBuildKwargs:
)
assert kw["extra_body"]["provider"] == {"only": ["openai"]}
def test_openrouter_pareto_min_coding_score(self, transport):
"""Profile path: model=openrouter/pareto-code + score → plugins block."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=0.65,
)
assert kw["extra_body"]["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_openrouter_pareto_score_ignored_for_other_models(self, transport):
"""Score must not be emitted for any model other than openrouter/pareto-code."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=0.65,
)
assert "plugins" not in (kw.get("extra_body") or {})
def test_openrouter_pareto_score_omitted_when_unset(self, transport):
"""No score → no plugins block (router uses its omission default = strongest coder)."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=None,
)
assert "plugins" not in (kw.get("extra_body") or {})
def test_openrouter_pareto_score_out_of_range_dropped(self, transport):
"""Out-of-range scores must be silently dropped, not forwarded."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
for bad in (1.5, -0.1, "not-a-number"):
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=bad,
)
assert "plugins" not in (kw.get("extra_body") or {}), f"bad={bad!r}"
def test_openrouter_pareto_legacy_path(self, transport):
"""Legacy flag path (no profile loaded) must also emit the plugins block."""
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
is_openrouter=True,
openrouter_min_coding_score=0.8,
)
assert kw["extra_body"]["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.8}
]
def test_nous_tags(self, transport):
from providers import get_provider_profile
profile = get_provider_profile("nous")

1
tests/cli/test_cli_approval_ui.py
View file

@ -57,6 +57,7 @@ def _make_background_cli_stub():
cli._provider_sort = None
cli._provider_require_params = None
cli._provider_data_collection = None
cli._openrouter_min_coding_score = None
cli._fallback_model = None
cli._agent_running = False
cli._spinner_text = ""

40
tests/providers/test_provider_profiles.py
View file

@ -99,6 +99,46 @@ class TestOpenRouterProfile:
body = p.build_extra_body()
assert body == {}
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=0.65,
)
assert body["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_pareto_score_ignored_for_other_models(self):
"""Score has no effect on any other model — plugins block must not appear."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="anthropic/claude-sonnet-4.6",
openrouter_min_coding_score=0.65,
)
assert "plugins" not in body
def test_pareto_score_unset_omits_plugins(self):
"""Empty/None score → no plugins block (router uses its omission default)."""
p = get_provider_profile("openrouter")
for unset in (None, ""):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=unset,
)
assert "plugins" not in body, f"unset={unset!r}"
def test_pareto_score_out_of_range_dropped(self):
"""Invalid scores are silently dropped — never forwarded to OR."""
p = get_provider_profile("openrouter")
for bad in (1.5, -0.1, "not-a-number"):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=bad,
)
assert "plugins" not in body, f"bad={bad!r}"
def test_reasoning_full_config(self):
p = get_provider_profile("openrouter")
eb, _ = p.build_api_kwargs_extras(

5
tools/delegate_tool.py
View file

@ -1077,11 +1077,15 @@ def _build_child_agent(
child_providers_ignored = getattr(parent_agent, "providers_ignored", None)
child_providers_order = getattr(parent_agent, "providers_order", None)
child_provider_sort = getattr(parent_agent, "provider_sort", None)
child_openrouter_min_coding_score = getattr(parent_agent, "openrouter_min_coding_score", None)
if override_provider:
child_providers_allowed = None
child_providers_ignored = None
child_providers_order = None
child_provider_sort = None
# Note: openrouter_min_coding_score is model-gated (only emitted on
# openrouter/pareto-code), so we keep it inherited even when the
# provider is overridden — it's a no-op on any other model.
child = AIAgent(
base_url=effective_base_url,
@ -1111,6 +1115,7 @@ def _build_child_agent(
providers_ignored=child_providers_ignored,
providers_order=child_providers_order,
provider_sort=child_provider_sort,
openrouter_min_coding_score=child_openrouter_min_coding_score,
tool_progress_callback=child_progress_cb,
iteration_budget=None, # fresh budget per subagent
)

1
tui_gateway/server.py
View file

@ -1817,6 +1817,7 @@ def _background_agent_kwargs(agent, task_id: str) -> dict:
agent, "provider_require_parameters", False
),
"provider_data_collection": getattr(agent, "provider_data_collection", None),
"openrouter_min_coding_score": getattr(agent, "openrouter_min_coding_score", None),
"session_id": task_id,
"reasoning_config": getattr(agent, "reasoning_config", None)
or _load_reasoning_config(),

20
website/docs/integrations/providers.md
View file

@ -1372,6 +1372,26 @@ provider_routing:
**Shortcuts:** Append `:nitro` to any model name for throughput sorting (e.g., `anthropic/claude-sonnet-4:nitro`), or `:floor` for price sorting.
## OpenRouter Pareto Code Router
OpenRouter ships an experimental coding-model router at `openrouter/pareto-code` that auto-routes requests to the cheapest model meeting a coding-quality bar (ranked by [Artificial Analysis](https://artificialanalysis.ai/)). Pick this model and tune the `min_coding_score` knob in `~/.hermes/config.yaml`:
```yaml
model:
provider: openrouter
model: openrouter/pareto-code
openrouter:
min_coding_score: 0.65 # 0.0–1.0; higher = stronger (more expensive) coders. Default 0.65.
```
Notes:
- `min_coding_score` is **only** sent when `model.model` is `openrouter/pareto-code`. On any other model the value is a no-op.
- Set to empty string (or remove the line) to let OpenRouter pick the strongest available coder — its documented behavior when the plugins block is omitted.
- Selection is deterministic per score on a given day, but the actual model chosen can shift as the Pareto frontier moves (new models, benchmark updates).
- See OpenRouter's [Pareto Router docs](https://openrouter.ai/docs/guides/routing/routers/pareto-router) for the full router behavior.
## Fallback Model
Configure a backup provider:model that Hermes switches to automatically when your primary model fails (rate limits, server errors, auth failures):