feat(openrouter): wire Pareto Code router with min_coding_score knob (#22838)

Pick openrouter/pareto-code as your model and OpenRouter auto-routes each
request to the cheapest model meeting your coding-quality bar (ranked by
Artificial Analysis). The new openrouter.min_coding_score config key (0.0-1.0,
default 0.65) tunes the floor.

- hermes_cli/models.py: add openrouter/pareto-code to OPENROUTER_MODELS so
  it shows up in the picker with a description
- hermes_cli/config.py: add openrouter.min_coding_score (default 0.65 — lands
  on a mid-tier coder on the current Pareto frontier)
- plugins/model-providers/openrouter: emit extra_body.plugins =
  [{id: pareto-router, min_coding_score: X}] when model is openrouter/pareto-code
  AND the score is a valid float in [0.0, 1.0]
- agent/transports/chat_completions.py: same emission on the legacy flag
  path (when no provider profile is loaded)
- run_agent.py: openrouter_min_coding_score kwarg + storage; plumbed into
  both build_kwargs() invocations and the context-summary extra_body path
- cli.py: read openrouter.min_coding_score once at init, validate float in
  [0,1], pass to AIAgent constructions (CLI + background-task paths)
- cron/scheduler.py, batch_runner.py, tools/delegate_tool.py,
  tui_gateway/server.py: propagate the kwarg (mirrors providers_order
  plumbing — subagents inherit, cron/batch read from config)
- tests: profile-level + transport-level coverage of the model gating,
  unset/empty/out-of-range handling, and the legacy flag path
- docs: new 'OpenRouter Pareto Code Router' section in providers.md

Verified end-to-end against api.openrouter.ai: at score=0.65 we land on a
mid-tier coder, at omission we get the strongest. Score is silently dropped
on any model other than openrouter/pareto-code, so it's safe to leave set.
This commit is contained in:
Teknium 2026-05-09 14:47:00 -07:00 committed by GitHub
parent b349ae1e4c
commit c7f0aab949
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 222 additions and 0 deletions

View file

@ -83,6 +83,69 @@ class TestChatCompletionsBuildKwargs:
)
assert kw["extra_body"]["provider"] == {"only": ["openai"]}
def test_openrouter_pareto_min_coding_score(self, transport):
"""Profile path: model=openrouter/pareto-code + score → plugins block."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=0.65,
)
assert kw["extra_body"]["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_openrouter_pareto_score_ignored_for_other_models(self, transport):
"""Score must not be emitted for any model other than openrouter/pareto-code."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=0.65,
)
assert "plugins" not in (kw.get("extra_body") or {})
def test_openrouter_pareto_score_omitted_when_unset(self, transport):
"""No score → no plugins block (router uses its omission default = strongest coder)."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=None,
)
assert "plugins" not in (kw.get("extra_body") or {})
def test_openrouter_pareto_score_out_of_range_dropped(self, transport):
"""Out-of-range scores must be silently dropped, not forwarded."""
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
msgs = [{"role": "user", "content": "Hi"}]
for bad in (1.5, -0.1, "not-a-number"):
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
provider_profile=profile,
openrouter_min_coding_score=bad,
)
assert "plugins" not in (kw.get("extra_body") or {}), f"bad={bad!r}"
def test_openrouter_pareto_legacy_path(self, transport):
"""Legacy flag path (no profile loaded) must also emit the plugins block."""
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="openrouter/pareto-code", messages=msgs,
is_openrouter=True,
openrouter_min_coding_score=0.8,
)
assert kw["extra_body"]["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.8}
]
def test_nous_tags(self, transport):
from providers import get_provider_profile
profile = get_provider_profile("nous")

View file

@ -57,6 +57,7 @@ def _make_background_cli_stub():
cli._provider_sort = None
cli._provider_require_params = None
cli._provider_data_collection = None
cli._openrouter_min_coding_score = None
cli._fallback_model = None
cli._agent_running = False
cli._spinner_text = ""

View file

@ -99,6 +99,46 @@ class TestOpenRouterProfile:
body = p.build_extra_body()
assert body == {}
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=0.65,
)
assert body["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_pareto_score_ignored_for_other_models(self):
"""Score has no effect on any other model — plugins block must not appear."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="anthropic/claude-sonnet-4.6",
openrouter_min_coding_score=0.65,
)
assert "plugins" not in body
def test_pareto_score_unset_omits_plugins(self):
"""Empty/None score → no plugins block (router uses its omission default)."""
p = get_provider_profile("openrouter")
for unset in (None, ""):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=unset,
)
assert "plugins" not in body, f"unset={unset!r}"
def test_pareto_score_out_of_range_dropped(self):
"""Invalid scores are silently dropped — never forwarded to OR."""
p = get_provider_profile("openrouter")
for bad in (1.5, -0.1, "not-a-number"):
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=bad,
)
assert "plugins" not in body, f"bad={bad!r}"
def test_reasoning_full_config(self):
p = get_provider_profile("openrouter")
eb, _ = p.build_api_kwargs_extras(