mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-06-27 11:22:03 +00:00
fix(models): gate openai-codex/xai-oauth soft-accept to family-shaped slugs (#45006)
Completes the #45006 fix. PR-base commit (configured-provider routing) handles the case where a typed model IS declared in user/custom provider config. This commit closes the other root: when a typed model is NOT in any config and the current provider is a soft-accepting one (openai-codex / xai-oauth), the hidden-model soft-accept (#16172 / #19729) would accept ANY unknown name as a hidden model — so `qwen3.5-4b` typed on a Codex-default session "succeeded" and mislabeled the provider as "OpenAI Codex" (the exact reported symptom), then 400'd on the next turn. Gate the soft-accept to slugs that plausibly belong to the provider's family (openai-codex -> gpt-/codex-/o1/o3/o4; xai-oauth -> grok-). Family-shaped unknown slugs are still soft-accepted (preserving the #16172 entitlement-gated hidden-model intent); unrelated names are rejected with actionable guidance to pin the right provider via `--provider <slug>` or the picker. Adds TestCodexSoftAcceptPlausibilityGate (5 tests): unrelated names rejected on codex/xai, family-shaped hidden slugs still accepted, real catalog models unaffected. Verified load-bearing.
This commit is contained in:
parent
791c992b55
commit
1a174dfb50
2 changed files with 73 additions and 0 deletions
|
|
@ -3793,6 +3793,37 @@ def validate_requested_model(
|
|||
if suggestions:
|
||||
suggestion_text = "\n Similar models: " + ", ".join(f"`{s}`" for s in suggestions)
|
||||
provider_label = "OpenAI Codex" if normalized == "openai-codex" else "xAI Grok OAuth (SuperGrok / Premium+)"
|
||||
# Plausibility gate (#45006): the soft-accept (#16172 / #19729) exists
|
||||
# for entitlement-gated *hidden* slugs the curated listing hasn't
|
||||
# caught up with — but those are always the provider's own family
|
||||
# (openai-codex -> gpt-*; xai-oauth -> grok-*). Accepting an
|
||||
# unrelated typed name (e.g. `qwen3.5-4b`, `llama-3.1-8b`) here turns
|
||||
# what should be an actionable "did you mean --provider <x>?" error
|
||||
# into a confusing success that 400s on the next turn. Only soft-
|
||||
# accept names that share the provider's family prefix; reject the
|
||||
# rest with guidance to pin the right provider.
|
||||
_family_prefixes = {
|
||||
"openai-codex": ("gpt-", "codex-", "o1", "o3", "o4"),
|
||||
"xai-oauth": ("grok-",),
|
||||
}.get(normalized, ())
|
||||
_lower = requested_for_lookup.strip().lower()
|
||||
_plausible = (not _family_prefixes) or any(
|
||||
_lower.startswith(p) for p in _family_prefixes
|
||||
)
|
||||
if not _plausible:
|
||||
return {
|
||||
"accepted": False,
|
||||
"persist": False,
|
||||
"recognized": False,
|
||||
"message": (
|
||||
f"`{requested}` doesn't look like a {provider_label} model "
|
||||
f"and isn't in its listing, so it was not accepted. If it "
|
||||
f"belongs to another configured provider, switch with "
|
||||
f"`--provider <slug>` (or select it from the `/model` "
|
||||
f"picker)."
|
||||
f"{suggestion_text}"
|
||||
),
|
||||
}
|
||||
return {
|
||||
"accepted": True,
|
||||
"persist": True,
|
||||
|
|
|
|||
|
|
@ -907,3 +907,45 @@ class TestNousRecommendedModels:
|
|||
patch("hermes_cli.models.check_nous_free_tier", side_effect=RuntimeError("boom")),
|
||||
):
|
||||
assert get_nous_recommended_aux_model(vision=False) == "paid-model"
|
||||
|
||||
|
||||
class TestCodexSoftAcceptPlausibilityGate:
|
||||
"""#45006 kernel (b): the openai-codex / xai-oauth hidden-model soft-accept
|
||||
(#16172 / #19729) must only accept slugs that plausibly belong to that
|
||||
provider's family. An undeclared, unrelated typed name (e.g. a local model
|
||||
name) must be REJECTED with actionable --provider guidance instead of being
|
||||
fake-accepted as a hidden Codex/Grok model (which would 400 on the next turn
|
||||
and mislabel the provider as 'OpenAI Codex')."""
|
||||
|
||||
def test_unrelated_name_rejected_on_openai_codex(self):
|
||||
from hermes_cli.models import validate_requested_model
|
||||
r = validate_requested_model("qwen3.5-4b", "openai-codex")
|
||||
assert r["accepted"] is False
|
||||
assert r["persist"] is False
|
||||
assert "--provider" in (r["message"] or "")
|
||||
|
||||
def test_unrelated_name_rejected_on_xai_oauth(self):
|
||||
from hermes_cli.models import validate_requested_model
|
||||
r = validate_requested_model("llama-3.1-8b", "xai-oauth")
|
||||
assert r["accepted"] is False
|
||||
assert "--provider" in (r["message"] or "")
|
||||
|
||||
def test_family_shaped_hidden_slug_still_soft_accepted_codex(self):
|
||||
"""#16172 intent preserved: a gpt-/codex-shaped unknown slug is still
|
||||
soft-accepted (entitlement-gated hidden models)."""
|
||||
from hermes_cli.models import validate_requested_model
|
||||
r = validate_requested_model("gpt-5.9-codex-hidden", "openai-codex")
|
||||
assert r["accepted"] is True
|
||||
assert r["recognized"] is False
|
||||
|
||||
def test_family_shaped_hidden_slug_still_soft_accepted_xai(self):
|
||||
from hermes_cli.models import validate_requested_model
|
||||
r = validate_requested_model("grok-9-hidden", "xai-oauth")
|
||||
assert r["accepted"] is True
|
||||
assert r["recognized"] is False
|
||||
|
||||
def test_real_catalog_model_unaffected(self):
|
||||
from hermes_cli.models import validate_requested_model
|
||||
r = validate_requested_model("gpt-5.5", "openai-codex")
|
||||
assert r["accepted"] is True
|
||||
assert r["recognized"] is True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue