test(desktop): lock GUI⊇hermes model provider parity; surface Bedrock

Adds the end-to-end parity contract test: every CANONICAL_PROVIDERS entry (the
`hermes model` universe) must be configurable on a desktop Providers tab —
keys(/api/env) ∪ ids(/api/providers/oauth) ⊇ canonical. Asserted as an
invariant against the live endpoints so the GUI can never silently drift from
the CLI again.

Surfacing this contract caught Bedrock: it's aws_sdk (no api-key vars), so it
had no Keys card. /api/env now tags AWS_REGION/AWS_PROFILE to the bedrock
provider card. Anthropic is whitelisted as a legitimate dual-tab provider
(direct API key + subscription OAuth).

Also refreshes the _OAUTH_PROVIDER_CATALOG docstring to describe its new role
as the override base for _build_oauth_catalog().
This commit is contained in:
Austin Pickett 2026-06-18 23:33:02 -04:00 committed by Teknium
parent 6cb04be779
commit 8fe7b52ebf
3 changed files with 130 additions and 7 deletions

View file

@ -4038,6 +4038,24 @@ def _catalog_provider_env_metadata() -> dict:
"category": "provider",
},
)
# AWS-SDK providers (Bedrock) authenticate via the AWS credential chain
# rather than a pasted API key, so they have no api_key_env_vars. Tag
# their AWS_* settings to the provider card so they still appear on the
# Keys tab (otherwise Bedrock — a `hermes model` provider — would be
# invisible in the desktop app).
if d.auth_type == "aws_sdk":
for aws_var in ("AWS_REGION", "AWS_PROFILE"):
existing = meta.get(aws_var, {})
meta[aws_var] = {
"provider": d.slug,
"provider_label": d.label,
"description": existing.get("description") or f"{d.label} ({aws_var})",
"url": existing.get("url"),
"is_password": False,
"advanced": existing.get("advanced", True),
"category": "provider",
}
return meta
@ -5584,13 +5602,19 @@ def _copilot_acp_status() -> Dict[str, Any]:
}
# Provider catalog. The order matters — it's how we render the UI list.
# ``cli_command`` is what the dashboard surfaces as the copy-to-clipboard
# fallback while Phase 2 (in-browser flows) isn't built yet.
# ``flow`` describes the OAuth shape so the future modal can pick the
# right UI: ``pkce`` = open URL + paste callback code, ``device_code`` =
# show code + verification URL + poll, ``external`` = read-only (delegated
# to a third-party CLI like Claude Code or Qwen).
# Explicit, hand-tuned OAuth/account provider cards. These carry the bits that
# can't be derived from the unified provider catalog: the OAuth ``flow`` shape,
# the per-provider ``status_fn``, the ``cli_command`` fallback, and curated
# display order. They are the OVERRIDE BASE for ``_build_oauth_catalog()``,
# which unions them with every accounts-tab provider in ``provider_catalog()``
# so newly-added OAuth/external providers appear automatically (no hand edit).
# This tuple also still includes two entries that are NOT catalog providers but
# must show on the Accounts tab: the api-key Anthropic PKCE card and the
# synthetic ``claude-code`` subscription row.
# ``flow`` describes the OAuth shape so the modal can pick the right UI:
# ``pkce`` = open URL + paste callback code, ``device_code`` = show code +
# verification URL + poll, ``external`` = read-only (delegated to a third-party
# CLI like Claude Code or Qwen), ``loopback`` = 127.0.0.1 callback listener.
_OAUTH_PROVIDER_CATALOG: tuple[Dict[str, Any], ...] = (
{
"id": "nous",

View file

@ -0,0 +1,90 @@
"""End-to-end provider parity contract: the desktop Providers tabs must show
the SAME provider universe as ``hermes model`` (the CLI/TUI picker).
This is the single load-bearing invariant of the unified provider catalog:
keys(/api/env provider rows) ids(/api/providers/oauth) CANONICAL_PROVIDERS
i.e. every provider the CLI picker offers is configurable from the desktop app,
on one of the two Providers sub-tabs (API keys or Accounts). It is asserted as
an invariant against the real FastAPI endpoints (not a snapshot / count), so it
can never silently drift again when a provider plugin is added.
"""
from fastapi.testclient import TestClient
from hermes_cli.models import CANONICAL_PROVIDERS
from hermes_cli.provider_catalog import provider_catalog
from hermes_cli.web_server import _SESSION_TOKEN, app
client = TestClient(app)
HEADERS = {"X-Hermes-Session-Token": _SESSION_TOKEN}
# `custom` is the bring-your-own-endpoint pseudo-provider configured inline via
# the model picker's local-endpoint flow, not a fixed credential card. It is in
# the CLI picker's universe but intentionally has no dedicated Providers-tab
# card. Exempt it from the union check.
_EXEMPT = {"custom"}
# Providers that legitimately offer BOTH auth methods and so intentionally
# appear on both desktop tabs (an API-key card AND an account sign-in card).
# Anthropic supports a direct API key (Keys tab) and a subscription OAuth /
# Claude Code login (Accounts tab); surfacing both is correct, not a bug.
_DUAL_TAB = {"anthropic"}
def _keys_tab_providers() -> set[str]:
"""Provider slugs that have at least one card on the desktop API-keys tab."""
data = client.get("/api/env", headers=HEADERS).json()
return {
info.get("provider")
for info in data.values()
if info.get("category") == "provider" and info.get("provider")
}
def _accounts_tab_providers() -> set[str]:
"""Provider slugs offered on the desktop Accounts tab."""
data = client.get("/api/providers/oauth", headers=HEADERS).json()
return {p["id"] for p in data["providers"]}
def test_every_hermes_model_provider_is_configurable_in_desktop():
"""PARITY CONTRACT: GUI (keys accounts) ⊇ `hermes model` universe."""
gui = _keys_tab_providers() | _accounts_tab_providers()
missing = [
e.slug
for e in CANONICAL_PROVIDERS
if e.slug not in _EXEMPT and e.slug not in gui
]
assert not missing, (
"providers shown in `hermes model` but not configurable in the desktop "
f"Providers tabs: {missing}"
)
def test_each_provider_lands_on_the_tab_its_auth_type_dictates():
"""A keys-tab provider must surface under /api/env; an accounts-tab provider
under /api/providers/oauth. Cross-checks the catalog's tab routing against
where each provider actually renders.
"""
keys = _keys_tab_providers()
accounts = _accounts_tab_providers()
for d in provider_catalog():
if d.slug in _EXEMPT:
continue
if d.tab == "keys" and d.api_key_env_vars:
assert d.slug in keys, f"{d.slug} (keys tab) missing from /api/env"
elif d.tab == "accounts":
assert d.slug in accounts, f"{d.slug} (accounts tab) missing from /api/providers/oauth"
def test_no_provider_appears_on_both_tabs():
"""A provider should be configured exactly one way — not duplicated across
both tabs (which would confuse users about where to put credentials).
Exception: genuinely dual-auth providers (see ``_DUAL_TAB``) intentionally
appear on both tabs.
"""
overlap = (_keys_tab_providers() & _accounts_tab_providers()) - _EXEMPT - _DUAL_TAB
assert not overlap, f"providers appearing on BOTH desktop tabs: {sorted(overlap)}"

View file

@ -1341,6 +1341,15 @@ class TestWebServerEndpoints:
# Shared GITHUB_TOKEN must NOT be hijacked into the copilot provider card.
assert data.get("GITHUB_TOKEN", {}).get("provider", "") != "copilot"
def test_get_env_vars_bedrock_aws_vars_tagged_to_provider(self):
"""Bedrock (aws_sdk, no api-key) must still appear on the Keys tab: its
AWS_REGION/AWS_PROFILE settings are tagged to the bedrock provider card.
"""
data = self.client.get("/api/env").json()
assert data["AWS_REGION"]["provider"] == "bedrock"
assert data["AWS_REGION"]["category"] == "provider"
assert data["AWS_PROFILE"]["provider"] == "bedrock"
def test_platform_scoped_messaging_env_vars_are_channel_managed(self):
from hermes_cli.web_server import (
_MESSAGING_KEYS_PAGE_KEYS,