From 8fe7b52ebf3bafe06d1854ac12011340d7f87099 Mon Sep 17 00:00:00 2001 From: Austin Pickett Date: Thu, 18 Jun 2026 23:33:02 -0400 Subject: [PATCH] =?UTF-8?q?test(desktop):=20lock=20GUI=E2=8A=87`hermes=20m?= =?UTF-8?q?odel`=20provider=20parity;=20surface=20Bedrock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the end-to-end parity contract test: every CANONICAL_PROVIDERS entry (the `hermes model` universe) must be configurable on a desktop Providers tab — keys(/api/env) ∪ ids(/api/providers/oauth) ⊇ canonical. Asserted as an invariant against the live endpoints so the GUI can never silently drift from the CLI again. Surfacing this contract caught Bedrock: it's aws_sdk (no api-key vars), so it had no Keys card. /api/env now tags AWS_REGION/AWS_PROFILE to the bedrock provider card. Anthropic is whitelisted as a legitimate dual-tab provider (direct API key + subscription OAuth). Also refreshes the _OAUTH_PROVIDER_CATALOG docstring to describe its new role as the override base for _build_oauth_catalog(). --- hermes_cli/web_server.py | 38 ++++++++-- tests/hermes_cli/test_provider_parity.py | 90 ++++++++++++++++++++++++ tests/hermes_cli/test_web_server.py | 9 +++ 3 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 tests/hermes_cli/test_provider_parity.py diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index fbdbff3723f..79f7806ab2c 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -4038,6 +4038,24 @@ def _catalog_provider_env_metadata() -> dict: "category": "provider", }, ) + + # AWS-SDK providers (Bedrock) authenticate via the AWS credential chain + # rather than a pasted API key, so they have no api_key_env_vars. Tag + # their AWS_* settings to the provider card so they still appear on the + # Keys tab (otherwise Bedrock — a `hermes model` provider — would be + # invisible in the desktop app). + if d.auth_type == "aws_sdk": + for aws_var in ("AWS_REGION", "AWS_PROFILE"): + existing = meta.get(aws_var, {}) + meta[aws_var] = { + "provider": d.slug, + "provider_label": d.label, + "description": existing.get("description") or f"{d.label} ({aws_var})", + "url": existing.get("url"), + "is_password": False, + "advanced": existing.get("advanced", True), + "category": "provider", + } return meta @@ -5584,13 +5602,19 @@ def _copilot_acp_status() -> Dict[str, Any]: } -# Provider catalog. The order matters — it's how we render the UI list. -# ``cli_command`` is what the dashboard surfaces as the copy-to-clipboard -# fallback while Phase 2 (in-browser flows) isn't built yet. -# ``flow`` describes the OAuth shape so the future modal can pick the -# right UI: ``pkce`` = open URL + paste callback code, ``device_code`` = -# show code + verification URL + poll, ``external`` = read-only (delegated -# to a third-party CLI like Claude Code or Qwen). +# Explicit, hand-tuned OAuth/account provider cards. These carry the bits that +# can't be derived from the unified provider catalog: the OAuth ``flow`` shape, +# the per-provider ``status_fn``, the ``cli_command`` fallback, and curated +# display order. They are the OVERRIDE BASE for ``_build_oauth_catalog()``, +# which unions them with every accounts-tab provider in ``provider_catalog()`` +# so newly-added OAuth/external providers appear automatically (no hand edit). +# This tuple also still includes two entries that are NOT catalog providers but +# must show on the Accounts tab: the api-key Anthropic PKCE card and the +# synthetic ``claude-code`` subscription row. +# ``flow`` describes the OAuth shape so the modal can pick the right UI: +# ``pkce`` = open URL + paste callback code, ``device_code`` = show code + +# verification URL + poll, ``external`` = read-only (delegated to a third-party +# CLI like Claude Code or Qwen), ``loopback`` = 127.0.0.1 callback listener. _OAUTH_PROVIDER_CATALOG: tuple[Dict[str, Any], ...] = ( { "id": "nous", diff --git a/tests/hermes_cli/test_provider_parity.py b/tests/hermes_cli/test_provider_parity.py new file mode 100644 index 00000000000..0f49f260e71 --- /dev/null +++ b/tests/hermes_cli/test_provider_parity.py @@ -0,0 +1,90 @@ +"""End-to-end provider parity contract: the desktop Providers tabs must show +the SAME provider universe as ``hermes model`` (the CLI/TUI picker). + +This is the single load-bearing invariant of the unified provider catalog: + + keys(/api/env provider rows) ∪ ids(/api/providers/oauth) ⊇ CANONICAL_PROVIDERS + +i.e. every provider the CLI picker offers is configurable from the desktop app, +on one of the two Providers sub-tabs (API keys or Accounts). It is asserted as +an invariant against the real FastAPI endpoints (not a snapshot / count), so it +can never silently drift again when a provider plugin is added. +""" + +from fastapi.testclient import TestClient + +from hermes_cli.models import CANONICAL_PROVIDERS +from hermes_cli.provider_catalog import provider_catalog +from hermes_cli.web_server import _SESSION_TOKEN, app + +client = TestClient(app) +HEADERS = {"X-Hermes-Session-Token": _SESSION_TOKEN} + +# `custom` is the bring-your-own-endpoint pseudo-provider configured inline via +# the model picker's local-endpoint flow, not a fixed credential card. It is in +# the CLI picker's universe but intentionally has no dedicated Providers-tab +# card. Exempt it from the union check. +_EXEMPT = {"custom"} + +# Providers that legitimately offer BOTH auth methods and so intentionally +# appear on both desktop tabs (an API-key card AND an account sign-in card). +# Anthropic supports a direct API key (Keys tab) and a subscription OAuth / +# Claude Code login (Accounts tab); surfacing both is correct, not a bug. +_DUAL_TAB = {"anthropic"} + + +def _keys_tab_providers() -> set[str]: + """Provider slugs that have at least one card on the desktop API-keys tab.""" + data = client.get("/api/env", headers=HEADERS).json() + return { + info.get("provider") + for info in data.values() + if info.get("category") == "provider" and info.get("provider") + } + + +def _accounts_tab_providers() -> set[str]: + """Provider slugs offered on the desktop Accounts tab.""" + data = client.get("/api/providers/oauth", headers=HEADERS).json() + return {p["id"] for p in data["providers"]} + + +def test_every_hermes_model_provider_is_configurable_in_desktop(): + """PARITY CONTRACT: GUI (keys ∪ accounts) ⊇ `hermes model` universe.""" + gui = _keys_tab_providers() | _accounts_tab_providers() + missing = [ + e.slug + for e in CANONICAL_PROVIDERS + if e.slug not in _EXEMPT and e.slug not in gui + ] + assert not missing, ( + "providers shown in `hermes model` but not configurable in the desktop " + f"Providers tabs: {missing}" + ) + + +def test_each_provider_lands_on_the_tab_its_auth_type_dictates(): + """A keys-tab provider must surface under /api/env; an accounts-tab provider + under /api/providers/oauth. Cross-checks the catalog's tab routing against + where each provider actually renders. + """ + keys = _keys_tab_providers() + accounts = _accounts_tab_providers() + for d in provider_catalog(): + if d.slug in _EXEMPT: + continue + if d.tab == "keys" and d.api_key_env_vars: + assert d.slug in keys, f"{d.slug} (keys tab) missing from /api/env" + elif d.tab == "accounts": + assert d.slug in accounts, f"{d.slug} (accounts tab) missing from /api/providers/oauth" + + +def test_no_provider_appears_on_both_tabs(): + """A provider should be configured exactly one way — not duplicated across + both tabs (which would confuse users about where to put credentials). + + Exception: genuinely dual-auth providers (see ``_DUAL_TAB``) intentionally + appear on both tabs. + """ + overlap = (_keys_tab_providers() & _accounts_tab_providers()) - _EXEMPT - _DUAL_TAB + assert not overlap, f"providers appearing on BOTH desktop tabs: {sorted(overlap)}" diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 8faf1b8823c..0a5319a0518 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -1341,6 +1341,15 @@ class TestWebServerEndpoints: # Shared GITHUB_TOKEN must NOT be hijacked into the copilot provider card. assert data.get("GITHUB_TOKEN", {}).get("provider", "") != "copilot" + def test_get_env_vars_bedrock_aws_vars_tagged_to_provider(self): + """Bedrock (aws_sdk, no api-key) must still appear on the Keys tab: its + AWS_REGION/AWS_PROFILE settings are tagged to the bedrock provider card. + """ + data = self.client.get("/api/env").json() + assert data["AWS_REGION"]["provider"] == "bedrock" + assert data["AWS_REGION"]["category"] == "provider" + assert data["AWS_PROFILE"]["provider"] == "bedrock" + def test_platform_scoped_messaging_env_vars_are_channel_managed(self): from hermes_cli.web_server import ( _MESSAGING_KEYS_PAGE_KEYS,