mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
78 lines
3.1 KiB
Python
78 lines
3.1 KiB
Python
"""Tests for the unified provider catalog (hermes_cli.provider_catalog).
|
|
|
|
These are invariant tests, not snapshots: they assert the parity *contract*
|
|
between what ``hermes model`` shows (``CANONICAL_PROVIDERS``) and what the
|
|
catalog exposes, plus how each provider's ``auth_type`` maps to a desktop tab —
|
|
never a specific provider count or a frozen vendor list (both change over time).
|
|
"""
|
|
|
|
from hermes_cli.models import CANONICAL_PROVIDERS
|
|
from hermes_cli.provider_catalog import (
|
|
ProviderDescriptor,
|
|
provider_catalog,
|
|
provider_catalog_by_slug,
|
|
tab_for_auth_type,
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_profileless_providers_still_present():
|
|
"""Providers without a ProviderProfile must still resolve via fallbacks.
|
|
|
|
lmstudio / openai-api / tencent-tokenhub / xai-oauth have no profile on
|
|
main; they exist only as registry + canonical entries. The catalog must
|
|
not require a profile to include a provider.
|
|
"""
|
|
by = provider_catalog_by_slug()
|
|
for slug in ("lmstudio", "openai-api", "tencent-tokenhub", "xai-oauth"):
|
|
assert slug in by, f"{slug} dropped from catalog (profile-less provider)"
|
|
assert by[slug].label, f"{slug} has empty label despite canonical fallback"
|
|
assert by[slug].description, f"{slug} has empty description despite fallback"
|
|
|
|
|
|
def test_copilot_surfaces_as_a_provider_with_its_own_token_var():
|
|
"""Regression for the reported bug: a GitHub Copilot login showed up under
|
|
tools, never as a provider, because the shared GITHUB_TOKEN is tool-category.
|
|
|
|
Copilot authenticates via the `copilot`/api_key path, so it belongs on the
|
|
keys tab — but its PRIMARY credential var must be the provider-owned
|
|
COPILOT_GITHUB_TOKEN, not the shared tool-category GITHUB_TOKEN. That is what
|
|
lets the desktop render Copilot as its own provider card.
|
|
"""
|
|
by = provider_catalog_by_slug()
|
|
assert "copilot" in by
|
|
d = by["copilot"]
|
|
assert d.tab == "keys"
|
|
assert d.api_key_env_vars, "Copilot must expose a credential env var"
|
|
assert d.api_key_env_vars[0] == "COPILOT_GITHUB_TOKEN", (
|
|
"Copilot's primary var must be the provider-owned token, not shared GITHUB_TOKEN"
|
|
)
|
|
|
|
|
|
def test_api_key_providers_expose_a_credential_env_var():
|
|
"""Every keys-tab provider that authenticates via a pasted API key must
|
|
surface at least one env var to write the key into (otherwise the GUI can't
|
|
configure it).
|
|
|
|
Exemptions: ``aws_sdk`` (bedrock — uses AWS_REGION/AWS_PROFILE) and the
|
|
``custom`` bring-your-own-endpoint pseudo-provider, which is configured
|
|
inline via the local-endpoint flow rather than a fixed env var.
|
|
"""
|
|
exempt = {"custom"}
|
|
for d in provider_catalog():
|
|
if d.auth_type == "api_key" and d.slug not in exempt:
|
|
assert d.api_key_env_vars, f"{d.slug} is api_key but exposes no env var"
|
|
|
|
|
|
|
|
|
|
def test_tab_for_auth_type_helper():
|
|
assert tab_for_auth_type("api_key") == "keys"
|
|
assert tab_for_auth_type("aws_sdk") == "keys"
|
|
assert tab_for_auth_type("oauth_external") == "accounts"
|
|
assert tab_for_auth_type("oauth_device_code") == "accounts"
|
|
assert tab_for_auth_type("copilot") == "accounts"
|
|
assert tab_for_auth_type("external_process") == "accounts"
|