mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-11 03:31:55 +00:00
Merge remote-tracking branch 'origin/main' into feat/dashboard-chat
This commit is contained in:
commit
1cd2b280fd
373 changed files with 35795 additions and 7622 deletions
|
|
@ -15,6 +15,8 @@ from hermes_cli.auth import (
|
|||
get_auth_status,
|
||||
AuthError,
|
||||
KIMI_CODE_BASE_URL,
|
||||
STEPFUN_STEP_PLAN_INTL_BASE_URL,
|
||||
STEPFUN_STEP_PLAN_CN_BASE_URL,
|
||||
_resolve_kimi_base_url,
|
||||
)
|
||||
from hermes_cli.copilot_auth import _try_gh_cli_token
|
||||
|
|
@ -35,6 +37,7 @@ class TestProviderRegistry:
|
|||
("xai", "xAI", "api_key"),
|
||||
("nvidia", "NVIDIA NIM", "api_key"),
|
||||
("kimi-coding", "Kimi / Moonshot", "api_key"),
|
||||
("stepfun", "StepFun Step Plan", "api_key"),
|
||||
("minimax", "MiniMax", "api_key"),
|
||||
("minimax-cn", "MiniMax (China)", "api_key"),
|
||||
("ai-gateway", "Vercel AI Gateway", "api_key"),
|
||||
|
|
@ -71,7 +74,11 @@ class TestProviderRegistry:
|
|||
|
||||
def test_kimi_env_vars(self):
|
||||
pconfig = PROVIDER_REGISTRY["kimi-coding"]
|
||||
assert pconfig.api_key_env_vars == ("KIMI_API_KEY",)
|
||||
# KIMI_API_KEY is the primary env var; KIMI_CODING_API_KEY is a
|
||||
# secondary fallback for Kimi Code sk-kimi- keys so users don't
|
||||
# have to overload the same variable.
|
||||
assert "KIMI_API_KEY" in pconfig.api_key_env_vars
|
||||
assert "KIMI_CODING_API_KEY" in pconfig.api_key_env_vars
|
||||
assert pconfig.base_url_env_var == "KIMI_BASE_URL"
|
||||
|
||||
def test_minimax_env_vars(self):
|
||||
|
|
@ -79,6 +86,11 @@ class TestProviderRegistry:
|
|||
assert pconfig.api_key_env_vars == ("MINIMAX_API_KEY",)
|
||||
assert pconfig.base_url_env_var == "MINIMAX_BASE_URL"
|
||||
|
||||
def test_stepfun_env_vars(self):
|
||||
pconfig = PROVIDER_REGISTRY["stepfun"]
|
||||
assert pconfig.api_key_env_vars == ("STEPFUN_API_KEY",)
|
||||
assert pconfig.base_url_env_var == "STEPFUN_BASE_URL"
|
||||
|
||||
def test_minimax_cn_env_vars(self):
|
||||
pconfig = PROVIDER_REGISTRY["minimax-cn"]
|
||||
assert pconfig.api_key_env_vars == ("MINIMAX_CN_API_KEY",)
|
||||
|
|
@ -104,6 +116,7 @@ class TestProviderRegistry:
|
|||
assert PROVIDER_REGISTRY["copilot-acp"].inference_base_url == "acp://copilot"
|
||||
assert PROVIDER_REGISTRY["zai"].inference_base_url == "https://api.z.ai/api/paas/v4"
|
||||
assert PROVIDER_REGISTRY["kimi-coding"].inference_base_url == "https://api.moonshot.ai/v1"
|
||||
assert PROVIDER_REGISTRY["stepfun"].inference_base_url == STEPFUN_STEP_PLAN_INTL_BASE_URL
|
||||
assert PROVIDER_REGISTRY["minimax"].inference_base_url == "https://api.minimax.io/anthropic"
|
||||
assert PROVIDER_REGISTRY["minimax-cn"].inference_base_url == "https://api.minimaxi.com/anthropic"
|
||||
assert PROVIDER_REGISTRY["ai-gateway"].inference_base_url == "https://ai-gateway.vercel.sh/v1"
|
||||
|
|
@ -126,7 +139,8 @@ PROVIDER_ENV_VARS = (
|
|||
"OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN",
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"GLM_API_KEY", "ZAI_API_KEY", "Z_AI_API_KEY",
|
||||
"KIMI_API_KEY", "KIMI_BASE_URL", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY",
|
||||
"KIMI_API_KEY", "KIMI_BASE_URL", "STEPFUN_API_KEY", "STEPFUN_BASE_URL",
|
||||
"MINIMAX_API_KEY", "MINIMAX_CN_API_KEY",
|
||||
"AI_GATEWAY_API_KEY", "AI_GATEWAY_BASE_URL",
|
||||
"KILOCODE_API_KEY", "KILOCODE_BASE_URL",
|
||||
"DASHSCOPE_API_KEY", "OPENCODE_ZEN_API_KEY", "OPENCODE_GO_API_KEY",
|
||||
|
|
@ -152,6 +166,9 @@ class TestResolveProvider:
|
|||
def test_explicit_kimi_coding(self):
|
||||
assert resolve_provider("kimi-coding") == "kimi-coding"
|
||||
|
||||
def test_explicit_stepfun(self):
|
||||
assert resolve_provider("stepfun") == "stepfun"
|
||||
|
||||
def test_explicit_minimax(self):
|
||||
assert resolve_provider("minimax") == "minimax"
|
||||
|
||||
|
|
@ -176,6 +193,9 @@ class TestResolveProvider:
|
|||
def test_alias_moonshot(self):
|
||||
assert resolve_provider("moonshot") == "kimi-coding"
|
||||
|
||||
def test_alias_step(self):
|
||||
assert resolve_provider("step") == "stepfun"
|
||||
|
||||
def test_alias_minimax_underscore(self):
|
||||
assert resolve_provider("minimax_cn") == "minimax-cn"
|
||||
|
||||
|
|
@ -244,6 +264,10 @@ class TestResolveProvider:
|
|||
monkeypatch.setenv("KIMI_API_KEY", "test-kimi-key")
|
||||
assert resolve_provider("auto") == "kimi-coding"
|
||||
|
||||
def test_auto_detects_stepfun_key(self, monkeypatch):
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "test-stepfun-key")
|
||||
assert resolve_provider("auto") == "stepfun"
|
||||
|
||||
def test_auto_detects_minimax_key(self, monkeypatch):
|
||||
monkeypatch.setenv("MINIMAX_API_KEY", "test-mm-key")
|
||||
assert resolve_provider("auto") == "minimax"
|
||||
|
|
@ -308,6 +332,13 @@ class TestApiKeyProviderStatus:
|
|||
status = get_api_key_provider_status("kimi-coding")
|
||||
assert status["base_url"] == "https://custom.kimi.example/v1"
|
||||
|
||||
def test_stepfun_status_uses_configured_base_url(self, monkeypatch):
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "stepfun-key")
|
||||
monkeypatch.setenv("STEPFUN_BASE_URL", STEPFUN_STEP_PLAN_CN_BASE_URL)
|
||||
status = get_api_key_provider_status("stepfun")
|
||||
assert status["configured"] is True
|
||||
assert status["base_url"] == STEPFUN_STEP_PLAN_CN_BASE_URL
|
||||
|
||||
def test_copilot_status_uses_gh_cli_token(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.copilot_auth._try_gh_cli_token", lambda: "gho_gh_cli_token")
|
||||
status = get_api_key_provider_status("copilot")
|
||||
|
|
@ -425,6 +456,19 @@ class TestResolveApiKeyProviderCredentials:
|
|||
assert creds["api_key"] == "kimi-secret-key"
|
||||
assert creds["base_url"] == "https://api.moonshot.ai/v1"
|
||||
|
||||
def test_resolve_stepfun_with_key(self, monkeypatch):
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "stepfun-secret-key")
|
||||
creds = resolve_api_key_provider_credentials("stepfun")
|
||||
assert creds["provider"] == "stepfun"
|
||||
assert creds["api_key"] == "stepfun-secret-key"
|
||||
assert creds["base_url"] == STEPFUN_STEP_PLAN_INTL_BASE_URL
|
||||
|
||||
def test_resolve_stepfun_custom_base_url(self, monkeypatch):
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "stepfun-secret-key")
|
||||
monkeypatch.setenv("STEPFUN_BASE_URL", STEPFUN_STEP_PLAN_CN_BASE_URL)
|
||||
creds = resolve_api_key_provider_credentials("stepfun")
|
||||
assert creds["base_url"] == STEPFUN_STEP_PLAN_CN_BASE_URL
|
||||
|
||||
def test_resolve_minimax_with_key(self, monkeypatch):
|
||||
monkeypatch.setenv("MINIMAX_API_KEY", "mm-secret-key")
|
||||
creds = resolve_api_key_provider_credentials("minimax")
|
||||
|
|
@ -515,6 +559,16 @@ class TestRuntimeProviderResolution:
|
|||
assert result["api_mode"] == "chat_completions"
|
||||
assert result["api_key"] == "kimi-key"
|
||||
|
||||
def test_runtime_stepfun(self, monkeypatch):
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "stepfun-key")
|
||||
monkeypatch.setenv("STEPFUN_BASE_URL", STEPFUN_STEP_PLAN_CN_BASE_URL)
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
result = resolve_runtime_provider(requested="stepfun")
|
||||
assert result["provider"] == "stepfun"
|
||||
assert result["api_mode"] == "chat_completions"
|
||||
assert result["api_key"] == "stepfun-key"
|
||||
assert result["base_url"] == STEPFUN_STEP_PLAN_CN_BASE_URL
|
||||
|
||||
def test_runtime_minimax(self, monkeypatch):
|
||||
monkeypatch.setenv("MINIMAX_API_KEY", "mm-key")
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
|
|
|||
90
tests/hermes_cli/test_at_context_completion_filter.py
Normal file
90
tests/hermes_cli/test_at_context_completion_filter.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Regression test: `@folder:` completion must only surface directories and
|
||||
`@file:` must only surface regular files.
|
||||
|
||||
Reported during TUI v2 blitz testing: typing `@folder:` showed .dockerignore,
|
||||
.env, .gitignore, etc. alongside the actual directories because the path-
|
||||
completion branch yielded every entry regardless of the explicit prefix, and
|
||||
auto-switched the completion kind based on `is_dir`. That defeated the user's
|
||||
explicit choice and rendered the `@folder:` / `@file:` prefixes useless for
|
||||
filtering.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from hermes_cli.commands import SlashCommandCompleter
|
||||
|
||||
|
||||
def _run(tmp_path: Path, word: str) -> list[tuple[str, str]]:
|
||||
(tmp_path / "readme.md").write_text("x")
|
||||
(tmp_path / ".env").write_text("x")
|
||||
(tmp_path / "src").mkdir()
|
||||
(tmp_path / "docs").mkdir()
|
||||
|
||||
completer = SlashCommandCompleter.__new__(SlashCommandCompleter)
|
||||
completions: Iterable = completer._context_completions(word)
|
||||
|
||||
return [(c.text, c.display_meta) for c in completions if c.text.startswith(("@file:", "@folder:"))]
|
||||
|
||||
|
||||
def test_at_folder_only_yields_directories(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
texts = [t for t, _ in _run(tmp_path, "@folder:")]
|
||||
|
||||
assert all(t.startswith("@folder:") for t in texts), texts
|
||||
assert any(t == "@folder:src/" for t in texts)
|
||||
assert any(t == "@folder:docs/" for t in texts)
|
||||
assert not any(t == "@folder:readme.md" for t in texts)
|
||||
assert not any(t == "@folder:.env" for t in texts)
|
||||
|
||||
|
||||
def test_at_file_only_yields_files(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
texts = [t for t, _ in _run(tmp_path, "@file:")]
|
||||
|
||||
assert all(t.startswith("@file:") for t in texts), texts
|
||||
assert any(t == "@file:readme.md" for t in texts)
|
||||
assert any(t == "@file:.env" for t in texts)
|
||||
assert not any(t == "@file:src/" for t in texts)
|
||||
assert not any(t == "@file:docs/" for t in texts)
|
||||
|
||||
|
||||
def test_at_folder_preserves_prefix_on_empty_match(tmp_path, monkeypatch):
|
||||
"""User typed `@folder:` (no partial) — completion text must keep the
|
||||
`@folder:` prefix even though the previous implementation auto-rewrote
|
||||
it to `@file:` for non-dir entries.
|
||||
"""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
texts = [t for t, _ in _run(tmp_path, "@folder:")]
|
||||
|
||||
assert texts, "expected at least one directory completion"
|
||||
for t in texts:
|
||||
assert t.startswith("@folder:"), f"prefix leaked: {t}"
|
||||
|
||||
|
||||
def test_at_folder_bare_without_colon_lists_directories(tmp_path, monkeypatch):
|
||||
"""Typing `@folder` alone (no colon yet) should surface directories so
|
||||
users don't need to first accept the static `@folder:` hint before
|
||||
seeing what they're picking from.
|
||||
"""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
texts = [t for t, _ in _run(tmp_path, "@folder")]
|
||||
|
||||
assert any(t == "@folder:src/" for t in texts), texts
|
||||
assert any(t == "@folder:docs/" for t in texts), texts
|
||||
assert not any(t == "@folder:readme.md" for t in texts)
|
||||
|
||||
|
||||
def test_at_file_bare_without_colon_lists_files(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
texts = [t for t, _ in _run(tmp_path, "@file")]
|
||||
|
||||
assert any(t == "@file:readme.md" for t in texts), texts
|
||||
assert not any(t == "@file:src/" for t in texts)
|
||||
|
|
@ -1011,3 +1011,466 @@ def test_seed_from_singletons_respects_codex_suppression(tmp_path, monkeypatch):
|
|||
# Verify the auth store was NOT modified (no auto-import happened)
|
||||
after = json.loads((hermes_home / "auth.json").read_text())
|
||||
assert "openai-codex" not in after.get("providers", {})
|
||||
|
||||
|
||||
def test_auth_remove_env_seeded_suppresses_shell_exported_var(tmp_path, monkeypatch, capsys):
|
||||
"""`hermes auth remove xai 1` must stick even when the env var is exported
|
||||
by the shell (not written into ~/.hermes/.env). Before PR for #13371 the
|
||||
removal silently restored on next load_pool() because _seed_from_env()
|
||||
re-read os.environ. Now env:<VAR> is suppressed in auth.json.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
# Simulate shell export (NOT written to .env)
|
||||
monkeypatch.setenv("XAI_API_KEY", "sk-xai-shell-export")
|
||||
(hermes_home / ".env").write_text("")
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"credential_pool": {
|
||||
"xai": [{
|
||||
"id": "env-1",
|
||||
"label": "XAI_API_KEY",
|
||||
"auth_type": "api_key",
|
||||
"priority": 0,
|
||||
"source": "env:XAI_API_KEY",
|
||||
"access_token": "sk-xai-shell-export",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth_commands import auth_remove_command
|
||||
auth_remove_command(SimpleNamespace(provider="xai", target="1"))
|
||||
|
||||
# Suppression marker written
|
||||
after = json.loads((hermes_home / "auth.json").read_text())
|
||||
assert "env:XAI_API_KEY" in after.get("suppressed_sources", {}).get("xai", [])
|
||||
|
||||
# Diagnostic printed pointing at the shell
|
||||
out = capsys.readouterr().out
|
||||
assert "still set in your shell environment" in out
|
||||
assert "Cleared XAI_API_KEY from .env" not in out # wasn't in .env
|
||||
|
||||
# Fresh simulation: shell re-exports, reload pool
|
||||
monkeypatch.setenv("XAI_API_KEY", "sk-xai-shell-export")
|
||||
from agent.credential_pool import load_pool
|
||||
pool = load_pool("xai")
|
||||
assert not pool.has_credentials(), "pool must stay empty — env:XAI_API_KEY suppressed"
|
||||
|
||||
|
||||
def test_auth_remove_env_seeded_dotenv_only_no_shell_hint(tmp_path, monkeypatch, capsys):
|
||||
"""When the env var lives only in ~/.hermes/.env (not the shell), the
|
||||
shell-hint should NOT be printed — avoid scaring the user about a
|
||||
non-existent shell export.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
# Key ONLY in .env, shell must not have it
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
||||
(hermes_home / ".env").write_text("DEEPSEEK_API_KEY=sk-ds-only\n")
|
||||
# Mimic load_env() populating os.environ
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-ds-only")
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"credential_pool": {
|
||||
"deepseek": [{
|
||||
"id": "env-1",
|
||||
"label": "DEEPSEEK_API_KEY",
|
||||
"auth_type": "api_key",
|
||||
"priority": 0,
|
||||
"source": "env:DEEPSEEK_API_KEY",
|
||||
"access_token": "sk-ds-only",
|
||||
}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth_commands import auth_remove_command
|
||||
auth_remove_command(SimpleNamespace(provider="deepseek", target="1"))
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "Cleared DEEPSEEK_API_KEY from .env" in out
|
||||
assert "still set in your shell environment" not in out
|
||||
assert (hermes_home / ".env").read_text().strip() == ""
|
||||
|
||||
|
||||
def test_auth_add_clears_env_suppression_for_provider(tmp_path, monkeypatch):
|
||||
"""Re-adding a credential via `hermes auth add <provider>` clears any
|
||||
env:<VAR> suppression marker — strong signal the user wants auth back.
|
||||
Matches the Codex device_code re-link behaviour.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"xai": ["env:XAI_API_KEY"]},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth import is_source_suppressed
|
||||
from hermes_cli.auth_commands import auth_add_command
|
||||
|
||||
assert is_source_suppressed("xai", "env:XAI_API_KEY") is True
|
||||
auth_add_command(SimpleNamespace(
|
||||
provider="xai", auth_type="api_key",
|
||||
api_key="sk-xai-manual", label="manual",
|
||||
))
|
||||
assert is_source_suppressed("xai", "env:XAI_API_KEY") is False
|
||||
|
||||
|
||||
def test_seed_from_env_respects_env_suppression(tmp_path, monkeypatch):
|
||||
"""_seed_from_env() must skip env:<VAR> sources that the user suppressed
|
||||
via `hermes auth remove`. This is the gate that prevents shell-exported
|
||||
keys from resurrecting removed credentials.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.setenv("XAI_API_KEY", "sk-xai-shell-export")
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"xai": ["env:XAI_API_KEY"]},
|
||||
}))
|
||||
|
||||
from agent.credential_pool import _seed_from_env
|
||||
|
||||
entries = []
|
||||
changed, active = _seed_from_env("xai", entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert active == set()
|
||||
|
||||
|
||||
def test_seed_from_env_respects_openrouter_suppression(tmp_path, monkeypatch):
|
||||
"""OpenRouter is the special-case branch in _seed_from_env; verify it
|
||||
honours suppression too.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-shell-export")
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"openrouter": ["env:OPENROUTER_API_KEY"]},
|
||||
}))
|
||||
|
||||
from agent.credential_pool import _seed_from_env
|
||||
|
||||
entries = []
|
||||
changed, active = _seed_from_env("openrouter", entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert active == set()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Unified credential-source stickiness — every source Hermes reads from has a
|
||||
# registered RemovalStep in agent.credential_sources, and every seeding path
|
||||
# gates on is_source_suppressed. Below: one test per source proving remove
|
||||
# sticks across a fresh load_pool() call.
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def test_seed_from_singletons_respects_nous_suppression(tmp_path, monkeypatch):
|
||||
"""nous device_code must not re-seed from auth.json when suppressed."""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {"nous": {"access_token": "tok", "refresh_token": "r", "expires_at": 9999999999}},
|
||||
"suppressed_sources": {"nous": ["device_code"]},
|
||||
}))
|
||||
|
||||
from agent.credential_pool import _seed_from_singletons
|
||||
entries = []
|
||||
changed, active = _seed_from_singletons("nous", entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert active == set()
|
||||
|
||||
|
||||
def test_seed_from_singletons_respects_copilot_suppression(tmp_path, monkeypatch):
|
||||
"""copilot gh_cli must not re-seed when suppressed."""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"copilot": ["gh_cli"]},
|
||||
}))
|
||||
|
||||
# Stub resolve_copilot_token to return a live token
|
||||
import hermes_cli.copilot_auth as ca
|
||||
monkeypatch.setattr(ca, "resolve_copilot_token", lambda: ("ghp_fake", "gh auth token"))
|
||||
|
||||
from agent.credential_pool import _seed_from_singletons
|
||||
entries = []
|
||||
changed, active = _seed_from_singletons("copilot", entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert active == set()
|
||||
|
||||
|
||||
def test_seed_from_singletons_respects_qwen_suppression(tmp_path, monkeypatch):
|
||||
"""qwen-oauth qwen-cli must not re-seed from ~/.qwen/oauth_creds.json when suppressed."""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"qwen-oauth": ["qwen-cli"]},
|
||||
}))
|
||||
|
||||
import hermes_cli.auth as ha
|
||||
monkeypatch.setattr(ha, "resolve_qwen_runtime_credentials", lambda **kw: {
|
||||
"api_key": "tok", "source": "qwen-cli", "base_url": "https://q",
|
||||
})
|
||||
|
||||
from agent.credential_pool import _seed_from_singletons
|
||||
entries = []
|
||||
changed, active = _seed_from_singletons("qwen-oauth", entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert active == set()
|
||||
|
||||
|
||||
def test_seed_from_singletons_respects_hermes_pkce_suppression(tmp_path, monkeypatch):
|
||||
"""anthropic hermes_pkce must not re-seed from ~/.hermes/.anthropic_oauth.json when suppressed."""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
import yaml
|
||||
(hermes_home / "config.yaml").write_text(yaml.dump({"model": {"provider": "anthropic", "model": "claude"}}))
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {"anthropic": ["hermes_pkce"]},
|
||||
}))
|
||||
|
||||
# Stub the readers so only hermes_pkce is "available"; claude_code returns None
|
||||
import agent.anthropic_adapter as aa
|
||||
monkeypatch.setattr(aa, "read_hermes_oauth_credentials", lambda: {
|
||||
"accessToken": "tok", "refreshToken": "r", "expiresAt": 9999999999000,
|
||||
})
|
||||
monkeypatch.setattr(aa, "read_claude_code_credentials", lambda: None)
|
||||
|
||||
from agent.credential_pool import _seed_from_singletons
|
||||
entries = []
|
||||
changed, active = _seed_from_singletons("anthropic", entries)
|
||||
# hermes_pkce suppressed, claude_code returns None → nothing should be seeded
|
||||
assert entries == []
|
||||
assert "hermes_pkce" not in active
|
||||
|
||||
|
||||
def test_seed_custom_pool_respects_config_suppression(tmp_path, monkeypatch):
|
||||
"""Custom provider config:<name> source must not re-seed when suppressed."""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
import yaml
|
||||
(hermes_home / "config.yaml").write_text(yaml.dump({
|
||||
"model": {},
|
||||
"custom_providers": [
|
||||
{"name": "my", "base_url": "https://c.example.com", "api_key": "sk-custom"},
|
||||
],
|
||||
}))
|
||||
|
||||
from agent.credential_pool import _seed_custom_pool, get_custom_provider_pool_key
|
||||
pool_key = get_custom_provider_pool_key("https://c.example.com")
|
||||
|
||||
(hermes_home / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {pool_key: ["config:my"]},
|
||||
}))
|
||||
|
||||
entries = []
|
||||
changed, active = _seed_custom_pool(pool_key, entries)
|
||||
assert changed is False
|
||||
assert entries == []
|
||||
assert "config:my" not in active
|
||||
|
||||
|
||||
def test_credential_sources_registry_has_expected_steps():
|
||||
"""Sanity check — the registry contains the expected RemovalSteps.
|
||||
|
||||
Guards against accidentally dropping a step during future refactors.
|
||||
If you add a new credential source, add it to the expected set below.
|
||||
"""
|
||||
from agent.credential_sources import _REGISTRY
|
||||
|
||||
descriptions = {step.description for step in _REGISTRY}
|
||||
expected = {
|
||||
"gh auth token / COPILOT_GITHUB_TOKEN / GH_TOKEN",
|
||||
"Any env-seeded credential (XAI_API_KEY, DEEPSEEK_API_KEY, etc.)",
|
||||
"~/.claude/.credentials.json",
|
||||
"~/.hermes/.anthropic_oauth.json",
|
||||
"auth.json providers.nous",
|
||||
"auth.json providers.openai-codex + ~/.codex/auth.json",
|
||||
"~/.qwen/oauth_creds.json",
|
||||
"Custom provider config.yaml api_key field",
|
||||
}
|
||||
assert descriptions == expected, f"Registry mismatch. Got: {descriptions}"
|
||||
|
||||
|
||||
def test_credential_sources_find_step_returns_none_for_manual():
|
||||
"""Manual entries have nothing external to clean up — no step registered."""
|
||||
from agent.credential_sources import find_removal_step
|
||||
assert find_removal_step("openrouter", "manual") is None
|
||||
assert find_removal_step("xai", "manual") is None
|
||||
|
||||
|
||||
def test_credential_sources_find_step_copilot_before_generic_env(tmp_path, monkeypatch):
|
||||
"""copilot env:GH_TOKEN must dispatch to the copilot step, not the
|
||||
generic env-var step. The copilot step handles the duplicate-source
|
||||
problem (same token seeded as both gh_cli and env:<VAR>); the generic
|
||||
env step would only suppress one of the variants.
|
||||
"""
|
||||
from agent.credential_sources import find_removal_step
|
||||
|
||||
step = find_removal_step("copilot", "env:GH_TOKEN")
|
||||
assert step is not None
|
||||
assert "copilot" in step.description.lower() or "gh" in step.description.lower()
|
||||
|
||||
# Generic step still matches any other provider's env var
|
||||
step = find_removal_step("xai", "env:XAI_API_KEY")
|
||||
assert step is not None
|
||||
assert "env-seeded" in step.description.lower()
|
||||
|
||||
|
||||
def test_auth_remove_copilot_suppresses_all_variants(tmp_path, monkeypatch):
|
||||
"""Removing any copilot source must suppress gh_cli + all env:* variants
|
||||
so the duplicate-seed paths don't resurrect the credential.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"credential_pool": {
|
||||
"copilot": [{
|
||||
"id": "c1",
|
||||
"label": "gh auth token",
|
||||
"auth_type": "api_key",
|
||||
"priority": 0,
|
||||
"source": "gh_cli",
|
||||
"access_token": "ghp_fake",
|
||||
}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth import is_source_suppressed
|
||||
from hermes_cli.auth_commands import auth_remove_command
|
||||
|
||||
auth_remove_command(SimpleNamespace(provider="copilot", target="1"))
|
||||
|
||||
assert is_source_suppressed("copilot", "gh_cli")
|
||||
assert is_source_suppressed("copilot", "env:COPILOT_GITHUB_TOKEN")
|
||||
assert is_source_suppressed("copilot", "env:GH_TOKEN")
|
||||
assert is_source_suppressed("copilot", "env:GITHUB_TOKEN")
|
||||
|
||||
|
||||
def test_auth_add_clears_all_suppressions_including_non_env(tmp_path, monkeypatch):
|
||||
"""Re-adding a credential via `hermes auth add <provider>` clears ALL
|
||||
suppression markers for the provider, not just env:*. This matches
|
||||
the single "re-engage" semantic — the user wants auth back, period.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
"suppressed_sources": {
|
||||
"copilot": ["gh_cli", "env:GH_TOKEN", "env:COPILOT_GITHUB_TOKEN"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth import is_source_suppressed
|
||||
from hermes_cli.auth_commands import auth_add_command
|
||||
|
||||
auth_add_command(SimpleNamespace(
|
||||
provider="copilot", auth_type="api_key",
|
||||
api_key="ghp-manual", label="m",
|
||||
))
|
||||
|
||||
assert not is_source_suppressed("copilot", "gh_cli")
|
||||
assert not is_source_suppressed("copilot", "env:GH_TOKEN")
|
||||
assert not is_source_suppressed("copilot", "env:COPILOT_GITHUB_TOKEN")
|
||||
|
||||
|
||||
def test_auth_remove_codex_manual_device_code_suppresses_canonical(tmp_path, monkeypatch):
|
||||
"""Removing a manual:device_code entry (from `hermes auth add openai-codex`)
|
||||
must suppress the canonical ``device_code`` key, not ``manual:device_code``.
|
||||
The re-seed gate in _seed_from_singletons checks ``device_code``.
|
||||
"""
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
_write_auth_store(
|
||||
tmp_path,
|
||||
{
|
||||
"version": 1,
|
||||
"providers": {"openai-codex": {"tokens": {"access_token": "t", "refresh_token": "r"}}},
|
||||
"credential_pool": {
|
||||
"openai-codex": [{
|
||||
"id": "cdx",
|
||||
"label": "manual-codex",
|
||||
"auth_type": "oauth",
|
||||
"priority": 0,
|
||||
"source": "manual:device_code",
|
||||
"access_token": "t",
|
||||
}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
from types import SimpleNamespace
|
||||
from hermes_cli.auth import is_source_suppressed
|
||||
from hermes_cli.auth_commands import auth_remove_command
|
||||
|
||||
auth_remove_command(SimpleNamespace(provider="openai-codex", target="1"))
|
||||
assert is_source_suppressed("openai-codex", "device_code")
|
||||
|
|
|
|||
|
|
@ -376,7 +376,6 @@ class TestLoginNousSkipKeepsCurrent:
|
|||
lambda *a, **kw: prompt_returns,
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda p: {})
|
||||
monkeypatch.setattr(models_mod, "filter_nous_free_models", lambda ids, p: ids)
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod, "partition_nous_models_by_tier",
|
||||
|
|
|
|||
|
|
@ -1208,3 +1208,119 @@ class TestDiscordSkillCommandsByCategory:
|
|||
assert "axolotl" in names
|
||||
assert "vllm" in names
|
||||
assert len(uncategorized) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin slash command integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPluginCommandEnumeration:
|
||||
"""Plugin commands registered via ctx.register_command() must be surfaced
|
||||
by every gateway enumerator (Telegram menu, Slack subcommand map, etc.).
|
||||
"""
|
||||
|
||||
def _patch_plugin_commands(self, monkeypatch, commands):
|
||||
"""Monkeypatch hermes_cli.plugins.get_plugin_commands() to a fixed dict."""
|
||||
from hermes_cli import plugins as _plugins_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
_plugins_mod, "get_plugin_commands", lambda: dict(commands)
|
||||
)
|
||||
|
||||
def test_plugin_command_appears_in_telegram_menu(self, monkeypatch):
|
||||
"""/metricas registered by a plugin must appear in Telegram BotCommand menu."""
|
||||
self._patch_plugin_commands(monkeypatch, {
|
||||
"metricas": {
|
||||
"handler": lambda _a: "ok",
|
||||
"description": "Metrics dashboard",
|
||||
"args_hint": "dias:7",
|
||||
"plugin": "metrics-plugin",
|
||||
}
|
||||
})
|
||||
names = {name for name, _desc in telegram_bot_commands()}
|
||||
assert "metricas" in names
|
||||
|
||||
def test_plugin_command_appears_in_slack_subcommand_map(self, monkeypatch):
|
||||
"""/hermes metricas must route through the Slack subcommand map."""
|
||||
self._patch_plugin_commands(monkeypatch, {
|
||||
"metricas": {
|
||||
"handler": lambda _a: "ok",
|
||||
"description": "Metrics",
|
||||
"args_hint": "",
|
||||
"plugin": "metrics-plugin",
|
||||
}
|
||||
})
|
||||
mapping = slack_subcommand_map()
|
||||
assert mapping.get("metricas") == "/metricas"
|
||||
|
||||
def test_plugin_command_does_not_shadow_builtin_in_slack(self, monkeypatch):
|
||||
"""If a plugin registers a name that collides with a built-in, the built-in mapping wins."""
|
||||
self._patch_plugin_commands(monkeypatch, {
|
||||
"status": {
|
||||
"handler": lambda _a: "plugin-status",
|
||||
"description": "Plugin status",
|
||||
"args_hint": "",
|
||||
"plugin": "shadow-plugin",
|
||||
}
|
||||
})
|
||||
mapping = slack_subcommand_map()
|
||||
# Built-in /status must still be present and not overwritten.
|
||||
assert mapping.get("status") == "/status"
|
||||
|
||||
def test_plugin_command_with_hyphens_sanitized_for_telegram(self, monkeypatch):
|
||||
"""Plugin names containing hyphens must be underscore-normalized for Telegram."""
|
||||
self._patch_plugin_commands(monkeypatch, {
|
||||
"my-plugin-cmd": {
|
||||
"handler": lambda _a: "ok",
|
||||
"description": "desc",
|
||||
"args_hint": "",
|
||||
"plugin": "p",
|
||||
}
|
||||
})
|
||||
names = {name for name, _desc in telegram_bot_commands()}
|
||||
assert "my_plugin_cmd" in names
|
||||
assert "my-plugin-cmd" not in names
|
||||
|
||||
def test_is_gateway_known_command_recognizes_plugin_commands(self, monkeypatch):
|
||||
"""is_gateway_known_command() must return True for plugin commands."""
|
||||
from hermes_cli.commands import is_gateway_known_command
|
||||
|
||||
self._patch_plugin_commands(monkeypatch, {
|
||||
"metricas": {
|
||||
"handler": lambda _a: "ok",
|
||||
"description": "Metrics",
|
||||
"args_hint": "",
|
||||
"plugin": "p",
|
||||
}
|
||||
})
|
||||
assert is_gateway_known_command("metricas") is True
|
||||
assert is_gateway_known_command("definitely-not-registered") is False
|
||||
|
||||
def test_is_gateway_known_command_still_recognizes_builtins(self, monkeypatch):
|
||||
"""Built-in commands must remain known even when plugin discovery fails."""
|
||||
from hermes_cli import plugins as _plugins_mod
|
||||
from hermes_cli.commands import is_gateway_known_command
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("plugin system down")
|
||||
|
||||
monkeypatch.setattr(_plugins_mod, "get_plugin_commands", _boom)
|
||||
|
||||
assert is_gateway_known_command("status") is True
|
||||
assert is_gateway_known_command(None) is False
|
||||
assert is_gateway_known_command("") is False
|
||||
|
||||
def test_plugin_enumerator_handles_missing_plugin_manager(self, monkeypatch):
|
||||
"""Enumerators must never raise when plugin discovery raises."""
|
||||
from hermes_cli import plugins as _plugins_mod
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("plugin system down")
|
||||
|
||||
monkeypatch.setattr(_plugins_mod, "get_plugin_commands", _boom)
|
||||
|
||||
# Both calls should succeed and just return the built-in set.
|
||||
tg_names = {name for name, _desc in telegram_bot_commands()}
|
||||
slack_names = set(slack_subcommand_map())
|
||||
assert "status" in tg_names
|
||||
assert "status" in slack_names
|
||||
|
|
|
|||
36
tests/hermes_cli/test_config_drift.py
Normal file
36
tests/hermes_cli/test_config_drift.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""Regression tests for removed dead config keys.
|
||||
|
||||
This file guards against accidental re-introduction of config keys that were
|
||||
documented or declared at some point but never actually wired up to read code.
|
||||
Future dead-config regressions can accumulate here.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
|
||||
def test_delegation_default_toolsets_removed_from_cli_config():
|
||||
"""delegation.default_toolsets was dead config — never read by
|
||||
_load_config() or anywhere else. Removed.
|
||||
|
||||
Guards against accidental re-introduction in cli.py's CLI_CONFIG default
|
||||
dict. If this test fails, someone re-added the key without wiring it up
|
||||
to _load_config() in tools/delegate_tool.py.
|
||||
|
||||
We inspect the source of load_cli_config() instead of asserting on the
|
||||
runtime CLI_CONFIG dict because CLI_CONFIG is populated by deep-merging
|
||||
the user's ~/.hermes/config.yaml over the defaults (cli.py:359-366).
|
||||
A contributor who still has the legacy key set in their own config
|
||||
would cause a false failure, and HERMES_HOME patching via conftest
|
||||
doesn't help because cli._hermes_home is frozen at module import time
|
||||
(cli.py:76) — before any autouse fixture can fire. Source inspection
|
||||
sidesteps all of that: it tests the defaults literal directly.
|
||||
"""
|
||||
from cli import load_cli_config
|
||||
|
||||
source = inspect.getsource(load_cli_config)
|
||||
assert '"default_toolsets"' not in source, (
|
||||
"delegation.default_toolsets was removed because it was never read. "
|
||||
"Do not re-add it to cli.py's CLI_CONFIG default dict; "
|
||||
"use tools/delegate_tool.py's DEFAULT_TOOLSETS module constant or "
|
||||
"wire a new config key through _load_config()."
|
||||
)
|
||||
|
|
@ -137,50 +137,105 @@ class TestUploadToPastebin:
|
|||
# Log reading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestReadFullLog:
|
||||
"""Test _read_full_log for standalone log uploads."""
|
||||
class TestCaptureLogSnapshot:
|
||||
"""Test _capture_log_snapshot for log reading and truncation."""
|
||||
|
||||
def test_reads_small_file(self, hermes_home):
|
||||
from hermes_cli.debug import _read_full_log
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
content = _read_full_log("agent")
|
||||
assert content is not None
|
||||
assert "session started" in content
|
||||
snap = _capture_log_snapshot("agent", tail_lines=10)
|
||||
assert snap.full_text is not None
|
||||
assert "session started" in snap.full_text
|
||||
assert "session started" in snap.tail_text
|
||||
|
||||
def test_returns_none_for_missing(self, tmp_path, monkeypatch):
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from hermes_cli.debug import _read_full_log
|
||||
assert _read_full_log("agent") is None
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
snap = _capture_log_snapshot("agent", tail_lines=10)
|
||||
assert snap.full_text is None
|
||||
assert snap.tail_text == "(file not found)"
|
||||
|
||||
def test_returns_none_for_empty(self, hermes_home):
|
||||
# Truncate agent.log to empty
|
||||
def test_empty_primary_reports_file_empty(self, hermes_home):
|
||||
"""Empty primary (no .1 fallback) surfaces as '(file empty)', not missing."""
|
||||
(hermes_home / "logs" / "agent.log").write_text("")
|
||||
|
||||
from hermes_cli.debug import _read_full_log
|
||||
assert _read_full_log("agent") is None
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
snap = _capture_log_snapshot("agent", tail_lines=10)
|
||||
assert snap.full_text is None
|
||||
assert snap.tail_text == "(file empty)"
|
||||
|
||||
def test_race_truncate_after_resolve_reports_empty(self, hermes_home, monkeypatch):
|
||||
"""If the log is truncated between resolve and stat, say 'empty', not 'missing'."""
|
||||
log_path = hermes_home / "logs" / "agent.log"
|
||||
from hermes_cli import debug
|
||||
|
||||
monkeypatch.setattr(debug, "_resolve_log_path", lambda _name: log_path)
|
||||
log_path.write_text("")
|
||||
|
||||
snap = debug._capture_log_snapshot("agent", tail_lines=10)
|
||||
assert snap.path == log_path
|
||||
assert snap.full_text is None
|
||||
assert snap.tail_text == "(file empty)"
|
||||
|
||||
def test_truncates_large_file(self, hermes_home):
|
||||
"""Files larger than max_bytes get tail-truncated."""
|
||||
from hermes_cli.debug import _read_full_log
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
# Write a file larger than 1KB
|
||||
big_content = "x" * 100 + "\n"
|
||||
(hermes_home / "logs" / "agent.log").write_text(big_content * 200)
|
||||
|
||||
content = _read_full_log("agent", max_bytes=1024)
|
||||
assert content is not None
|
||||
assert "truncated" in content
|
||||
snap = _capture_log_snapshot("agent", tail_lines=10, max_bytes=1024)
|
||||
assert snap.full_text is not None
|
||||
assert "truncated" in snap.full_text
|
||||
|
||||
def test_keeps_first_line_when_truncation_on_boundary(self, hermes_home):
|
||||
"""When truncation lands on a line boundary, keep the first full line."""
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
# File must exceed the initial chunk_size (8192) used by the
|
||||
# backward-reading loop so the truncation path actually fires.
|
||||
line = "A" * 99 + "\n" # 100 bytes per line
|
||||
num_lines = 200 # 20000 bytes
|
||||
(hermes_home / "logs" / "agent.log").write_text(line * num_lines)
|
||||
|
||||
# max_bytes = 1000 = 100 * 10 → cut at byte 20000 - 1000 = 19000,
|
||||
# and byte 19000 - 1 is '\n'. Boundary hit → keep all 10 lines.
|
||||
snap = _capture_log_snapshot("agent", tail_lines=5, max_bytes=1000)
|
||||
assert snap.full_text is not None
|
||||
assert "truncated" in snap.full_text
|
||||
raw = snap.full_text.split("\n", 1)[1]
|
||||
kept = [l for l in raw.strip().splitlines() if l.startswith("A")]
|
||||
assert len(kept) == 10
|
||||
|
||||
def test_drops_partial_when_truncation_mid_line(self, hermes_home):
|
||||
"""When truncation lands mid-line, drop the partial fragment."""
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
line = "A" * 99 + "\n" # 100 bytes per line
|
||||
num_lines = 200 # 20000 bytes
|
||||
(hermes_home / "logs" / "agent.log").write_text(line * num_lines)
|
||||
|
||||
# max_bytes = 950 doesn't divide evenly into 100 → mid-line cut.
|
||||
snap = _capture_log_snapshot("agent", tail_lines=5, max_bytes=950)
|
||||
assert snap.full_text is not None
|
||||
assert "truncated" in snap.full_text
|
||||
raw = snap.full_text.split("\n", 1)[1]
|
||||
kept = [l for l in raw.strip().splitlines() if l.startswith("A")]
|
||||
# 950 / 100 = 9.5 → 9 complete lines after dropping partial
|
||||
assert len(kept) == 9
|
||||
|
||||
def test_unknown_log_returns_none(self, hermes_home):
|
||||
from hermes_cli.debug import _read_full_log
|
||||
assert _read_full_log("nonexistent") is None
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
snap = _capture_log_snapshot("nonexistent", tail_lines=10)
|
||||
assert snap.full_text is None
|
||||
|
||||
def test_falls_back_to_rotated_file(self, hermes_home):
|
||||
"""When gateway.log doesn't exist, falls back to gateway.log.1."""
|
||||
from hermes_cli.debug import _read_full_log
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
logs_dir = hermes_home / "logs"
|
||||
# Remove the primary (if any) and create a .1 rotation
|
||||
|
|
@ -189,33 +244,33 @@ class TestReadFullLog:
|
|||
"2026-04-12 10:00:00 INFO gateway.run: rotated content\n"
|
||||
)
|
||||
|
||||
content = _read_full_log("gateway")
|
||||
assert content is not None
|
||||
assert "rotated content" in content
|
||||
snap = _capture_log_snapshot("gateway", tail_lines=10)
|
||||
assert snap.full_text is not None
|
||||
assert "rotated content" in snap.full_text
|
||||
|
||||
def test_prefers_primary_over_rotated(self, hermes_home):
|
||||
"""Primary log is used when it exists, even if .1 also exists."""
|
||||
from hermes_cli.debug import _read_full_log
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
logs_dir = hermes_home / "logs"
|
||||
(logs_dir / "gateway.log").write_text("primary content\n")
|
||||
(logs_dir / "gateway.log.1").write_text("rotated content\n")
|
||||
|
||||
content = _read_full_log("gateway")
|
||||
assert "primary content" in content
|
||||
assert "rotated" not in content
|
||||
snap = _capture_log_snapshot("gateway", tail_lines=10)
|
||||
assert "primary content" in snap.full_text
|
||||
assert "rotated" not in snap.full_text
|
||||
|
||||
def test_falls_back_when_primary_empty(self, hermes_home):
|
||||
"""Empty primary log falls back to .1 rotation."""
|
||||
from hermes_cli.debug import _read_full_log
|
||||
from hermes_cli.debug import _capture_log_snapshot
|
||||
|
||||
logs_dir = hermes_home / "logs"
|
||||
(logs_dir / "agent.log").write_text("")
|
||||
(logs_dir / "agent.log.1").write_text("rotated agent data\n")
|
||||
|
||||
content = _read_full_log("agent")
|
||||
assert content is not None
|
||||
assert "rotated agent data" in content
|
||||
snap = _capture_log_snapshot("agent", tail_lines=10)
|
||||
assert snap.full_text is not None
|
||||
assert "rotated agent data" in snap.full_text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -283,6 +338,44 @@ class TestCollectDebugReport:
|
|||
class TestRunDebugShare:
|
||||
"""Test the run_debug_share CLI handler."""
|
||||
|
||||
def test_share_sweeps_expired_pastes(self, hermes_home, capsys):
|
||||
"""Slash-command path should sweep old pending deletes before uploading."""
|
||||
from hermes_cli.debug import run_debug_share
|
||||
|
||||
args = MagicMock()
|
||||
args.lines = 50
|
||||
args.expire = 7
|
||||
args.local = False
|
||||
|
||||
with patch("hermes_cli.dump.run_dump"), \
|
||||
patch("hermes_cli.debug._sweep_expired_pastes", return_value=(0, 0)) as mock_sweep, \
|
||||
patch("hermes_cli.debug.upload_to_pastebin",
|
||||
return_value="https://paste.rs/test"):
|
||||
run_debug_share(args)
|
||||
|
||||
mock_sweep.assert_called_once()
|
||||
assert "Debug report uploaded" in capsys.readouterr().out
|
||||
|
||||
def test_share_survives_sweep_failure(self, hermes_home, capsys):
|
||||
"""Expired-paste cleanup is best-effort and must not block sharing."""
|
||||
from hermes_cli.debug import run_debug_share
|
||||
|
||||
args = MagicMock()
|
||||
args.lines = 50
|
||||
args.expire = 7
|
||||
args.local = False
|
||||
|
||||
with patch("hermes_cli.dump.run_dump"), \
|
||||
patch(
|
||||
"hermes_cli.debug._sweep_expired_pastes",
|
||||
side_effect=RuntimeError("offline"),
|
||||
), \
|
||||
patch("hermes_cli.debug.upload_to_pastebin",
|
||||
return_value="https://paste.rs/test"):
|
||||
run_debug_share(args)
|
||||
|
||||
assert "https://paste.rs/test" in capsys.readouterr().out
|
||||
|
||||
def test_local_flag_prints_full_logs(self, hermes_home, capsys):
|
||||
"""--local prints the report plus full log contents."""
|
||||
from hermes_cli.debug import run_debug_share
|
||||
|
|
@ -340,6 +433,55 @@ class TestRunDebugShare:
|
|||
assert "--- hermes dump ---" in gateway_paste
|
||||
assert "--- full gateway.log ---" in gateway_paste
|
||||
|
||||
def test_share_keeps_report_and_full_log_on_same_snapshot(self, hermes_home, capsys):
|
||||
"""A mid-run rotation must not make full agent.log older than the report."""
|
||||
from hermes_cli.debug import run_debug_share, collect_debug_report as real_collect_debug_report
|
||||
|
||||
logs_dir = hermes_home / "logs"
|
||||
(logs_dir / "agent.log").write_text(
|
||||
"2026-04-22 12:00:00 INFO agent: newest line\n"
|
||||
)
|
||||
(logs_dir / "agent.log.1").write_text(
|
||||
"2026-04-10 12:00:00 INFO agent: old rotated line\n"
|
||||
)
|
||||
|
||||
args = MagicMock()
|
||||
args.lines = 50
|
||||
args.expire = 7
|
||||
args.local = False
|
||||
|
||||
uploaded_content = []
|
||||
|
||||
def _mock_upload(content, expiry_days=7):
|
||||
uploaded_content.append(content)
|
||||
return f"https://paste.rs/paste{len(uploaded_content)}"
|
||||
|
||||
def _wrapped_collect_debug_report(*, log_lines=200, dump_text="", log_snapshots=None):
|
||||
report = real_collect_debug_report(
|
||||
log_lines=log_lines,
|
||||
dump_text=dump_text,
|
||||
log_snapshots=log_snapshots,
|
||||
)
|
||||
# Simulate the live log rotating after the report is built but
|
||||
# before the old implementation would have re-read agent.log for
|
||||
# standalone upload.
|
||||
(logs_dir / "agent.log").write_text("")
|
||||
(logs_dir / "agent.log.1").write_text(
|
||||
"2026-04-10 12:00:00 INFO agent: old rotated line\n"
|
||||
)
|
||||
return report
|
||||
|
||||
with patch("hermes_cli.dump.run_dump"), \
|
||||
patch("hermes_cli.debug.collect_debug_report", side_effect=_wrapped_collect_debug_report), \
|
||||
patch("hermes_cli.debug.upload_to_pastebin", side_effect=_mock_upload):
|
||||
run_debug_share(args)
|
||||
|
||||
report_paste = uploaded_content[0]
|
||||
agent_paste = uploaded_content[1]
|
||||
assert "2026-04-22 12:00:00 INFO agent: newest line" in report_paste
|
||||
assert "2026-04-22 12:00:00 INFO agent: newest line" in agent_paste
|
||||
assert "old rotated line" not in agent_paste
|
||||
|
||||
def test_share_skips_missing_logs(self, tmp_path, monkeypatch, capsys):
|
||||
"""Only uploads logs that exist."""
|
||||
home = tmp_path / ".hermes"
|
||||
|
|
|
|||
|
|
@ -33,6 +33,25 @@ def test_project_env_overrides_stale_shell_values_when_user_env_missing(tmp_path
|
|||
assert os.getenv("OPENAI_BASE_URL") == "https://project.example/v1"
|
||||
|
||||
|
||||
def test_project_env_is_sanitized_before_loading(tmp_path, monkeypatch):
|
||||
home = tmp_path / "hermes"
|
||||
project_env = tmp_path / ".env"
|
||||
project_env.write_text(
|
||||
"TELEGRAM_BOT_TOKEN=8356550917:AAGGEkzg06Hrc3Hjb3Sa1jkGVDOdU_lYy2Q"
|
||||
"ANTHROPIC_API_KEY=sk-ant-test123\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
|
||||
loaded = load_hermes_dotenv(hermes_home=home, project_env=project_env)
|
||||
|
||||
assert loaded == [project_env]
|
||||
assert os.getenv("TELEGRAM_BOT_TOKEN") == "8356550917:AAGGEkzg06Hrc3Hjb3Sa1jkGVDOdU_lYy2Q"
|
||||
assert os.getenv("ANTHROPIC_API_KEY") == "sk-ant-test123"
|
||||
|
||||
|
||||
def test_user_env_takes_precedence_over_project_env(tmp_path, monkeypatch):
|
||||
home = tmp_path / "hermes"
|
||||
home.mkdir()
|
||||
|
|
|
|||
|
|
@ -121,6 +121,12 @@ def test_systemd_status_warns_when_linger_disabled(monkeypatch, tmp_path, capsys
|
|||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
if cmd[:3] == ["systemctl", "--user", "is-active"]:
|
||||
return SimpleNamespace(returncode=0, stdout="active\n", stderr="")
|
||||
if cmd[:3] == ["systemctl", "--user", "show"]:
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="ActiveState=active\nSubState=running\nResult=success\nExecMainStatus=0\n",
|
||||
stderr="",
|
||||
)
|
||||
raise AssertionError(f"Unexpected command: {cmd}")
|
||||
|
||||
monkeypatch.setattr(gateway.subprocess, "run", fake_run)
|
||||
|
|
@ -352,3 +358,24 @@ class TestWaitForGatewayExit:
|
|||
|
||||
assert killed == 2
|
||||
assert calls == [(11, True), (22, True)]
|
||||
|
||||
|
||||
class TestStopProfileGateway:
|
||||
def test_stop_profile_gateway_keeps_pid_file_when_process_still_running(self, monkeypatch):
|
||||
calls = {"kill": 0, "remove": 0}
|
||||
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 12345)
|
||||
monkeypatch.setattr(
|
||||
gateway.os,
|
||||
"kill",
|
||||
lambda pid, sig: calls.__setitem__("kill", calls["kill"] + 1),
|
||||
)
|
||||
monkeypatch.setattr("time.sleep", lambda _: None)
|
||||
monkeypatch.setattr(
|
||||
"gateway.status.remove_pid_file",
|
||||
lambda: calls.__setitem__("remove", calls["remove"] + 1),
|
||||
)
|
||||
|
||||
assert gateway.stop_profile_gateway() is True
|
||||
assert calls["kill"] == 21
|
||||
assert calls["remove"] == 0
|
||||
|
|
|
|||
|
|
@ -77,8 +77,10 @@ class TestSystemdServiceRefresh:
|
|||
gateway_cli.systemd_restart()
|
||||
|
||||
assert unit_path.read_text(encoding="utf-8") == "new unit\n"
|
||||
assert calls[:2] == [
|
||||
assert calls[:4] == [
|
||||
["systemctl", "--user", "daemon-reload"],
|
||||
["systemctl", "--user", "show", gateway_cli.get_service_name(), "--no-pager", "--property", "ActiveState,SubState,Result,ExecMainStatus"],
|
||||
["systemctl", "--user", "reset-failed", gateway_cli.get_service_name()],
|
||||
["systemctl", "--user", "reload-or-restart", gateway_cli.get_service_name()],
|
||||
]
|
||||
|
||||
|
|
@ -474,13 +476,21 @@ class TestGatewaySystemServiceRouting:
|
|||
raise ProcessLookupError()
|
||||
monkeypatch.setattr(os, "kill", fake_kill)
|
||||
|
||||
# Simulate systemctl is-active returning "active" with a new PID
|
||||
# Simulate systemctl reset-failed/start followed by an active unit
|
||||
new_pid = [None]
|
||||
def fake_subprocess_run(cmd, **kwargs):
|
||||
if "is-active" in cmd:
|
||||
result = SimpleNamespace(stdout="active\n", returncode=0)
|
||||
new_pid[0] = 999 # new PID
|
||||
return result
|
||||
if "reset-failed" in cmd:
|
||||
calls.append(("reset-failed", cmd))
|
||||
return SimpleNamespace(stdout="", returncode=0)
|
||||
if "start" in cmd:
|
||||
calls.append(("start", cmd))
|
||||
return SimpleNamespace(stdout="", returncode=0)
|
||||
if "show" in cmd:
|
||||
new_pid[0] = 999
|
||||
return SimpleNamespace(
|
||||
stdout="ActiveState=active\nSubState=running\nResult=success\nExecMainStatus=0\n",
|
||||
returncode=0,
|
||||
)
|
||||
raise AssertionError(f"Unexpected systemctl call: {cmd}")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_subprocess_run)
|
||||
|
|
@ -494,9 +504,131 @@ class TestGatewaySystemServiceRouting:
|
|||
gateway_cli.systemd_restart()
|
||||
|
||||
assert ("self", 654) in calls
|
||||
assert any(call[0] == "reset-failed" for call in calls)
|
||||
assert any(call[0] == "start" for call in calls)
|
||||
out = capsys.readouterr().out.lower()
|
||||
assert "restarted" in out
|
||||
|
||||
def test_systemd_restart_recovers_failed_planned_restart(self, monkeypatch, capsys):
|
||||
monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False)
|
||||
monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: None)
|
||||
monkeypatch.setattr(
|
||||
"gateway.status.read_runtime_status",
|
||||
lambda: {"restart_requested": True, "gateway_state": "stopped"},
|
||||
)
|
||||
monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False)
|
||||
|
||||
calls = []
|
||||
started = {"value": False}
|
||||
|
||||
def fake_subprocess_run(cmd, **kwargs):
|
||||
if "show" in cmd:
|
||||
if not started["value"]:
|
||||
return SimpleNamespace(
|
||||
stdout=(
|
||||
"ActiveState=failed\n"
|
||||
"SubState=failed\n"
|
||||
"Result=exit-code\n"
|
||||
f"ExecMainStatus={GATEWAY_SERVICE_RESTART_EXIT_CODE}\n"
|
||||
),
|
||||
returncode=0,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
stdout="ActiveState=active\nSubState=running\nResult=success\nExecMainStatus=0\n",
|
||||
returncode=0,
|
||||
)
|
||||
if "reset-failed" in cmd:
|
||||
calls.append(("reset-failed", cmd))
|
||||
return SimpleNamespace(stdout="", returncode=0)
|
||||
if "start" in cmd:
|
||||
started["value"] = True
|
||||
calls.append(("start", cmd))
|
||||
return SimpleNamespace(stdout="", returncode=0)
|
||||
raise AssertionError(f"Unexpected command: {cmd}")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_subprocess_run)
|
||||
monkeypatch.setattr(
|
||||
"gateway.status.get_running_pid",
|
||||
lambda: 999 if started["value"] else None,
|
||||
)
|
||||
|
||||
gateway_cli.systemd_restart()
|
||||
|
||||
assert any(call[0] == "reset-failed" for call in calls)
|
||||
assert any(call[0] == "start" for call in calls)
|
||||
out = capsys.readouterr().out.lower()
|
||||
assert "restarted" in out
|
||||
|
||||
def test_systemd_status_surfaces_planned_restart_failure(self, monkeypatch, capsys):
|
||||
unit = SimpleNamespace(exists=lambda: True)
|
||||
monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False)
|
||||
monkeypatch.setattr(gateway_cli, "get_systemd_unit_path", lambda system=False: unit)
|
||||
monkeypatch.setattr(gateway_cli, "has_conflicting_systemd_units", lambda: False)
|
||||
monkeypatch.setattr(gateway_cli, "has_legacy_hermes_units", lambda: False)
|
||||
monkeypatch.setattr(gateway_cli, "systemd_unit_is_current", lambda system=False: True)
|
||||
monkeypatch.setattr(gateway_cli, "_runtime_health_lines", lambda: ["⚠ Last shutdown reason: Gateway restart requested"])
|
||||
monkeypatch.setattr(gateway_cli, "get_systemd_linger_status", lambda: (True, ""))
|
||||
monkeypatch.setattr(gateway_cli, "_read_systemd_unit_properties", lambda system=False: {
|
||||
"ActiveState": "failed",
|
||||
"SubState": "failed",
|
||||
"Result": "exit-code",
|
||||
"ExecMainStatus": str(GATEWAY_SERVICE_RESTART_EXIT_CODE),
|
||||
})
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_run_systemctl(args, **kwargs):
|
||||
calls.append(args)
|
||||
if args[:2] == ["status", gateway_cli.get_service_name()]:
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
if args[:2] == ["is-active", gateway_cli.get_service_name()]:
|
||||
return SimpleNamespace(returncode=3, stdout="failed\n", stderr="")
|
||||
raise AssertionError(f"Unexpected args: {args}")
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "_run_systemctl", fake_run_systemctl)
|
||||
|
||||
gateway_cli.systemd_status()
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "Planned restart is stuck in systemd failed state" in out
|
||||
|
||||
def test_gateway_status_dispatches_full_flag(self, monkeypatch):
|
||||
user_unit = SimpleNamespace(exists=lambda: True)
|
||||
system_unit = SimpleNamespace(exists=lambda: False)
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: True)
|
||||
monkeypatch.setattr(gateway_cli, "is_termux", lambda: False)
|
||||
monkeypatch.setattr(gateway_cli, "is_macos", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"get_systemd_unit_path",
|
||||
lambda system=False: system_unit if system else user_unit,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"get_gateway_runtime_snapshot",
|
||||
lambda system=False: gateway_cli.GatewayRuntimeSnapshot(
|
||||
manager="systemd (user)",
|
||||
service_installed=True,
|
||||
service_running=False,
|
||||
gateway_pids=(),
|
||||
service_scope="user",
|
||||
),
|
||||
)
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"systemd_status",
|
||||
lambda deep=False, system=False, full=False: calls.append((deep, system, full)),
|
||||
)
|
||||
|
||||
gateway_cli.gateway_command(
|
||||
SimpleNamespace(gateway_command="status", deep=False, system=False, full=True)
|
||||
)
|
||||
|
||||
assert calls == [(False, False, True)]
|
||||
|
||||
def test_gateway_install_passes_system_flags(self, monkeypatch):
|
||||
monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: True)
|
||||
monkeypatch.setattr(gateway_cli, "is_termux", lambda: False)
|
||||
|
|
@ -547,11 +679,15 @@ class TestGatewaySystemServiceRouting:
|
|||
)
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(gateway_cli, "systemd_status", lambda deep=False, system=False: calls.append((deep, system)))
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"systemd_status",
|
||||
lambda deep=False, system=False, full=False: calls.append((deep, system, full)),
|
||||
)
|
||||
|
||||
gateway_cli.gateway_command(SimpleNamespace(gateway_command="status", deep=False, system=False))
|
||||
|
||||
assert calls == [(False, False)]
|
||||
assert calls == [(False, False, False)]
|
||||
|
||||
def test_gateway_status_reports_manual_process_when_service_is_stopped(self, monkeypatch, capsys):
|
||||
user_unit = SimpleNamespace(exists=lambda: True)
|
||||
|
|
@ -565,7 +701,11 @@ class TestGatewaySystemServiceRouting:
|
|||
"get_systemd_unit_path",
|
||||
lambda system=False: system_unit if system else user_unit,
|
||||
)
|
||||
monkeypatch.setattr(gateway_cli, "systemd_status", lambda deep=False, system=False: print("service stopped"))
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"systemd_status",
|
||||
lambda deep=False, system=False, full=False: print("service stopped"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"get_gateway_runtime_snapshot",
|
||||
|
|
@ -1570,6 +1710,23 @@ class TestMigrateLegacyCommand:
|
|||
|
||||
assert called == {"interactive": False, "dry_run": False}
|
||||
|
||||
|
||||
class TestGatewayStatusParser:
|
||||
def test_gateway_status_subparser_accepts_full_flag(self):
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "hermes_cli.main", "gateway", "status", "-l", "--help"],
|
||||
cwd=str(gateway_cli.PROJECT_ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "unrecognized arguments" not in result.stderr
|
||||
|
||||
def test_gateway_command_migrate_legacy_dry_run_passes_through(
|
||||
self, monkeypatch
|
||||
):
|
||||
|
|
|
|||
174
tests/hermes_cli/test_image_gen_picker.py
Normal file
174
tests/hermes_cli/test_image_gen_picker.py
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
"""Tests for plugin image_gen providers injecting themselves into the picker.
|
||||
|
||||
Covers `_plugin_image_gen_providers`, `_visible_providers`, and
|
||||
`_toolset_needs_configuration_prompt` handling of plugin providers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import image_gen_registry
|
||||
from agent.image_gen_provider import ImageGenProvider
|
||||
|
||||
|
||||
class _FakeProvider(ImageGenProvider):
|
||||
def __init__(self, name: str, available: bool = True, schema=None, models=None):
|
||||
self._name = name
|
||||
self._available = available
|
||||
self._schema = schema or {
|
||||
"name": name.title(),
|
||||
"badge": "test",
|
||||
"tag": f"{name} test tag",
|
||||
"env_vars": [{"key": f"{name.upper()}_API_KEY", "prompt": f"{name} key"}],
|
||||
}
|
||||
self._models = models or [
|
||||
{"id": f"{name}-model-v1", "display": f"{name} v1",
|
||||
"speed": "~5s", "strengths": "test", "price": "$"},
|
||||
]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def list_models(self):
|
||||
return list(self._models)
|
||||
|
||||
def default_model(self):
|
||||
return self._models[0]["id"] if self._models else None
|
||||
|
||||
def get_setup_schema(self):
|
||||
return dict(self._schema)
|
||||
|
||||
def generate(self, prompt, aspect_ratio="landscape", **kw):
|
||||
return {"success": True, "image": f"{self._name}://{prompt}"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
image_gen_registry._reset_for_tests()
|
||||
yield
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class TestPluginPickerInjection:
|
||||
def test_plugin_providers_returns_registered(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("myimg"))
|
||||
|
||||
rows = tools_config._plugin_image_gen_providers()
|
||||
names = [r["name"] for r in rows]
|
||||
plugin_names = [r.get("image_gen_plugin_name") for r in rows]
|
||||
|
||||
assert "Myimg" in names
|
||||
assert "myimg" in plugin_names
|
||||
|
||||
def test_fal_skipped_to_avoid_duplicate(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
# Simulate a FAL plugin being registered — the picker already has
|
||||
# hardcoded FAL rows in TOOL_CATEGORIES, so plugin-FAL must be
|
||||
# skipped to avoid showing FAL twice.
|
||||
image_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
image_gen_registry.register_provider(_FakeProvider("openai"))
|
||||
|
||||
rows = tools_config._plugin_image_gen_providers()
|
||||
names = [r.get("image_gen_plugin_name") for r in rows]
|
||||
assert "fal" not in names
|
||||
assert "openai" in names
|
||||
|
||||
def test_visible_providers_includes_plugins_for_image_gen(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("someimg"))
|
||||
|
||||
cat = tools_config.TOOL_CATEGORIES["image_gen"]
|
||||
visible = tools_config._visible_providers(cat, {})
|
||||
plugin_names = [p.get("image_gen_plugin_name") for p in visible if p.get("image_gen_plugin_name")]
|
||||
assert "someimg" in plugin_names
|
||||
|
||||
def test_visible_providers_does_not_inject_into_other_categories(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("someimg"))
|
||||
|
||||
# Browser category must NOT see image_gen plugins.
|
||||
browser = tools_config.TOOL_CATEGORIES["browser"]
|
||||
visible = tools_config._visible_providers(browser, {})
|
||||
assert all(p.get("image_gen_plugin_name") is None for p in visible)
|
||||
|
||||
|
||||
class TestPluginCatalog:
|
||||
def test_plugin_catalog_returns_models(self):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("catimg"))
|
||||
|
||||
catalog, default = tools_config._plugin_image_gen_catalog("catimg")
|
||||
assert "catimg-model-v1" in catalog
|
||||
assert default == "catimg-model-v1"
|
||||
|
||||
def test_plugin_catalog_empty_for_unknown(self):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
catalog, default = tools_config._plugin_image_gen_catalog("does-not-exist")
|
||||
assert catalog == {}
|
||||
assert default is None
|
||||
|
||||
|
||||
class TestConfigPrompt:
|
||||
def test_image_gen_satisfied_by_plugin_provider(self, monkeypatch, tmp_path):
|
||||
"""When a plugin provider reports is_available(), the picker should
|
||||
not force a setup prompt on the user."""
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("avail-img", available=True))
|
||||
|
||||
assert tools_config._toolset_needs_configuration_prompt("image_gen", {}) is False
|
||||
|
||||
def test_image_gen_still_prompts_when_nothing_available(self, monkeypatch, tmp_path):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("unavail-img", available=False))
|
||||
|
||||
assert tools_config._toolset_needs_configuration_prompt("image_gen", {}) is True
|
||||
|
||||
|
||||
class TestConfigWriting:
|
||||
def test_picking_plugin_provider_writes_provider_and_model(self, monkeypatch, tmp_path):
|
||||
"""When a user picks a plugin-backed image_gen provider with no
|
||||
env vars needed, ``_configure_provider`` should write both
|
||||
``image_gen.provider`` and ``image_gen.model``."""
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
image_gen_registry.register_provider(_FakeProvider("noenv", schema={
|
||||
"name": "NoEnv",
|
||||
"badge": "free",
|
||||
"tag": "",
|
||||
"env_vars": [],
|
||||
}))
|
||||
|
||||
# Stub out the interactive model picker — no TTY in tests.
|
||||
monkeypatch.setattr(tools_config, "_prompt_choice", lambda *a, **kw: 0)
|
||||
|
||||
config: dict = {}
|
||||
provider_row = {
|
||||
"name": "NoEnv",
|
||||
"env_vars": [],
|
||||
"image_gen_plugin_name": "noenv",
|
||||
}
|
||||
tools_config._configure_provider(provider_row, config)
|
||||
|
||||
assert config["image_gen"]["provider"] == "noenv"
|
||||
assert config["image_gen"]["model"] == "noenv-model-v1"
|
||||
|
|
@ -32,6 +32,8 @@ def config_home(tmp_path, monkeypatch):
|
|||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
monkeypatch.delenv("STEPFUN_API_KEY", raising=False)
|
||||
monkeypatch.delenv("STEPFUN_BASE_URL", raising=False)
|
||||
return home
|
||||
|
||||
|
||||
|
|
@ -330,3 +332,33 @@ class TestBaseUrlValidation:
|
|||
|
||||
saved = get_env_value("GLM_BASE_URL") or ""
|
||||
assert saved == "", "Empty input should not save a base URL"
|
||||
|
||||
def test_stepfun_provider_saved_with_selected_region(self, config_home, monkeypatch):
|
||||
from hermes_cli.main import _model_flow_stepfun
|
||||
from hermes_cli.config import load_config, get_env_value
|
||||
|
||||
monkeypatch.setenv("STEPFUN_API_KEY", "stepfun-test-key")
|
||||
|
||||
with patch(
|
||||
"hermes_cli.main._prompt_provider_choice",
|
||||
return_value=1,
|
||||
), patch(
|
||||
"hermes_cli.models.fetch_api_models",
|
||||
return_value=["step-3.5-flash", "step-3-agent-lite"],
|
||||
), patch(
|
||||
"hermes_cli.auth._prompt_model_selection",
|
||||
return_value="step-3-agent-lite",
|
||||
), patch(
|
||||
"hermes_cli.auth.deactivate_provider",
|
||||
):
|
||||
_model_flow_stepfun(load_config(), "old-model")
|
||||
|
||||
import yaml
|
||||
|
||||
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
||||
model = config.get("model")
|
||||
assert isinstance(model, dict)
|
||||
assert model.get("provider") == "stepfun"
|
||||
assert model.get("default") == "step-3-agent-lite"
|
||||
assert model.get("base_url") == "https://api.stepfun.com/step_plan/v1"
|
||||
assert get_env_value("STEPFUN_BASE_URL") == "https://api.stepfun.com/step_plan/v1"
|
||||
|
|
|
|||
|
|
@ -63,6 +63,11 @@ class TestParseModelInput:
|
|||
assert provider == "zai"
|
||||
assert model == "glm-5"
|
||||
|
||||
def test_stepfun_alias_resolved(self):
|
||||
provider, model = parse_model_input("step:step-3.5-flash", "openrouter")
|
||||
assert provider == "stepfun"
|
||||
assert model == "step-3.5-flash"
|
||||
|
||||
def test_no_slash_no_colon_keeps_provider(self):
|
||||
provider, model = parse_model_input("gpt-5.4", "openrouter")
|
||||
assert provider == "openrouter"
|
||||
|
|
@ -154,6 +159,7 @@ class TestNormalizeProvider:
|
|||
assert normalize_provider("glm") == "zai"
|
||||
assert normalize_provider("kimi") == "kimi-coding"
|
||||
assert normalize_provider("moonshot") == "kimi-coding"
|
||||
assert normalize_provider("step") == "stepfun"
|
||||
assert normalize_provider("github-copilot") == "copilot"
|
||||
|
||||
def test_case_insensitive(self):
|
||||
|
|
@ -164,6 +170,7 @@ class TestProviderLabel:
|
|||
def test_known_labels_and_auto(self):
|
||||
assert provider_label("anthropic") == "Anthropic"
|
||||
assert provider_label("kimi") == "Kimi / Kimi Coding Plan"
|
||||
assert provider_label("stepfun") == "StepFun Step Plan"
|
||||
assert provider_label("copilot") == "GitHub Copilot"
|
||||
assert provider_label("copilot-acp") == "GitHub Copilot ACP"
|
||||
assert provider_label("auto") == "Auto"
|
||||
|
|
@ -193,6 +200,16 @@ class TestProviderModelIds:
|
|||
def test_zai_returns_glm_models(self):
|
||||
assert "glm-5" in provider_model_ids("zai")
|
||||
|
||||
def test_stepfun_prefers_live_catalog(self):
|
||||
with patch(
|
||||
"hermes_cli.auth.resolve_api_key_provider_credentials",
|
||||
return_value={"api_key": "***", "base_url": "https://api.stepfun.com/step_plan/v1"},
|
||||
), patch(
|
||||
"hermes_cli.models.fetch_api_models",
|
||||
return_value=["step-3.5-flash", "step-3-agent-lite"],
|
||||
):
|
||||
assert provider_model_ids("stepfun") == ["step-3.5-flash", "step-3-agent-lite"]
|
||||
|
||||
def test_copilot_prefers_live_catalog(self):
|
||||
with patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={"api_key": "gh-token"}), \
|
||||
patch("hermes_cli.models._fetch_github_models", return_value=["gpt-5.4", "claude-sonnet-4.6"]):
|
||||
|
|
@ -457,29 +474,62 @@ class TestValidateApiNotFound:
|
|||
assert "not found" in result["message"]
|
||||
|
||||
|
||||
# -- validate — API unreachable — reject with guidance ----------------
|
||||
# -- validate — API unreachable — soft-accept via catalog or warning --------
|
||||
|
||||
class TestValidateApiFallback:
|
||||
def test_any_model_rejected_when_api_down(self):
|
||||
result = _validate("anthropic/claude-opus-4.6", api_models=None)
|
||||
assert result["accepted"] is False
|
||||
assert result["persist"] is False
|
||||
"""When /models is unreachable, the validator must accept the model (with
|
||||
a warning) rather than reject it outright — otherwise provider switches
|
||||
fail in the gateway for any provider whose /models endpoint is down or
|
||||
doesn't exist (e.g. opencode-go returns 404 HTML).
|
||||
|
||||
def test_unknown_model_also_rejected_when_api_down(self):
|
||||
result = _validate("anthropic/claude-next-gen", api_models=None)
|
||||
assert result["accepted"] is False
|
||||
assert result["persist"] is False
|
||||
assert "could not reach" in result["message"].lower()
|
||||
Two paths:
|
||||
1. Provider has a curated catalog (``_PROVIDER_MODELS`` / live fetch):
|
||||
validate against it (recognized=True for known models,
|
||||
recognized=False with 'Note:' for unknown).
|
||||
2. Provider has no catalog: accept with a generic 'Note:' warning.
|
||||
|
||||
def test_zai_model_rejected_when_api_down(self):
|
||||
In both cases ``accepted`` and ``persist`` must be True so the gateway can
|
||||
write the ``_session_model_overrides`` entry.
|
||||
"""
|
||||
|
||||
def test_known_model_accepted_via_catalog_when_api_down(self):
|
||||
# Force the openrouter catalog lookup to return a deterministic list.
|
||||
with patch(
|
||||
"hermes_cli.models.provider_model_ids",
|
||||
return_value=["anthropic/claude-opus-4.6", "openai/gpt-5.4"],
|
||||
):
|
||||
result = _validate("anthropic/claude-opus-4.6", api_models=None)
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is True
|
||||
|
||||
def test_unknown_model_accepted_with_note_when_api_down(self):
|
||||
with patch(
|
||||
"hermes_cli.models.provider_model_ids",
|
||||
return_value=["anthropic/claude-opus-4.6", "openai/gpt-5.4"],
|
||||
):
|
||||
result = _validate("anthropic/claude-next-gen", api_models=None)
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is False
|
||||
# Message flags it as unverified against the catalog.
|
||||
assert "not found" in result["message"].lower() or "note" in result["message"].lower()
|
||||
|
||||
def test_zai_known_model_accepted_via_catalog_when_api_down(self):
|
||||
# glm-5 is in the zai curated catalog (_PROVIDER_MODELS["zai"]).
|
||||
result = _validate("glm-5", provider="zai", api_models=None)
|
||||
assert result["accepted"] is False
|
||||
assert result["persist"] is False
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is True
|
||||
|
||||
def test_unknown_provider_rejected_when_api_down(self):
|
||||
result = _validate("some-model", provider="totally-unknown", api_models=None)
|
||||
assert result["accepted"] is False
|
||||
assert result["persist"] is False
|
||||
def test_unknown_provider_soft_accepted_when_api_down(self):
|
||||
# No catalog for unknown providers — soft-accept with a Note.
|
||||
with patch("hermes_cli.models.provider_model_ids", return_value=[]):
|
||||
result = _validate("some-model", provider="totally-unknown", api_models=None)
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is False
|
||||
assert "note" in result["message"].lower()
|
||||
|
||||
def test_custom_endpoint_warns_with_probed_url_and_v1_hint(self):
|
||||
with patch(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from unittest.mock import patch, MagicMock
|
|||
|
||||
from hermes_cli.models import (
|
||||
OPENROUTER_MODELS, fetch_openrouter_models, model_ids, detect_provider_for_model,
|
||||
filter_nous_free_models, _NOUS_ALLOWED_FREE_MODELS,
|
||||
is_nous_free_tier, partition_nous_models_by_tier,
|
||||
check_nous_free_tier, _FREE_TIER_CACHE_TTL,
|
||||
)
|
||||
|
|
@ -88,6 +87,131 @@ class TestFetchOpenRouterModels:
|
|||
|
||||
assert models == OPENROUTER_MODELS
|
||||
|
||||
def test_filters_out_models_without_tool_support(self, monkeypatch):
|
||||
"""Models whose supported_parameters omits 'tools' must not appear in the picker.
|
||||
|
||||
hermes-agent is tool-calling-first — surfacing a non-tool model leads to
|
||||
immediate runtime failures when the user selects it. Ported from
|
||||
Kilo-Org/kilocode#9068.
|
||||
"""
|
||||
class _Resp:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
# opus-4.6 advertises tools → kept
|
||||
# nano-image has explicit supported_parameters that OMITS tools → dropped
|
||||
# qwen3.6-plus advertises tools → kept
|
||||
return (
|
||||
b'{"data":['
|
||||
b'{"id":"anthropic/claude-opus-4.6","pricing":{"prompt":"0.000015","completion":"0.000075"},'
|
||||
b'"supported_parameters":["temperature","tools","tool_choice"]},'
|
||||
b'{"id":"google/gemini-3-pro-image-preview","pricing":{"prompt":"0.00001","completion":"0.00003"},'
|
||||
b'"supported_parameters":["temperature","response_format"]},'
|
||||
b'{"id":"qwen/qwen3.6-plus","pricing":{"prompt":"0.000000325","completion":"0.00000195"},'
|
||||
b'"supported_parameters":["tools","temperature"]}'
|
||||
b']}'
|
||||
)
|
||||
|
||||
# Include the image-only id in the curated list so it has a chance to be surfaced.
|
||||
monkeypatch.setattr(
|
||||
_models_mod,
|
||||
"OPENROUTER_MODELS",
|
||||
[
|
||||
("anthropic/claude-opus-4.6", ""),
|
||||
("google/gemini-3-pro-image-preview", ""),
|
||||
("qwen/qwen3.6-plus", ""),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None)
|
||||
with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()):
|
||||
models = fetch_openrouter_models(force_refresh=True)
|
||||
|
||||
ids = [mid for mid, _ in models]
|
||||
assert "anthropic/claude-opus-4.6" in ids
|
||||
assert "qwen/qwen3.6-plus" in ids
|
||||
# Image-only model advertised supported_parameters WITHOUT tools → must be dropped.
|
||||
assert "google/gemini-3-pro-image-preview" not in ids
|
||||
|
||||
def test_permissive_when_supported_parameters_missing(self, monkeypatch):
|
||||
"""Models missing the supported_parameters field keep appearing in the picker.
|
||||
|
||||
Some OpenRouter-compatible gateways (Nous Portal, private mirrors, older
|
||||
catalog snapshots) don't populate supported_parameters. Treating missing
|
||||
as 'unknown → allow' prevents the picker from silently emptying on
|
||||
those gateways.
|
||||
"""
|
||||
class _Resp:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
# No supported_parameters field at all on either entry.
|
||||
return (
|
||||
b'{"data":['
|
||||
b'{"id":"anthropic/claude-opus-4.6","pricing":{"prompt":"0.000015","completion":"0.000075"}},'
|
||||
b'{"id":"qwen/qwen3.6-plus","pricing":{"prompt":"0.000000325","completion":"0.00000195"}}'
|
||||
b']}'
|
||||
)
|
||||
|
||||
monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None)
|
||||
with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()):
|
||||
models = fetch_openrouter_models(force_refresh=True)
|
||||
|
||||
ids = [mid for mid, _ in models]
|
||||
assert "anthropic/claude-opus-4.6" in ids
|
||||
assert "qwen/qwen3.6-plus" in ids
|
||||
|
||||
|
||||
class TestOpenRouterToolSupportHelper:
|
||||
"""Unit tests for _openrouter_model_supports_tools (Kilo port #9068)."""
|
||||
|
||||
def test_tools_in_supported_parameters(self):
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools(
|
||||
{"id": "x", "supported_parameters": ["temperature", "tools"]}
|
||||
) is True
|
||||
|
||||
def test_tools_missing_from_supported_parameters(self):
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools(
|
||||
{"id": "x", "supported_parameters": ["temperature", "response_format"]}
|
||||
) is False
|
||||
|
||||
def test_supported_parameters_absent_is_permissive(self):
|
||||
"""Missing field → allow (so older / non-OR gateways still work)."""
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools({"id": "x"}) is True
|
||||
|
||||
def test_supported_parameters_none_is_permissive(self):
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools({"id": "x", "supported_parameters": None}) is True
|
||||
|
||||
def test_supported_parameters_malformed_is_permissive(self):
|
||||
"""Malformed (non-list) value → allow rather than silently drop."""
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools(
|
||||
{"id": "x", "supported_parameters": "tools,temperature"}
|
||||
) is True
|
||||
|
||||
def test_non_dict_item_is_permissive(self):
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools(None) is True
|
||||
assert _openrouter_model_supports_tools("anthropic/claude-opus-4.6") is True
|
||||
|
||||
def test_empty_supported_parameters_list_drops_model(self):
|
||||
"""Explicit empty list → no tools → drop."""
|
||||
from hermes_cli.models import _openrouter_model_supports_tools
|
||||
assert _openrouter_model_supports_tools(
|
||||
{"id": "x", "supported_parameters": []}
|
||||
) is False
|
||||
|
||||
|
||||
class TestFindOpenrouterSlug:
|
||||
def test_exact_match(self):
|
||||
|
|
@ -168,89 +292,6 @@ class TestDetectProviderForModel:
|
|||
assert result[0] not in ("nous",) # nous has claude models but shouldn't be suggested
|
||||
|
||||
|
||||
class TestFilterNousFreeModels:
|
||||
"""Tests for filter_nous_free_models — Nous Portal free-model policy."""
|
||||
|
||||
_PAID = {"prompt": "0.000003", "completion": "0.000015"}
|
||||
_FREE = {"prompt": "0", "completion": "0"}
|
||||
|
||||
def test_paid_models_kept(self):
|
||||
"""Regular paid models pass through unchanged."""
|
||||
models = ["anthropic/claude-opus-4.6", "openai/gpt-5.4"]
|
||||
pricing = {m: self._PAID for m in models}
|
||||
assert filter_nous_free_models(models, pricing) == models
|
||||
|
||||
def test_free_non_allowlist_models_removed(self):
|
||||
"""Free models NOT in the allowlist are filtered out."""
|
||||
models = ["anthropic/claude-opus-4.6", "arcee-ai/trinity-large-preview:free"]
|
||||
pricing = {
|
||||
"anthropic/claude-opus-4.6": self._PAID,
|
||||
"arcee-ai/trinity-large-preview:free": self._FREE,
|
||||
}
|
||||
result = filter_nous_free_models(models, pricing)
|
||||
assert result == ["anthropic/claude-opus-4.6"]
|
||||
|
||||
def test_allowlist_model_kept_when_free(self):
|
||||
"""Allowlist models are kept when they report as free."""
|
||||
models = ["anthropic/claude-opus-4.6", "xiaomi/mimo-v2-pro"]
|
||||
pricing = {
|
||||
"anthropic/claude-opus-4.6": self._PAID,
|
||||
"xiaomi/mimo-v2-pro": self._FREE,
|
||||
}
|
||||
result = filter_nous_free_models(models, pricing)
|
||||
assert result == ["anthropic/claude-opus-4.6", "xiaomi/mimo-v2-pro"]
|
||||
|
||||
def test_allowlist_model_removed_when_paid(self):
|
||||
"""Allowlist models are removed when they are NOT free."""
|
||||
models = ["anthropic/claude-opus-4.6", "xiaomi/mimo-v2-pro"]
|
||||
pricing = {
|
||||
"anthropic/claude-opus-4.6": self._PAID,
|
||||
"xiaomi/mimo-v2-pro": self._PAID,
|
||||
}
|
||||
result = filter_nous_free_models(models, pricing)
|
||||
assert result == ["anthropic/claude-opus-4.6"]
|
||||
|
||||
def test_no_pricing_returns_all(self):
|
||||
"""When pricing data is unavailable, all models pass through."""
|
||||
models = ["anthropic/claude-opus-4.6", "nvidia/nemotron-3-super-120b-a12b:free"]
|
||||
assert filter_nous_free_models(models, {}) == models
|
||||
|
||||
def test_model_with_no_pricing_entry_treated_as_paid(self):
|
||||
"""A model missing from the pricing dict is kept (assumed paid)."""
|
||||
models = ["anthropic/claude-opus-4.6", "openai/gpt-5.4"]
|
||||
pricing = {"anthropic/claude-opus-4.6": self._PAID} # gpt-5.4 not in pricing
|
||||
result = filter_nous_free_models(models, pricing)
|
||||
assert result == models
|
||||
|
||||
def test_mixed_scenario(self):
|
||||
"""End-to-end: mix of paid, free-allowed, free-disallowed, allowlist-not-free."""
|
||||
models = [
|
||||
"anthropic/claude-opus-4.6", # paid, not allowlist → keep
|
||||
"nvidia/nemotron-3-super-120b-a12b:free", # free, not allowlist → drop
|
||||
"xiaomi/mimo-v2-pro", # free, allowlist → keep
|
||||
"xiaomi/mimo-v2-omni", # paid, allowlist → drop
|
||||
"openai/gpt-5.4", # paid, not allowlist → keep
|
||||
]
|
||||
pricing = {
|
||||
"anthropic/claude-opus-4.6": self._PAID,
|
||||
"nvidia/nemotron-3-super-120b-a12b:free": self._FREE,
|
||||
"xiaomi/mimo-v2-pro": self._FREE,
|
||||
"xiaomi/mimo-v2-omni": self._PAID,
|
||||
"openai/gpt-5.4": self._PAID,
|
||||
}
|
||||
result = filter_nous_free_models(models, pricing)
|
||||
assert result == [
|
||||
"anthropic/claude-opus-4.6",
|
||||
"xiaomi/mimo-v2-pro",
|
||||
"openai/gpt-5.4",
|
||||
]
|
||||
|
||||
def test_allowlist_contains_expected_models(self):
|
||||
"""Sanity: the allowlist has the models we expect."""
|
||||
assert "xiaomi/mimo-v2-pro" in _NOUS_ALLOWED_FREE_MODELS
|
||||
assert "xiaomi/mimo-v2-omni" in _NOUS_ALLOWED_FREE_MODELS
|
||||
|
||||
|
||||
class TestIsNousFreeTier:
|
||||
"""Tests for is_nous_free_tier — account tier detection."""
|
||||
|
||||
|
|
@ -376,3 +417,190 @@ class TestCheckNousFreeTierCache:
|
|||
def test_cache_ttl_is_short(self):
|
||||
"""TTL should be short enough to catch upgrades quickly (<=5 min)."""
|
||||
assert _FREE_TIER_CACHE_TTL <= 300
|
||||
|
||||
|
||||
class TestNousRecommendedModels:
|
||||
"""Tests for fetch_nous_recommended_models + get_nous_recommended_aux_model."""
|
||||
|
||||
_SAMPLE_PAYLOAD = {
|
||||
"paidRecommendedModels": [],
|
||||
"freeRecommendedModels": [],
|
||||
"paidRecommendedCompactionModel": None,
|
||||
"paidRecommendedVisionModel": None,
|
||||
"freeRecommendedCompactionModel": {
|
||||
"modelName": "google/gemini-3-flash-preview",
|
||||
"displayName": "Google: Gemini 3 Flash Preview",
|
||||
},
|
||||
"freeRecommendedVisionModel": {
|
||||
"modelName": "google/gemini-3-flash-preview",
|
||||
"displayName": "Google: Gemini 3 Flash Preview",
|
||||
},
|
||||
}
|
||||
|
||||
def setup_method(self):
|
||||
_models_mod._nous_recommended_cache.clear()
|
||||
|
||||
def teardown_method(self):
|
||||
_models_mod._nous_recommended_cache.clear()
|
||||
|
||||
def _mock_urlopen(self, payload):
|
||||
"""Return a context-manager mock mimicking urllib.request.urlopen()."""
|
||||
import json as _json
|
||||
response = MagicMock()
|
||||
response.read.return_value = _json.dumps(payload).encode()
|
||||
cm = MagicMock()
|
||||
cm.__enter__.return_value = response
|
||||
cm.__exit__.return_value = False
|
||||
return cm
|
||||
|
||||
def test_fetch_caches_per_portal_url(self):
|
||||
from hermes_cli.models import fetch_nous_recommended_models
|
||||
mock_cm = self._mock_urlopen(self._SAMPLE_PAYLOAD)
|
||||
with patch("urllib.request.urlopen", return_value=mock_cm) as mock_urlopen:
|
||||
a = fetch_nous_recommended_models("https://portal.example.com")
|
||||
b = fetch_nous_recommended_models("https://portal.example.com")
|
||||
assert a == self._SAMPLE_PAYLOAD
|
||||
assert b == self._SAMPLE_PAYLOAD
|
||||
assert mock_urlopen.call_count == 1 # second call served from cache
|
||||
|
||||
def test_fetch_cache_is_keyed_per_portal(self):
|
||||
from hermes_cli.models import fetch_nous_recommended_models
|
||||
mock_cm = self._mock_urlopen(self._SAMPLE_PAYLOAD)
|
||||
with patch("urllib.request.urlopen", return_value=mock_cm) as mock_urlopen:
|
||||
fetch_nous_recommended_models("https://portal.example.com")
|
||||
fetch_nous_recommended_models("https://portal.staging-nousresearch.com")
|
||||
assert mock_urlopen.call_count == 2 # different portals → separate fetches
|
||||
|
||||
def test_fetch_returns_empty_on_network_failure(self):
|
||||
from hermes_cli.models import fetch_nous_recommended_models
|
||||
with patch("urllib.request.urlopen", side_effect=OSError("boom")):
|
||||
result = fetch_nous_recommended_models("https://portal.example.com")
|
||||
assert result == {}
|
||||
|
||||
def test_fetch_force_refresh_bypasses_cache(self):
|
||||
from hermes_cli.models import fetch_nous_recommended_models
|
||||
mock_cm = self._mock_urlopen(self._SAMPLE_PAYLOAD)
|
||||
with patch("urllib.request.urlopen", return_value=mock_cm) as mock_urlopen:
|
||||
fetch_nous_recommended_models("https://portal.example.com")
|
||||
fetch_nous_recommended_models("https://portal.example.com", force_refresh=True)
|
||||
assert mock_urlopen.call_count == 2
|
||||
|
||||
def test_get_aux_model_returns_vision_recommendation(self):
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
with patch(
|
||||
"hermes_cli.models.fetch_nous_recommended_models",
|
||||
return_value=self._SAMPLE_PAYLOAD,
|
||||
):
|
||||
# Free tier → free vision recommendation.
|
||||
model = get_nous_recommended_aux_model(vision=True, free_tier=True)
|
||||
assert model == "google/gemini-3-flash-preview"
|
||||
|
||||
def test_get_aux_model_returns_compaction_recommendation(self):
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = dict(self._SAMPLE_PAYLOAD)
|
||||
payload["freeRecommendedCompactionModel"] = {"modelName": "minimax/minimax-m2.7"}
|
||||
with patch(
|
||||
"hermes_cli.models.fetch_nous_recommended_models",
|
||||
return_value=payload,
|
||||
):
|
||||
model = get_nous_recommended_aux_model(vision=False, free_tier=True)
|
||||
assert model == "minimax/minimax-m2.7"
|
||||
|
||||
def test_get_aux_model_returns_none_when_field_null(self):
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = dict(self._SAMPLE_PAYLOAD)
|
||||
payload["freeRecommendedCompactionModel"] = None
|
||||
with patch(
|
||||
"hermes_cli.models.fetch_nous_recommended_models",
|
||||
return_value=payload,
|
||||
):
|
||||
model = get_nous_recommended_aux_model(vision=False, free_tier=True)
|
||||
assert model is None
|
||||
|
||||
def test_get_aux_model_returns_none_on_empty_payload(self):
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
with patch("hermes_cli.models.fetch_nous_recommended_models", return_value={}):
|
||||
assert get_nous_recommended_aux_model(vision=False, free_tier=True) is None
|
||||
assert get_nous_recommended_aux_model(vision=True, free_tier=False) is None
|
||||
|
||||
def test_get_aux_model_returns_none_when_modelname_blank(self):
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {"freeRecommendedCompactionModel": {"modelName": " "}}
|
||||
with patch(
|
||||
"hermes_cli.models.fetch_nous_recommended_models",
|
||||
return_value=payload,
|
||||
):
|
||||
assert get_nous_recommended_aux_model(vision=False, free_tier=True) is None
|
||||
|
||||
def test_paid_tier_prefers_paid_recommendation(self):
|
||||
"""Paid-tier users should get the paid model when it's populated."""
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {
|
||||
"paidRecommendedCompactionModel": {"modelName": "anthropic/claude-opus-4.7"},
|
||||
"freeRecommendedCompactionModel": {"modelName": "google/gemini-3-flash-preview"},
|
||||
"paidRecommendedVisionModel": {"modelName": "openai/gpt-5.4"},
|
||||
"freeRecommendedVisionModel": {"modelName": "google/gemini-3-flash-preview"},
|
||||
}
|
||||
with patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload):
|
||||
text = get_nous_recommended_aux_model(vision=False, free_tier=False)
|
||||
vision = get_nous_recommended_aux_model(vision=True, free_tier=False)
|
||||
assert text == "anthropic/claude-opus-4.7"
|
||||
assert vision == "openai/gpt-5.4"
|
||||
|
||||
def test_paid_tier_falls_back_to_free_when_paid_is_null(self):
|
||||
"""If the Portal returns null for the paid field, fall back to free."""
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {
|
||||
"paidRecommendedCompactionModel": None,
|
||||
"freeRecommendedCompactionModel": {"modelName": "google/gemini-3-flash-preview"},
|
||||
"paidRecommendedVisionModel": None,
|
||||
"freeRecommendedVisionModel": {"modelName": "google/gemini-3-flash-preview"},
|
||||
}
|
||||
with patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload):
|
||||
text = get_nous_recommended_aux_model(vision=False, free_tier=False)
|
||||
vision = get_nous_recommended_aux_model(vision=True, free_tier=False)
|
||||
assert text == "google/gemini-3-flash-preview"
|
||||
assert vision == "google/gemini-3-flash-preview"
|
||||
|
||||
def test_free_tier_never_uses_paid_recommendation(self):
|
||||
"""Free-tier users must not get paid-only recommendations."""
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {
|
||||
"paidRecommendedCompactionModel": {"modelName": "anthropic/claude-opus-4.7"},
|
||||
"freeRecommendedCompactionModel": None, # no free recommendation
|
||||
}
|
||||
with patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload):
|
||||
model = get_nous_recommended_aux_model(vision=False, free_tier=True)
|
||||
# Free tier must return None — never leak the paid model.
|
||||
assert model is None
|
||||
|
||||
def test_auto_detects_tier_when_not_supplied(self):
|
||||
"""Default behaviour: call check_nous_free_tier() to pick the tier."""
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {
|
||||
"paidRecommendedCompactionModel": {"modelName": "paid-model"},
|
||||
"freeRecommendedCompactionModel": {"modelName": "free-model"},
|
||||
}
|
||||
with (
|
||||
patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload),
|
||||
patch("hermes_cli.models.check_nous_free_tier", return_value=True),
|
||||
):
|
||||
assert get_nous_recommended_aux_model(vision=False) == "free-model"
|
||||
with (
|
||||
patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload),
|
||||
patch("hermes_cli.models.check_nous_free_tier", return_value=False),
|
||||
):
|
||||
assert get_nous_recommended_aux_model(vision=False) == "paid-model"
|
||||
|
||||
def test_tier_detection_error_defaults_to_paid(self):
|
||||
"""If tier detection raises, assume paid so we don't downgrade silently."""
|
||||
from hermes_cli.models import get_nous_recommended_aux_model
|
||||
payload = {
|
||||
"paidRecommendedCompactionModel": {"modelName": "paid-model"},
|
||||
"freeRecommendedCompactionModel": {"modelName": "free-model"},
|
||||
}
|
||||
with (
|
||||
patch("hermes_cli.models.fetch_nous_recommended_models", return_value=payload),
|
||||
patch("hermes_cli.models.check_nous_free_tier", side_effect=RuntimeError("boom")),
|
||||
):
|
||||
assert get_nous_recommended_aux_model(vision=False) == "paid-model"
|
||||
|
|
|
|||
124
tests/hermes_cli/test_models_dev_preferred_merge.py
Normal file
124
tests/hermes_cli/test_models_dev_preferred_merge.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
"""Tests for the models.dev-preferred merge behavior in provider_model_ids
|
||||
and list_authenticated_providers.
|
||||
|
||||
These guard the contract:
|
||||
|
||||
* For providers in ``_MODELS_DEV_PREFERRED`` (opencode-go, opencode-zen,
|
||||
xiaomi, deepseek, smaller inference providers), both the CLI model
|
||||
picker path (``provider_model_ids``) and the gateway ``/model`` picker
|
||||
path (``list_authenticated_providers``) merge fresh models.dev entries
|
||||
on top of the curated static list.
|
||||
* OpenRouter and Nous Portal are NEVER merged — they keep their curated
|
||||
(OpenRouter) or live-Portal (Nous) semantics.
|
||||
* If models.dev is unreachable (offline / CI), the curated list is the
|
||||
fallback — no crash, no empty list.
|
||||
|
||||
Merging is what lets new models (e.g. ``mimo-v2.5-pro`` on opencode-go)
|
||||
appear in ``/model`` without a Hermes release.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.models import (
|
||||
_MODELS_DEV_PREFERRED,
|
||||
_merge_with_models_dev,
|
||||
provider_model_ids,
|
||||
)
|
||||
|
||||
|
||||
class TestMergeHelper:
|
||||
def test_merge_empty_mdev_returns_curated(self):
|
||||
"""When models.dev returns nothing, curated list is preserved verbatim."""
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=[]):
|
||||
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro", "kimi-k2.6"])
|
||||
assert out == ["mimo-v2-pro", "kimi-k2.6"]
|
||||
|
||||
def test_merge_mdev_raises_returns_curated(self):
|
||||
"""Offline / broken models.dev must not break the catalog path."""
|
||||
def boom(_provider):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
with patch("agent.models_dev.list_agentic_models", side_effect=boom):
|
||||
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro"])
|
||||
assert out == ["mimo-v2-pro"]
|
||||
|
||||
def test_merge_mdev_first_then_curated_extras(self):
|
||||
"""models.dev entries come first; curated-only entries are appended."""
|
||||
mdev = ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6"]
|
||||
curated = ["kimi-k2.6", "kimi-k2.5", "mimo-v2-pro"] # kimi-k2.5 is curated-only
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = _merge_with_models_dev("opencode-go", curated)
|
||||
# models.dev entries first (in order), then curated-only entries
|
||||
assert out == ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6", "kimi-k2.5"]
|
||||
|
||||
def test_merge_case_insensitive_dedup(self):
|
||||
"""Dedup is case-insensitive but preserves the first occurrence's casing."""
|
||||
mdev = ["MiniMax-M2.7"]
|
||||
curated = ["minimax-m2.7", "minimax-m2.5"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = _merge_with_models_dev("minimax", curated)
|
||||
# models.dev casing wins since it came first
|
||||
assert out == ["MiniMax-M2.7", "minimax-m2.5"]
|
||||
|
||||
|
||||
class TestProviderModelIdsPreferred:
|
||||
def test_opencode_go_is_preferred(self):
|
||||
assert "opencode-go" in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_opencode_go_includes_fresh_models_dev_entries(self):
|
||||
"""provider_model_ids('opencode-go') adds models.dev entries on top."""
|
||||
mdev = ["mimo-v2.5-pro", "mimo-v2.5", "mimo-v2-pro", "kimi-k2.6"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = provider_model_ids("opencode-go")
|
||||
# Fresh models must surface (this is exactly the reported bug fix:
|
||||
# mimo-v2.5-pro should be pickable on opencode-go).
|
||||
assert "mimo-v2.5-pro" in out
|
||||
assert "mimo-v2.5" in out
|
||||
# Curated entries are still present.
|
||||
assert "mimo-v2-pro" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
def test_opencode_go_offline_falls_back_to_curated(self):
|
||||
"""Offline models.dev → curated-only list, no crash."""
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=[]):
|
||||
out = provider_model_ids("opencode-go")
|
||||
# Curated floor (see hermes_cli/models.py _PROVIDER_MODELS["opencode-go"])
|
||||
assert "mimo-v2-pro" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
def test_opencode_zen_includes_fresh_models(self):
|
||||
"""opencode-zen follows the same pattern as opencode-go."""
|
||||
assert "opencode-zen" in _MODELS_DEV_PREFERRED
|
||||
mdev = ["claude-opus-4-7", "kimi-k2.6", "glm-5.1"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = provider_model_ids("opencode-zen")
|
||||
assert "claude-opus-4-7" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
|
||||
class TestOpenRouterAndNousUnchanged:
|
||||
"""Per Teknium: openrouter and nous are NEVER merged with models.dev."""
|
||||
|
||||
def test_openrouter_not_in_preferred_set(self):
|
||||
assert "openrouter" not in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_nous_not_in_preferred_set(self):
|
||||
assert "nous" not in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_openrouter_does_not_call_merge(self):
|
||||
"""openrouter takes its own live path — merge helper must NOT run."""
|
||||
with patch(
|
||||
"hermes_cli.models._merge_with_models_dev",
|
||||
side_effect=AssertionError("merge should not be called for openrouter"),
|
||||
):
|
||||
# Even if model_ids() fails for some other reason, we just care
|
||||
# that the merge path isn't invoked.
|
||||
try:
|
||||
provider_model_ids("openrouter")
|
||||
except AssertionError:
|
||||
raise
|
||||
except Exception:
|
||||
pass # model_ids() may fail in the hermetic test env — that's fine.
|
||||
|
|
@ -6,16 +6,41 @@ from unittest.mock import patch
|
|||
from hermes_cli.model_switch import list_authenticated_providers
|
||||
|
||||
|
||||
# Minimum set of models that must be present for opencode-go no matter
|
||||
# whether the picker sourced its list from curated-only or curated+models.dev.
|
||||
# The curated list in hermes_cli/models.py defines the floor; models.dev only
|
||||
# ever adds names on top of it via _merge_with_models_dev.
|
||||
_OPENCODE_GO_REQUIRED = {
|
||||
"kimi-k2.6",
|
||||
"kimi-k2.5",
|
||||
"glm-5.1",
|
||||
"glm-5",
|
||||
"mimo-v2-pro",
|
||||
"mimo-v2-omni",
|
||||
"minimax-m2.7",
|
||||
"minimax-m2.5",
|
||||
}
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENCODE_GO_API_KEY": "test-key"}, clear=False)
|
||||
def test_opencode_go_appears_when_api_key_set():
|
||||
"""opencode-go should appear in list_authenticated_providers when OPENCODE_GO_API_KEY is set."""
|
||||
providers = list_authenticated_providers(current_provider="openrouter")
|
||||
|
||||
providers = list_authenticated_providers(current_provider="openrouter", max_models=50)
|
||||
|
||||
# Find opencode-go in results
|
||||
opencode_go = next((p for p in providers if p["slug"] == "opencode-go"), None)
|
||||
|
||||
|
||||
assert opencode_go is not None, "opencode-go should appear when OPENCODE_GO_API_KEY is set"
|
||||
assert opencode_go["models"] == ["kimi-k2.5", "glm-5.1", "glm-5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5"]
|
||||
# Behavior check: the curated floor must be present. The list may also
|
||||
# include extra models.dev entries (e.g. mimo-v2.5-pro) when the registry
|
||||
# is reachable — that's the whole point of the models.dev-preferred merge
|
||||
# introduced for opencode-go, so don't pin to an exact list here.
|
||||
present = set(opencode_go["models"])
|
||||
missing = _OPENCODE_GO_REQUIRED - present
|
||||
assert not missing, (
|
||||
f"opencode-go picker should include the curated floor; missing: {sorted(missing)}. "
|
||||
f"Got: {opencode_go['models']}"
|
||||
)
|
||||
# opencode-go can appear as "built-in" (from PROVIDER_TO_MODELS_DEV when
|
||||
# models.dev is reachable) or "hermes" (from HERMES_OVERLAYS fallback when
|
||||
# the API is unavailable, e.g. in CI).
|
||||
|
|
@ -26,10 +51,10 @@ def test_opencode_go_not_appears_when_no_creds():
|
|||
"""opencode-go should NOT appear when no credentials are set."""
|
||||
# Ensure OPENCODE_GO_API_KEY is not set
|
||||
env_without_key = {k: v for k, v in os.environ.items() if k != "OPENCODE_GO_API_KEY"}
|
||||
|
||||
|
||||
with patch.dict(os.environ, env_without_key, clear=True):
|
||||
providers = list_authenticated_providers(current_provider="openrouter")
|
||||
|
||||
|
||||
# opencode-go should not be in results
|
||||
opencode_go = next((p for p in providers if p["slug"] == "opencode-go"), None)
|
||||
assert opencode_go is None, "opencode-go should not appear without credentials"
|
||||
|
|
|
|||
133
tests/hermes_cli/test_opencode_go_validation_fallback.py
Normal file
133
tests/hermes_cli/test_opencode_go_validation_fallback.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Tests for the static-catalog fallback in validate_requested_model.
|
||||
|
||||
OpenCode Go and OpenCode Zen publish an OpenAI-compatible API at paths that do
|
||||
NOT expose ``/models`` (the path returns the marketing site's HTML 404). This
|
||||
caused ``validate_requested_model`` to return ``accepted=False`` for every
|
||||
model on those providers, which in turn made ``switch_model()`` fail and the
|
||||
gateway's ``/model <name> --provider opencode-go`` command never write to
|
||||
``_session_model_overrides``.
|
||||
|
||||
These tests cover the catalog-fallback path: when ``fetch_api_models`` returns
|
||||
``None``, the validator must consult ``provider_model_ids()`` for the provider
|
||||
(populated from ``_PROVIDER_MODELS``) rather than rejecting outright.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from hermes_cli.models import validate_requested_model
|
||||
|
||||
|
||||
_UNREACHABLE_PROBE = {
|
||||
"models": None,
|
||||
"probed_url": "https://opencode.ai/zen/go/v1/models",
|
||||
"resolved_base_url": "https://opencode.ai/zen/go/v1",
|
||||
"suggested_base_url": None,
|
||||
"used_fallback": False,
|
||||
}
|
||||
|
||||
|
||||
def _patched(func):
|
||||
"""Decorator: force fetch_api_models / probe_api_models to simulate an
|
||||
unreachable /models endpoint, proving the catalog path is used."""
|
||||
def wrapper(*args, **kwargs):
|
||||
with patch("hermes_cli.models.fetch_api_models", return_value=None), \
|
||||
patch("hermes_cli.models.probe_api_models", return_value=_UNREACHABLE_PROBE):
|
||||
return func(*args, **kwargs)
|
||||
wrapper.__name__ = func.__name__
|
||||
return wrapper
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# opencode-go: curated catalog in _PROVIDER_MODELS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_go_known_model_accepted():
|
||||
"""A model present in the opencode-go curated catalog must be accepted
|
||||
even when /models is unreachable."""
|
||||
result = validate_requested_model("kimi-k2.6", "opencode-go")
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is True
|
||||
assert result["message"] is None
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_go_known_model_case_insensitive():
|
||||
"""Catalog lookup is case-insensitive."""
|
||||
result = validate_requested_model("KIMI-K2.6", "opencode-go")
|
||||
assert result["accepted"] is True
|
||||
assert result["recognized"] is True
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_go_typo_auto_corrected():
|
||||
"""A close typo (>= 0.9 similarity) is auto-corrected to the catalog
|
||||
entry."""
|
||||
# 'kimi-k2.55' vs 'kimi-k2.5' ratio ≈ 0.95 — within the 0.9 cutoff.
|
||||
result = validate_requested_model("kimi-k2.55", "opencode-go")
|
||||
assert result["accepted"] is True
|
||||
assert result["recognized"] is True
|
||||
assert result.get("corrected_model") == "kimi-k2.5"
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_go_unknown_model_accepted_with_suggestion():
|
||||
"""An unknown model that has a medium-similarity match (>= 0.5 but < 0.9)
|
||||
is accepted with recognized=False and a 'similar models' hint. The key
|
||||
invariant: the gateway MUST be able to persist this override, so
|
||||
accepted/persist must both be True."""
|
||||
# 'kimi-k3-preview' vs 'kimi-k2.6' — similar enough to suggest, not to auto-correct.
|
||||
result = validate_requested_model("kimi-k3-preview", "opencode-go")
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is False
|
||||
assert "kimi-k3-preview" in result["message"]
|
||||
assert "curated catalog" in result["message"]
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_go_totally_unknown_model_still_accepted():
|
||||
"""A model with zero similarity to the catalog is still accepted (no
|
||||
suggestion line) so the user can try a model that hasn't made it into the
|
||||
curated list yet."""
|
||||
result = validate_requested_model("some-brand-new-model", "opencode-go")
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is False
|
||||
# No suggestion text (no close matches)
|
||||
assert "Similar models" not in result["message"]
|
||||
assert "opencode" in result["message"].lower() or "opencode go" in result["message"].lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# opencode-zen: same pattern as opencode-go
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@_patched
|
||||
def test_opencode_zen_known_model_accepted():
|
||||
"""opencode-zen also uses _PROVIDER_MODELS; kimi-k2 is in its catalog."""
|
||||
result = validate_requested_model("kimi-k2", "opencode-zen")
|
||||
assert result["accepted"] is True
|
||||
assert result["recognized"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unknown provider with no catalog: soft-accept (honors the comment's intent)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@_patched
|
||||
def test_provider_without_catalog_accepts_with_warning():
|
||||
"""When a provider has no entry in _PROVIDER_MODELS and /models is
|
||||
unreachable, accept the model with a 'Note:' warning rather than reject.
|
||||
This matches the in-code comment: 'Accept and persist, but warn so typos
|
||||
don't silently break things.'"""
|
||||
# Use a made-up provider name that won't resolve to any catalog.
|
||||
result = validate_requested_model("some-model", "provider-that-does-not-exist")
|
||||
assert result["accepted"] is True
|
||||
assert result["persist"] is True
|
||||
assert result["recognized"] is False
|
||||
assert "Note:" in result["message"]
|
||||
357
tests/hermes_cli/test_plugin_scanner_recursion.py
Normal file
357
tests/hermes_cli/test_plugin_scanner_recursion.py
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
"""Tests for PR1 pluggable image gen: scanner recursion, kinds, path keys.
|
||||
|
||||
Covers ``_scan_directory`` recursion into category namespaces
|
||||
(``plugins/image_gen/openai/``), ``kind`` parsing, path-derived registry
|
||||
keys, and the new gate logic (bundled backends auto-load; user backends
|
||||
still opt-in; exclusive kind skipped; unknown kinds → standalone warning).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from hermes_cli.plugins import PluginManager, PluginManifest
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _write_plugin(
|
||||
root: Path,
|
||||
segments: list[str],
|
||||
*,
|
||||
manifest_extra: Dict[str, Any] | None = None,
|
||||
register_body: str = "pass",
|
||||
) -> Path:
|
||||
"""Create a plugin dir at ``root/<segments...>/`` with plugin.yaml + __init__.py.
|
||||
|
||||
``segments`` lets tests build both flat (``["my-plugin"]``) and
|
||||
category-namespaced (``["image_gen", "openai"]``) layouts.
|
||||
"""
|
||||
plugin_dir = root
|
||||
for seg in segments:
|
||||
plugin_dir = plugin_dir / seg
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = {
|
||||
"name": segments[-1],
|
||||
"version": "0.1.0",
|
||||
"description": f"Test plugin {'/'.join(segments)}",
|
||||
}
|
||||
if manifest_extra:
|
||||
manifest.update(manifest_extra)
|
||||
(plugin_dir / "plugin.yaml").write_text(yaml.dump(manifest))
|
||||
(plugin_dir / "__init__.py").write_text(
|
||||
f"def register(ctx):\n {register_body}\n"
|
||||
)
|
||||
return plugin_dir
|
||||
|
||||
|
||||
def _enable(hermes_home: Path, name: str) -> None:
|
||||
"""Append ``name`` to ``plugins.enabled`` in ``<hermes_home>/config.yaml``."""
|
||||
cfg_path = hermes_home / "config.yaml"
|
||||
cfg: dict = {}
|
||||
if cfg_path.exists():
|
||||
try:
|
||||
cfg = yaml.safe_load(cfg_path.read_text()) or {}
|
||||
except Exception:
|
||||
cfg = {}
|
||||
plugins_cfg = cfg.setdefault("plugins", {})
|
||||
enabled = plugins_cfg.setdefault("enabled", [])
|
||||
if isinstance(enabled, list) and name not in enabled:
|
||||
enabled.append(name)
|
||||
cfg_path.write_text(yaml.safe_dump(cfg))
|
||||
|
||||
|
||||
# ── Scanner recursion ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCategoryNamespaceRecursion:
|
||||
def test_category_namespace_discovered(self, tmp_path, monkeypatch):
|
||||
"""``<root>/image_gen/openai/plugin.yaml`` is discovered with key
|
||||
``image_gen/openai`` when the ``image_gen`` parent has no manifest."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
_write_plugin(user_plugins, ["image_gen", "openai"])
|
||||
_enable(hermes_home, "image_gen/openai")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert "image_gen/openai" in mgr._plugins
|
||||
loaded = mgr._plugins["image_gen/openai"]
|
||||
assert loaded.manifest.key == "image_gen/openai"
|
||||
assert loaded.manifest.name == "openai"
|
||||
assert loaded.enabled is True
|
||||
|
||||
def test_flat_plugin_key_matches_name(self, tmp_path, monkeypatch):
|
||||
"""Flat plugins keep their bare name as the key (back-compat)."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
_write_plugin(user_plugins, ["my-plugin"])
|
||||
_enable(hermes_home, "my-plugin")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert "my-plugin" in mgr._plugins
|
||||
assert mgr._plugins["my-plugin"].manifest.key == "my-plugin"
|
||||
|
||||
def test_depth_cap_two(self, tmp_path, monkeypatch):
|
||||
"""Plugins nested three levels deep are not discovered.
|
||||
|
||||
``<root>/a/b/c/plugin.yaml`` should NOT be picked up — cap is
|
||||
two segments.
|
||||
"""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
_write_plugin(user_plugins, ["a", "b", "c"])
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
non_bundled = [
|
||||
k for k, p in mgr._plugins.items()
|
||||
if p.manifest.source != "bundled"
|
||||
]
|
||||
assert non_bundled == []
|
||||
|
||||
def test_category_dir_with_manifest_is_leaf(self, tmp_path, monkeypatch):
|
||||
"""If ``image_gen/plugin.yaml`` exists, ``image_gen`` itself IS the
|
||||
plugin and its children are ignored."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
# parent has a manifest → stop recursing
|
||||
_write_plugin(user_plugins, ["image_gen"])
|
||||
# child also has a manifest — should NOT be found because we stop
|
||||
# at the parent.
|
||||
_write_plugin(user_plugins, ["image_gen", "openai"])
|
||||
_enable(hermes_home, "image_gen")
|
||||
_enable(hermes_home, "image_gen/openai")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
# The bundled plugins/image_gen/openai/ exists in the repo — filter
|
||||
# it out so we're only asserting on the user-dir layout.
|
||||
user_plugins_in_registry = {
|
||||
k for k, p in mgr._plugins.items() if p.manifest.source != "bundled"
|
||||
}
|
||||
assert "image_gen" in user_plugins_in_registry
|
||||
assert "image_gen/openai" not in user_plugins_in_registry
|
||||
|
||||
|
||||
# ── Kind parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestKindField:
|
||||
def test_default_kind_is_standalone(self, tmp_path, monkeypatch):
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
_write_plugin(hermes_home / "plugins", ["p1"])
|
||||
_enable(hermes_home, "p1")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr._plugins["p1"].manifest.kind == "standalone"
|
||||
|
||||
@pytest.mark.parametrize("kind", ["backend", "exclusive", "standalone"])
|
||||
def test_valid_kinds_parsed(self, kind, tmp_path, monkeypatch):
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
_write_plugin(
|
||||
hermes_home / "plugins",
|
||||
["p1"],
|
||||
manifest_extra={"kind": kind},
|
||||
)
|
||||
# Not all kinds auto-load, but manifest should parse.
|
||||
_enable(hermes_home, "p1")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert "p1" in mgr._plugins
|
||||
assert mgr._plugins["p1"].manifest.kind == kind
|
||||
|
||||
def test_unknown_kind_falls_back_to_standalone(self, tmp_path, monkeypatch, caplog):
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
_write_plugin(
|
||||
hermes_home / "plugins",
|
||||
["p1"],
|
||||
manifest_extra={"kind": "bogus"},
|
||||
)
|
||||
_enable(hermes_home, "p1")
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr._plugins["p1"].manifest.kind == "standalone"
|
||||
assert any(
|
||||
"unknown kind" in rec.getMessage() for rec in caplog.records
|
||||
)
|
||||
|
||||
|
||||
# ── Gate logic ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBackendGate:
|
||||
def test_user_backend_still_gated_by_enabled(self, tmp_path, monkeypatch):
|
||||
"""User-installed ``kind: backend`` plugins still require opt-in —
|
||||
they're not trusted by default."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
_write_plugin(
|
||||
user_plugins,
|
||||
["image_gen", "fancy"],
|
||||
manifest_extra={"kind": "backend"},
|
||||
)
|
||||
# Do NOT opt in.
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
loaded = mgr._plugins["image_gen/fancy"]
|
||||
assert loaded.enabled is False
|
||||
assert "not enabled" in (loaded.error or "")
|
||||
|
||||
def test_user_backend_loads_when_enabled(self, tmp_path, monkeypatch):
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
user_plugins = hermes_home / "plugins"
|
||||
|
||||
_write_plugin(
|
||||
user_plugins,
|
||||
["image_gen", "fancy"],
|
||||
manifest_extra={"kind": "backend"},
|
||||
)
|
||||
_enable(hermes_home, "image_gen/fancy")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr._plugins["image_gen/fancy"].enabled is True
|
||||
|
||||
def test_exclusive_kind_skipped(self, tmp_path, monkeypatch):
|
||||
"""``kind: exclusive`` plugins are recorded but not loaded — the
|
||||
category's own discovery system handles them (memory today)."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
_write_plugin(
|
||||
hermes_home / "plugins",
|
||||
["some-backend"],
|
||||
manifest_extra={"kind": "exclusive"},
|
||||
)
|
||||
_enable(hermes_home, "some-backend")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
loaded = mgr._plugins["some-backend"]
|
||||
assert loaded.enabled is False
|
||||
assert "exclusive" in (loaded.error or "")
|
||||
|
||||
|
||||
# ── Bundled backend auto-load (integration with real bundled plugin) ────────
|
||||
|
||||
|
||||
class TestBundledBackendAutoLoad:
|
||||
def test_bundled_image_gen_openai_autoloads(self, tmp_path, monkeypatch):
|
||||
"""The bundled ``plugins/image_gen/openai/`` plugin loads without
|
||||
any opt-in — it's ``kind: backend`` and shipped in-repo."""
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert "image_gen/openai" in mgr._plugins
|
||||
loaded = mgr._plugins["image_gen/openai"]
|
||||
assert loaded.manifest.source == "bundled"
|
||||
assert loaded.manifest.kind == "backend"
|
||||
assert loaded.enabled is True, f"error: {loaded.error}"
|
||||
|
||||
|
||||
# ── PluginContext.register_image_gen_provider ───────────────────────────────
|
||||
|
||||
|
||||
class TestRegisterImageGenProvider:
|
||||
def test_accepts_valid_provider(self, tmp_path, monkeypatch):
|
||||
from agent import image_gen_registry
|
||||
from agent.image_gen_provider import ImageGenProvider
|
||||
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
||||
class FakeProvider(ImageGenProvider):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "fake-test"
|
||||
|
||||
def generate(self, prompt, aspect_ratio="landscape", **kw):
|
||||
return {"success": True, "image": "test://fake"}
|
||||
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
plugin_dir = _write_plugin(
|
||||
hermes_home / "plugins",
|
||||
["my-img-plugin"],
|
||||
register_body=(
|
||||
"from agent.image_gen_provider import ImageGenProvider\n"
|
||||
" class P(ImageGenProvider):\n"
|
||||
" @property\n"
|
||||
" def name(self): return 'fake-ctx'\n"
|
||||
" def generate(self, prompt, aspect_ratio='landscape', **kw):\n"
|
||||
" return {'success': True, 'image': 'x://y'}\n"
|
||||
" ctx.register_image_gen_provider(P())"
|
||||
),
|
||||
)
|
||||
_enable(hermes_home, "my-img-plugin")
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr._plugins["my-img-plugin"].enabled is True
|
||||
assert image_gen_registry.get_provider("fake-ctx") is not None
|
||||
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
||||
def test_rejects_non_provider(self, tmp_path, monkeypatch, caplog):
|
||||
from agent import image_gen_registry
|
||||
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
||||
import os
|
||||
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
|
||||
_write_plugin(
|
||||
hermes_home / "plugins",
|
||||
["bad-img-plugin"],
|
||||
register_body="ctx.register_image_gen_provider('not a provider')",
|
||||
)
|
||||
_enable(hermes_home, "bad-img-plugin")
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
# Plugin loaded (register returned normally) but nothing was
|
||||
# registered in the provider registry.
|
||||
assert mgr._plugins["bad-img-plugin"].enabled is True
|
||||
assert image_gen_registry.get_provider("not a provider") is None
|
||||
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
|
@ -250,6 +250,73 @@ class TestPluginLoading:
|
|||
|
||||
assert "hermes_plugins.ns_plugin" in sys.modules
|
||||
|
||||
def test_user_memory_plugin_auto_coerced_to_exclusive(self, tmp_path, monkeypatch):
|
||||
"""User-installed memory plugins must NOT be loaded by the general
|
||||
PluginManager — they belong to plugins/memory discovery.
|
||||
|
||||
Regression test for the mempalace crash:
|
||||
'PluginContext' object has no attribute 'register_memory_provider'
|
||||
|
||||
A plugin that calls ``ctx.register_memory_provider`` in its
|
||||
``__init__.py`` should be auto-detected and treated as
|
||||
``kind: exclusive`` so the general loader records the manifest but
|
||||
does not import/register() it. The real activation happens through
|
||||
``plugins/memory/__init__.py`` via ``memory.provider`` config.
|
||||
"""
|
||||
plugins_dir = tmp_path / "hermes_test" / "plugins"
|
||||
plugin_dir = plugins_dir / "mempalace"
|
||||
plugin_dir.mkdir(parents=True)
|
||||
# No explicit `kind:` — the heuristic should kick in.
|
||||
(plugin_dir / "plugin.yaml").write_text(yaml.dump({"name": "mempalace"}))
|
||||
(plugin_dir / "__init__.py").write_text(
|
||||
"class MemPalaceProvider:\n"
|
||||
" pass\n"
|
||||
"def register(ctx):\n"
|
||||
" ctx.register_memory_provider('mempalace', MemPalaceProvider)\n"
|
||||
)
|
||||
# Even if the user explicitly enables it in config, the loader
|
||||
# should still treat it as exclusive and skip general loading.
|
||||
hermes_home = tmp_path / "hermes_test"
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
yaml.safe_dump({"plugins": {"enabled": ["mempalace"]}})
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert "mempalace" in mgr._plugins
|
||||
entry = mgr._plugins["mempalace"]
|
||||
assert entry.manifest.kind == "exclusive", (
|
||||
f"Expected auto-coerced kind='exclusive', got {entry.manifest.kind}"
|
||||
)
|
||||
# Not loaded by general manager (no register() call, no AttributeError).
|
||||
assert not entry.enabled
|
||||
assert entry.module is None
|
||||
assert "exclusive" in (entry.error or "").lower()
|
||||
|
||||
def test_explicit_standalone_kind_not_coerced(self, tmp_path, monkeypatch):
|
||||
"""If a plugin explicitly declares ``kind: standalone`` in its
|
||||
manifest, the memory-provider heuristic must NOT override it —
|
||||
even if the source happens to mention ``MemoryProvider``.
|
||||
"""
|
||||
plugins_dir = tmp_path / "hermes_test" / "plugins"
|
||||
plugin_dir = plugins_dir / "not_memory"
|
||||
plugin_dir.mkdir(parents=True)
|
||||
(plugin_dir / "plugin.yaml").write_text(
|
||||
yaml.dump({"name": "not_memory", "kind": "standalone"})
|
||||
)
|
||||
(plugin_dir / "__init__.py").write_text(
|
||||
"# This plugin inspects MemoryProvider docs but isn't one.\n"
|
||||
"def register(ctx):\n pass\n"
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes_test"))
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr._plugins["not_memory"].manifest.kind == "standalone"
|
||||
|
||||
|
||||
# ── TestPluginHooks ────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -720,6 +787,33 @@ class TestPluginCommands:
|
|||
assert entry["handler"] is handler
|
||||
assert entry["description"] == "My custom command"
|
||||
assert entry["plugin"] == "test-plugin"
|
||||
# args_hint defaults to empty string when not passed.
|
||||
assert entry["args_hint"] == ""
|
||||
|
||||
def test_register_command_with_args_hint(self):
|
||||
"""args_hint is stored and surfaced for gateway-native UI registration."""
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="test-plugin", source="user")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
ctx.register_command(
|
||||
"metricas",
|
||||
lambda a: a,
|
||||
description="Metrics dashboard",
|
||||
args_hint="dias:7 formato:json",
|
||||
)
|
||||
|
||||
entry = mgr._plugin_commands["metricas"]
|
||||
assert entry["args_hint"] == "dias:7 formato:json"
|
||||
|
||||
def test_register_command_args_hint_whitespace_trimmed(self):
|
||||
"""args_hint leading/trailing whitespace is stripped."""
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="test-plugin", source="user")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
ctx.register_command("foo", lambda a: a, args_hint=" <file> ")
|
||||
assert mgr._plugin_commands["foo"]["args_hint"] == "<file>"
|
||||
|
||||
def test_register_command_normalizes_name(self):
|
||||
"""Names are lowercased, stripped, and leading slashes removed."""
|
||||
|
|
|
|||
|
|
@ -1412,3 +1412,90 @@ def test_named_custom_runtime_no_model_when_absent(monkeypatch):
|
|||
|
||||
resolved = rp.resolve_runtime_provider(requested="my-server")
|
||||
assert "model" not in resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GHSA-76xc-57q6-vm5m — Ollama URL substring leak
|
||||
#
|
||||
# Same bug class as the previously-fixed GHSA-xf8p-v2cg-h7h5 (OpenRouter).
|
||||
# _resolve_openrouter_runtime's custom-endpoint branch selects OLLAMA_API_KEY
|
||||
# when the base_url "looks like" ollama.com. Previous implementation used
|
||||
# raw substring match; a custom base_url whose PATH or look-alike host
|
||||
# merely contained "ollama.com" leaked OLLAMA_API_KEY to that endpoint.
|
||||
# Fix: use base_url_host_matches (same helper as the OpenRouter sweep).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestOllamaUrlSubstringLeak:
|
||||
"""Call-site regression tests for the fix in _resolve_openrouter_runtime."""
|
||||
|
||||
def _make_cfg(self, base_url):
|
||||
return {"base_url": base_url, "api_key": "", "provider": "custom"}
|
||||
|
||||
def test_ollama_key_not_leaked_to_path_injection(self, monkeypatch):
|
||||
"""http://127.0.0.1:9000/ollama.com/v1 — attacker endpoint with
|
||||
ollama.com in PATH. Must resolve to OPENAI_API_KEY, not OLLAMA_API_KEY."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oa-secret")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-secret")
|
||||
monkeypatch.setenv("OLLAMA_API_KEY", "ol-SECRET-should-not-leak")
|
||||
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "custom")
|
||||
monkeypatch.setattr(rp, "_get_model_config", lambda: self._make_cfg(
|
||||
"http://127.0.0.1:9000/ollama.com/v1"
|
||||
))
|
||||
monkeypatch.setattr(rp, "load_pool", lambda provider: None)
|
||||
monkeypatch.setattr(rp, "_try_resolve_from_custom_pool", lambda *a, **k: None)
|
||||
|
||||
resolved = rp.resolve_runtime_provider(requested="custom")
|
||||
|
||||
assert "ol-SECRET" not in resolved["api_key"], (
|
||||
"OLLAMA_API_KEY must not be sent to an endpoint whose "
|
||||
"hostname is not ollama.com (GHSA-76xc-57q6-vm5m)"
|
||||
)
|
||||
assert resolved["api_key"] == "oa-secret"
|
||||
|
||||
def test_ollama_key_not_leaked_to_lookalike_host(self, monkeypatch):
|
||||
"""ollama.com.attacker.test — look-alike host. OLLAMA_API_KEY
|
||||
must not be sent."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oa-secret")
|
||||
monkeypatch.setenv("OLLAMA_API_KEY", "ol-SECRET-should-not-leak")
|
||||
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "custom")
|
||||
monkeypatch.setattr(rp, "_get_model_config", lambda: self._make_cfg(
|
||||
"http://ollama.com.attacker.test:9000/v1"
|
||||
))
|
||||
monkeypatch.setattr(rp, "load_pool", lambda provider: None)
|
||||
monkeypatch.setattr(rp, "_try_resolve_from_custom_pool", lambda *a, **k: None)
|
||||
|
||||
resolved = rp.resolve_runtime_provider(requested="custom")
|
||||
|
||||
assert "ol-SECRET" not in resolved["api_key"]
|
||||
assert resolved["api_key"] == "oa-secret"
|
||||
|
||||
def test_ollama_key_sent_to_genuine_ollama_com(self, monkeypatch):
|
||||
"""https://ollama.com/v1 — legit Ollama Cloud. OLLAMA_API_KEY
|
||||
should be used."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oa-secret")
|
||||
monkeypatch.setenv("OLLAMA_API_KEY", "ol-legit-key")
|
||||
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "custom")
|
||||
monkeypatch.setattr(rp, "_get_model_config", lambda: self._make_cfg(
|
||||
"https://ollama.com/v1"
|
||||
))
|
||||
monkeypatch.setattr(rp, "load_pool", lambda provider: None)
|
||||
monkeypatch.setattr(rp, "_try_resolve_from_custom_pool", lambda *a, **k: None)
|
||||
|
||||
resolved = rp.resolve_runtime_provider(requested="custom")
|
||||
|
||||
assert resolved["api_key"] == "ol-legit-key"
|
||||
|
||||
def test_ollama_key_sent_to_ollama_subdomain(self, monkeypatch):
|
||||
"""https://api.ollama.com/v1 — legit subdomain."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oa-secret")
|
||||
monkeypatch.setenv("OLLAMA_API_KEY", "ol-legit-key")
|
||||
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "custom")
|
||||
monkeypatch.setattr(rp, "_get_model_config", lambda: self._make_cfg(
|
||||
"https://api.ollama.com/v1"
|
||||
))
|
||||
monkeypatch.setattr(rp, "load_pool", lambda provider: None)
|
||||
monkeypatch.setattr(rp, "_try_resolve_from_custom_pool", lambda *a, **k: None)
|
||||
|
||||
resolved = rp.resolve_runtime_provider(requested="custom")
|
||||
|
||||
assert resolved["api_key"] == "ol-legit-key"
|
||||
|
|
|
|||
|
|
@ -268,7 +268,6 @@ class TestCliBrandingHelpers:
|
|||
|
||||
def test_prompt_toolkit_style_overrides_cover_tui_classes(self):
|
||||
from hermes_cli.skin_engine import set_active_skin, get_prompt_toolkit_style_overrides
|
||||
|
||||
set_active_skin("ares")
|
||||
overrides = get_prompt_toolkit_style_overrides()
|
||||
required = {
|
||||
|
|
@ -277,6 +276,13 @@ class TestCliBrandingHelpers:
|
|||
"prompt",
|
||||
"prompt-working",
|
||||
"hint",
|
||||
"status-bar",
|
||||
"status-bar-strong",
|
||||
"status-bar-dim",
|
||||
"status-bar-good",
|
||||
"status-bar-warn",
|
||||
"status-bar-bad",
|
||||
"status-bar-critical",
|
||||
"input-rule",
|
||||
"image-badge",
|
||||
"completion-menu",
|
||||
|
|
@ -325,6 +331,15 @@ class TestCliBrandingHelpers:
|
|||
overrides = get_prompt_toolkit_style_overrides()
|
||||
assert overrides["prompt"] == skin.get_color("prompt")
|
||||
assert overrides["input-rule"] == skin.get_color("input_rule")
|
||||
assert overrides["status-bar"] == (
|
||||
f"bg:{skin.get_color('status_bar_bg')} {skin.get_color('status_bar_text')}"
|
||||
)
|
||||
assert overrides["status-bar-strong"] == (
|
||||
f"bg:{skin.get_color('status_bar_bg')} {skin.get_color('status_bar_strong')} bold"
|
||||
)
|
||||
assert overrides["status-bar-critical"] == (
|
||||
f"bg:{skin.get_color('status_bar_bg')} {skin.get_color('status_bar_critical')} bold"
|
||||
)
|
||||
assert overrides["clarify-title"] == f"{skin.get_color('banner_title')} bold"
|
||||
assert overrides["sudo-prompt"] == f"{skin.get_color('ui_error')} bold"
|
||||
assert overrides["approval-title"] == f"{skin.get_color('ui_warn')} bold"
|
||||
|
|
|
|||
|
|
@ -706,6 +706,7 @@ class TestNewEndpoints:
|
|||
assert "skills" in data
|
||||
assert isinstance(data["daily"], list)
|
||||
assert "total_sessions" in data["totals"]
|
||||
assert "total_api_calls" in data["totals"]
|
||||
assert data["skills"] == {
|
||||
"summary": {
|
||||
"total_skill_loads": 0,
|
||||
|
|
|
|||
148
tests/hermes_cli/test_web_server_host_header.py
Normal file
148
tests/hermes_cli/test_web_server_host_header.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
"""Tests for GHSA-ppp5-vxwm-4cf7 — Host-header validation.
|
||||
|
||||
DNS rebinding defence: a victim browser that has the dashboard open
|
||||
could be tricked into fetching from an attacker-controlled hostname
|
||||
that TTL-flips to 127.0.0.1. Same-origin / CORS checks won't help —
|
||||
the browser now treats the attacker origin as same-origin. Validating
|
||||
the Host header at the application layer rejects the attack.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_repo = str(Path(__file__).resolve().parents[1])
|
||||
if _repo not in sys.path:
|
||||
sys.path.insert(0, _repo)
|
||||
|
||||
|
||||
class TestHostHeaderValidator:
|
||||
"""Unit test the _is_accepted_host helper directly — cheaper and
|
||||
more thorough than spinning up the full FastAPI app."""
|
||||
|
||||
def test_loopback_bind_accepts_loopback_names(self):
|
||||
from hermes_cli.web_server import _is_accepted_host
|
||||
|
||||
for bound in ("127.0.0.1", "localhost", "::1"):
|
||||
for host_header in (
|
||||
"127.0.0.1", "127.0.0.1:9119",
|
||||
"localhost", "localhost:9119",
|
||||
"[::1]", "[::1]:9119",
|
||||
):
|
||||
assert _is_accepted_host(host_header, bound), (
|
||||
f"bound={bound} must accept host={host_header}"
|
||||
)
|
||||
|
||||
def test_loopback_bind_rejects_attacker_hostnames(self):
|
||||
"""The core rebinding defence: attacker-controlled hosts that
|
||||
TTL-flip to 127.0.0.1 must be rejected."""
|
||||
from hermes_cli.web_server import _is_accepted_host
|
||||
|
||||
for bound in ("127.0.0.1", "localhost"):
|
||||
for attacker in (
|
||||
"evil.example",
|
||||
"evil.example:9119",
|
||||
"rebind.attacker.test:80",
|
||||
"localhost.attacker.test", # subdomain trick
|
||||
"127.0.0.1.evil.test", # lookalike IP prefix
|
||||
"", # missing Host
|
||||
):
|
||||
assert not _is_accepted_host(attacker, bound), (
|
||||
f"bound={bound} must reject attacker host={attacker!r}"
|
||||
)
|
||||
|
||||
def test_zero_zero_bind_accepts_anything(self):
|
||||
"""0.0.0.0 means operator explicitly opted into all-interfaces
|
||||
(requires --insecure). No Host-layer defence is possible — rely
|
||||
on operator network controls."""
|
||||
from hermes_cli.web_server import _is_accepted_host
|
||||
|
||||
for host in ("10.0.0.5", "evil.example", "my-server.corp.net"):
|
||||
assert _is_accepted_host(host, "0.0.0.0")
|
||||
assert _is_accepted_host(host + ":9119", "0.0.0.0")
|
||||
|
||||
def test_explicit_non_loopback_bind_requires_exact_match(self):
|
||||
"""If the operator bound to a specific non-loopback hostname,
|
||||
the Host header must match exactly."""
|
||||
from hermes_cli.web_server import _is_accepted_host
|
||||
|
||||
assert _is_accepted_host("my-server.corp.net", "my-server.corp.net")
|
||||
assert _is_accepted_host("my-server.corp.net:9119", "my-server.corp.net")
|
||||
# Different host — reject
|
||||
assert not _is_accepted_host("evil.example", "my-server.corp.net")
|
||||
# Loopback — reject (we bound to a specific non-loopback name)
|
||||
assert not _is_accepted_host("localhost", "my-server.corp.net")
|
||||
|
||||
def test_case_insensitive_comparison(self):
|
||||
"""Host headers are case-insensitive per RFC — accept variations."""
|
||||
from hermes_cli.web_server import _is_accepted_host
|
||||
|
||||
assert _is_accepted_host("LOCALHOST", "127.0.0.1")
|
||||
assert _is_accepted_host("LocalHost:9119", "127.0.0.1")
|
||||
|
||||
|
||||
class TestHostHeaderMiddleware:
|
||||
"""End-to-end test via the FastAPI app — verify the middleware
|
||||
rejects bad Host headers with 400."""
|
||||
|
||||
def test_rebinding_request_rejected(self):
|
||||
from fastapi.testclient import TestClient
|
||||
from hermes_cli.web_server import app
|
||||
|
||||
# Simulate start_server having set the bound_host
|
||||
app.state.bound_host = "127.0.0.1"
|
||||
try:
|
||||
client = TestClient(app)
|
||||
# The TestClient sends Host: testserver by default — which is
|
||||
# NOT a loopback alias, so the middleware must reject it.
|
||||
resp = client.get(
|
||||
"/api/status",
|
||||
headers={"Host": "evil.example"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "Invalid Host header" in resp.json()["detail"]
|
||||
finally:
|
||||
# Clean up so other tests don't inherit the bound_host
|
||||
if hasattr(app.state, "bound_host"):
|
||||
del app.state.bound_host
|
||||
|
||||
def test_legit_loopback_request_accepted(self):
|
||||
from fastapi.testclient import TestClient
|
||||
from hermes_cli.web_server import app
|
||||
|
||||
app.state.bound_host = "127.0.0.1"
|
||||
try:
|
||||
client = TestClient(app)
|
||||
# /api/status is in _PUBLIC_API_PATHS — passes auth — so the
|
||||
# only thing that can reject is the host header middleware
|
||||
resp = client.get(
|
||||
"/api/status",
|
||||
headers={"Host": "localhost:9119"},
|
||||
)
|
||||
# Either 200 (endpoint served) or some other non-400 —
|
||||
# just not the host-rejection 400
|
||||
assert resp.status_code != 400 or (
|
||||
"Invalid Host header" not in resp.json().get("detail", "")
|
||||
)
|
||||
finally:
|
||||
if hasattr(app.state, "bound_host"):
|
||||
del app.state.bound_host
|
||||
|
||||
def test_no_bound_host_skips_validation(self):
|
||||
"""If app.state.bound_host isn't set (e.g. running under test
|
||||
infra without calling start_server), middleware must pass through
|
||||
rather than crash."""
|
||||
from fastapi.testclient import TestClient
|
||||
from hermes_cli.web_server import app
|
||||
|
||||
# Make sure bound_host isn't set
|
||||
if hasattr(app.state, "bound_host"):
|
||||
del app.state.bound_host
|
||||
|
||||
client = TestClient(app)
|
||||
resp = client.get("/api/status")
|
||||
# Should get through to the status endpoint, not a 400
|
||||
assert resp.status_code != 400
|
||||
Loading…
Add table
Add a link
Reference in a new issue