mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-06-27 11:22:03 +00:00
41ede84b93
A config migration (or hand-edit) that leaves an invalid toolset name in `platform_toolsets` — e.g. the #38798 corruption that rewrote `hermes-cli` to the non-existent `hermes` — silently disabled all affected tools: resolve_toolset() returns [] for an unknown name, so the agent quietly lost its tools with no error, warning, or log entry and degraded to text-only replies. Surface it loudly at two points: - After migration (migrate_config): validate platform_toolsets and record/print a warning per unknown name, with a `hermes-<platform>` suggestion when that would have been valid (the exact #38798 shape). - At runtime (_get_platform_tools): if a platform was explicitly configured but every toolset name is invalid, log a warning when tools are resolved for a session — so an ALREADY-corrupted config is caught at startup, not only on the next `hermes update`. Logic lives in a new pure, side-effect-free helper (toolset_validation.py) with validate_toolset injected, so it is unit-testable without the tool registry. Note: the original v25→v26 migration that caused the corruption no longer exists (config format is now v30; no migration step rewrites toolset names). This change is the durable defense against the silent-failure mode regardless of cause, matching the issue's "Expected: log a warning". Salvaged from #39207 by @lEWFkRAD (authorship preserved via cherry-pick). Tests: 9 helper cases (incl. the #38798 corruption shape, mixed valid/invalid, zero-tools state, non-dict/scalar/non-string) + a runtime caplog test — both the helper warning and the runtime guard mutation-verified to fail without the fix. Closes #38798. Supersedes #39581 (prevent-in-v25→v26 — that path is gone), #41006 / #40208 (repair-migration for already-corrupted configs).
91 lines
3.5 KiB
Python
91 lines
3.5 KiB
Python
"""Unit tests for hermes_cli.toolset_validation (see #38798).
|
|
|
|
Pure logic — the validity predicate is injected, so these tests need neither the
|
|
tool registry nor a running Hermes.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.toolset_validation import validate_platform_toolsets
|
|
|
|
# A representative set of real toolset names. `hermes` is deliberately absent —
|
|
# that is the corruption #38798 reported (`hermes-cli` rewritten to `hermes`).
|
|
_KNOWN = {
|
|
"hermes-cli",
|
|
"hermes-telegram",
|
|
"hermes-discord",
|
|
"terminal",
|
|
"web",
|
|
}
|
|
|
|
|
|
def _is_valid(name):
|
|
return name in _KNOWN
|
|
|
|
|
|
def test_valid_config_produces_no_warnings():
|
|
cfg = {"cli": ["hermes-cli"], "telegram": ["hermes-telegram"]}
|
|
assert validate_platform_toolsets(cfg, _is_valid) == []
|
|
|
|
|
|
def test_38798_corruption_warns_and_suggests_correct_name():
|
|
# The exact reported shape: cli holds 'hermes' instead of 'hermes-cli'.
|
|
warnings = validate_platform_toolsets({"cli": ["hermes"]}, _is_valid)
|
|
unknown = [w for w in warnings if "unknown toolset 'hermes'" in w]
|
|
assert len(unknown) == 1
|
|
# Actionable: points at the valid name the entry should have been.
|
|
assert "did you mean 'hermes-cli'?" in unknown[0]
|
|
# And the zero-valid-toolsets safety net fires.
|
|
assert any("zero valid toolsets" in w for w in warnings)
|
|
|
|
|
|
def test_mixed_valid_and_invalid_flags_only_the_invalid():
|
|
cfg = {"cli": ["hermes-cli"], "discord": ["bogus"]}
|
|
warnings = validate_platform_toolsets(cfg, _is_valid)
|
|
# One valid entry exists, so no zero-valid warning.
|
|
assert not any("zero valid toolsets" in w for w in warnings)
|
|
assert len(warnings) == 1
|
|
assert "platform 'discord'" in warnings[0]
|
|
assert "unknown toolset 'bogus'" in warnings[0]
|
|
|
|
|
|
def test_unknown_without_valid_platform_default_omits_suggestion():
|
|
# hermes-mystery is not a known toolset, so no "did you mean" hint.
|
|
warnings = validate_platform_toolsets({"mystery": ["nope"]}, _is_valid)
|
|
unknown = [w for w in warnings if "unknown toolset 'nope'" in w]
|
|
assert len(unknown) == 1
|
|
assert "did you mean" not in unknown[0]
|
|
|
|
|
|
@pytest.mark.parametrize("value", [None, {}, [], "hermes-cli", 42])
|
|
def test_non_dict_or_empty_yields_no_warnings(value):
|
|
assert validate_platform_toolsets(value, _is_valid) == []
|
|
|
|
|
|
def test_scalar_toolset_value_is_accepted():
|
|
# Some configs store the toolset as a bare string rather than a list.
|
|
assert validate_platform_toolsets({"cli": "hermes-cli"}, _is_valid) == []
|
|
|
|
|
|
def test_non_string_entries_are_skipped_not_counted_invalid():
|
|
cfg = {"cli": [None, 123, "hermes-cli"]}
|
|
# The junk entries are ignored; the valid one keeps it from being "zero".
|
|
assert validate_platform_toolsets(cfg, _is_valid) == []
|
|
|
|
|
|
def test_all_invalid_reports_each_and_the_zero_state():
|
|
cfg = {"cli": ["hermes"], "discord": ["hermes"]}
|
|
warnings = validate_platform_toolsets(cfg, _is_valid)
|
|
assert sum("unknown toolset" in w for w in warnings) == 2
|
|
assert any("zero valid toolsets" in w for w in warnings)
|
|
|
|
|
|
def test_real_validate_toolset_treats_hermes_cli_valid_and_hermes_invalid():
|
|
# Ties the helper to reality: the canonical registry check agrees that
|
|
# `hermes-cli` is the real toolset and `hermes` is not (the #38798 crux).
|
|
from toolsets import validate_toolset
|
|
|
|
assert validate_toolset("hermes-cli") is True
|
|
assert validate_toolset("hermes") is False
|
|
warnings = validate_platform_toolsets({"cli": ["hermes"]}, validate_toolset)
|
|
assert any("did you mean 'hermes-cli'?" in w for w in warnings)
|