mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
183 lines
5.1 KiB
Python
183 lines
5.1 KiB
Python
import pytest
|
|
|
|
from agent.errors import MoAPresetNotFoundError
|
|
from hermes_cli.moa_config import (
|
|
DEFAULT_MOA_AGGREGATOR,
|
|
DEFAULT_MOA_PRESET_NAME,
|
|
DEFAULT_MOA_REFERENCE_MODELS,
|
|
build_moa_turn_prompt,
|
|
decode_moa_turn,
|
|
exact_moa_preset_name,
|
|
normalize_moa_config,
|
|
resolve_moa_preset,
|
|
set_active_moa_preset,
|
|
)
|
|
|
|
|
|
def test_moa_slot_picker_excludes_unconfigured_providers(monkeypatch):
|
|
from hermes_cli import moa_cmd
|
|
|
|
captured = {}
|
|
monkeypatch.setattr(moa_cmd, "load_picker_context", lambda: object())
|
|
|
|
def fake_build(_context, **kwargs):
|
|
captured.update(kwargs)
|
|
return {
|
|
"providers": [
|
|
{"slug": "moa", "models": ["default"]},
|
|
{"slug": "opencode-go", "models": ["deepseek-v4-pro"]},
|
|
]
|
|
}
|
|
|
|
monkeypatch.setattr(moa_cmd, "build_models_payload", fake_build)
|
|
|
|
assert [row["slug"] for row in moa_cmd._model_options()] == ["opencode-go"]
|
|
assert captured["include_unconfigured"] is False
|
|
|
|
|
|
def _enabled_refs(refs):
|
|
return [{**slot, "enabled": True} for slot in refs]
|
|
|
|
|
|
def test_normalize_moa_config_uses_default_named_preset():
|
|
cfg = normalize_moa_config({})
|
|
|
|
assert cfg["default_preset"] == DEFAULT_MOA_PRESET_NAME
|
|
assert list(cfg["presets"]) == [DEFAULT_MOA_PRESET_NAME]
|
|
assert cfg["reference_models"] == _enabled_refs(DEFAULT_MOA_REFERENCE_MODELS)
|
|
assert cfg["aggregator"] == DEFAULT_MOA_AGGREGATOR
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_exact_preset_matching_skips_disabled_presets():
|
|
"""A disabled preset must not match the implicit bare-name switch path.
|
|
|
|
Regression for #55187: with ``enabled: false`` presets, a plain model
|
|
switch whose name collides with a preset key (e.g. ``default``) silently
|
|
pivoted the session onto the MoA virtual provider. The per-preset
|
|
``enabled`` opt-out must gate this implicit match.
|
|
"""
|
|
config = {
|
|
"presets": {
|
|
"default": {"enabled": False},
|
|
"klo": {"enabled": False},
|
|
},
|
|
}
|
|
assert exact_moa_preset_name(config, "default") is None
|
|
assert exact_moa_preset_name(config, "klo") is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_resolve_missing_moa_preset_has_actionable_error():
|
|
cfg = {
|
|
"default_preset": "日常对话-高峰",
|
|
"presets": {"日常对话-高峰": {}, "日常对话-非高峰": {}},
|
|
}
|
|
|
|
with pytest.raises(MoAPresetNotFoundError) as exc_info:
|
|
resolve_moa_preset(cfg, "日常对话-高峰期")
|
|
|
|
message = str(exc_info.value)
|
|
assert "日常对话-高峰期" in message
|
|
assert "日常对话-高峰" in message
|
|
assert "日常对话-非高峰" in message
|
|
assert "hermes moa list" in message
|
|
|
|
|
|
def test_missing_moa_preset_is_non_retryable():
|
|
from agent.error_classifier import FailoverReason, classify_api_error
|
|
|
|
result = classify_api_error(
|
|
MoAPresetNotFoundError("MoA preset 'old' was not found"),
|
|
provider="moa",
|
|
model="old",
|
|
)
|
|
|
|
assert result.reason == FailoverReason.model_not_found
|
|
assert result.retryable is False
|
|
assert result.should_fallback is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _preset(**extra):
|
|
base = {
|
|
"reference_models": [{"provider": "openrouter", "model": "anthropic/claude-opus-4.8"}],
|
|
"aggregator": {"provider": "openrouter", "model": "anthropic/claude-opus-4.8"},
|
|
}
|
|
base.update(extra)
|
|
return {"default_preset": "p", "presets": {"p": base}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── validate_moa_payload (write-boundary validation, #64156) ─────────────────
|
|
#
|
|
# normalize_moa_config is deliberately tolerant at READ time (hand-edited
|
|
# configs degrade to defaults). validate_moa_payload is the strict WRITE-time
|
|
# counterpart: it must flag exactly the payloads normalize would silently
|
|
# repair, so API save paths reject them instead of corrupting user config.
|
|
|
|
|
|
def _valid_preset_payload():
|
|
return {
|
|
"reference_models": [{"provider": "openrouter", "model": "deepseek/deepseek-v4-pro"}],
|
|
"aggregator": {"provider": "openrouter", "model": "anthropic/claude-opus-4.8"},
|
|
}
|
|
|
|
|
|
|
|
|
|
def test_validate_moa_payload_agrees_with_clean_slot():
|
|
"""Contract: a payload validate accepts must survive normalize UNCHANGED in
|
|
its slots — validate and _clean_slot can never disagree (else a payload
|
|
could pass validation and still be swapped for defaults)."""
|
|
from hermes_cli.moa_config import validate_moa_payload
|
|
|
|
payload = {"presets": {"p": _valid_preset_payload()}}
|
|
assert validate_moa_payload(payload) == []
|
|
|
|
cfg = normalize_moa_config(payload)
|
|
# Slots survive with only the canonical enabled=True default added — no
|
|
# provider/model swap, no defaults substitution.
|
|
assert cfg["presets"]["p"]["reference_models"] == _enabled_refs(payload["presets"]["p"]["reference_models"])
|
|
assert cfg["presets"]["p"]["aggregator"] == payload["presets"]["p"]["aggregator"]
|
|
|
|
|
|
# ── Per-slot max_tokens ────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --- fanout cadence normalization (every_n) ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --- privacy_filter normalization ---
|
|
|
|
|
|
|
|
|
|
|
|
|