mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-18 14:52:04 +00:00
feat(moa): add reference_max_tokens to cap advisor output and cut turn latency (#56756)
MoA per-turn latency is dominated by advisor GENERATION: turn wall time correlates ~0.88 with output tokens and ~-0.03 with input tokens (measured over 52 turns). Each turn waits for the slowest advisor to finish writing, and advisors were uncapped — writing multi-thousand-token essays the aggregator only needs the gist of. Add an opt-in per-preset reference_max_tokens knob (mirrors reference_temperature) that caps ADVISOR output only; the acting aggregator is never capped. Default None = uncapped, so existing presets are byte-for-byte unchanged (no regression). Wired through both MoA execution paths (MoAChatCompletions.create and aggregate_moa_context). E2E: same task, closed preset uncapped vs reference_max_tokens=600 -> 59s to 33s (~44% faster), final answer identical/correct. - hermes_cli/moa_config.py: _coerce_int_or_none helper + reference_max_tokens in _normalize_preset/_default_preset/flattened view - agent/moa_loop.py: read preset.reference_max_tokens, pass to reference fan-out - agent/conversation_loop.py: pass reference_max_tokens on the per-turn path - tests + docs
This commit is contained in:
parent
9be39de0f2
commit
543d305bbb
5 changed files with 117 additions and 5 deletions
|
|
@ -235,3 +235,46 @@ def test_moa_provider_rejected_case_insensitive():
|
|||
|
||||
assert cfg["presets"]["p"]["aggregator"]["provider"] != "moa"
|
||||
assert cfg["presets"]["p"]["aggregator"] == DEFAULT_MOA_AGGREGATOR
|
||||
|
||||
|
||||
def _preset(**extra):
|
||||
base = {
|
||||
"reference_models": [{"provider": "openrouter", "model": "anthropic/claude-opus-4.8"}],
|
||||
"aggregator": {"provider": "openrouter", "model": "anthropic/claude-opus-4.8"},
|
||||
}
|
||||
base.update(extra)
|
||||
return {"default_preset": "p", "presets": {"p": base}}
|
||||
|
||||
|
||||
def test_reference_max_tokens_defaults_to_none_uncapped():
|
||||
"""Unset reference_max_tokens resolves to None (no cap) so existing presets
|
||||
keep their prior uncapped advisor behavior — no silent regression."""
|
||||
p = resolve_moa_preset(_preset(), "p")
|
||||
assert p["reference_max_tokens"] is None
|
||||
|
||||
|
||||
def test_reference_max_tokens_positive_value_preserved():
|
||||
"""A positive cap flows through resolve_moa_preset to the runtime path."""
|
||||
p = resolve_moa_preset(_preset(reference_max_tokens=600), "p")
|
||||
assert p["reference_max_tokens"] == 600
|
||||
|
||||
|
||||
def test_reference_max_tokens_invalid_falls_back_to_none():
|
||||
"""Non-positive / non-numeric caps degrade to None (uncapped) rather than
|
||||
clamping advisors to a nonsense value or crashing."""
|
||||
for bad in (0, -5, "abc", "", None):
|
||||
p = resolve_moa_preset(_preset(reference_max_tokens=bad), "p")
|
||||
assert p["reference_max_tokens"] is None, bad
|
||||
|
||||
|
||||
def test_reference_max_tokens_string_number_coerced():
|
||||
"""A hand-edited config.yaml string like '600' coerces to int."""
|
||||
p = resolve_moa_preset(_preset(reference_max_tokens="600"), "p")
|
||||
assert p["reference_max_tokens"] == 600
|
||||
|
||||
|
||||
def test_reference_max_tokens_in_flattened_view():
|
||||
"""The flattened compatibility view (dashboard/desktop callers) exposes the
|
||||
active preset's reference_max_tokens."""
|
||||
cfg = normalize_moa_config(_preset(reference_max_tokens=750))
|
||||
assert cfg["reference_max_tokens"] == 750
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue