mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(moa): act aggregator-alone on the facade path when all references fail
Extends the all-references-failed short-circuit (#56975) to the persistent `provider: moa` facade path: MoAChatCompletions.create() previously attached 'use the reference responses below' guidance built entirely from failure sentinels and called the aggregator with it. Now an all-failed turn attaches either the sanitized unavailability notice (loud policy) or nothing (silent policy), and the aggregator — which IS the acting model — simply acts alone. Advisor accounting for the failed fan-out is still recorded. Co-authored-by: liuhao1024 <sunsky.lau@gmail.com>
This commit is contained in:
parent
f0ed77b627
commit
62c2b299a3
2 changed files with 130 additions and 3 deletions
|
|
@ -1627,9 +1627,31 @@ class MoAChatCompletions:
|
|||
for idx, (label, text, _usage) in enumerate(_agg_refs, start=1)
|
||||
)
|
||||
degraded = _degraded_notice(failed_labels, degraded_reference_policy)
|
||||
if degraded:
|
||||
joined = f"{joined}\n\n{degraded}" if joined else degraded
|
||||
if joined:
|
||||
if reference_outputs and not successful_outputs:
|
||||
# Every reference failed or was skipped: don't wrap a wall of
|
||||
# failure sentinels in "use the reference responses below"
|
||||
# guidance — the aggregator IS the acting model, so it simply
|
||||
# acts alone this turn. Under the loud policy it still gets the
|
||||
# sanitized unavailability notice so it can disclose degraded
|
||||
# mode; under silent it gets nothing.
|
||||
logger.warning(
|
||||
"MoA: all %d reference(s) failed — acting aggregator-alone "
|
||||
"without reference guidance",
|
||||
len(reference_outputs),
|
||||
)
|
||||
if degraded:
|
||||
guidance = (
|
||||
"[Mixture of Agents reference context]\n"
|
||||
f"Preset: {self.preset_name}\n"
|
||||
f"Aggregator/acting model: {_slot_label(aggregator)}\n\n"
|
||||
"All reference models failed this turn — no advisory "
|
||||
"guidance is available. Act on your own judgment.\n\n"
|
||||
f"{degraded}"
|
||||
)
|
||||
_attach_reference_guidance(agg_messages, guidance)
|
||||
elif joined or degraded:
|
||||
if degraded:
|
||||
joined = f"{joined}\n\n{degraded}" if joined else degraded
|
||||
guidance = (
|
||||
"[Mixture of Agents reference context]\n"
|
||||
f"Preset: {self.preset_name}\n"
|
||||
|
|
|
|||
|
|
@ -2129,3 +2129,108 @@ def test_aggregate_skips_aggregator_when_all_references_skipped(monkeypatch):
|
|||
|
||||
assert "all reference models failed" in result
|
||||
assert "Reference models unavailable" in result
|
||||
|
||||
|
||||
def _facade_all_failed_fixture(monkeypatch, tmp_path, policy):
|
||||
"""Common scaffolding: a 'review' preset whose references ALL fail."""
|
||||
from agent import moa_loop
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
(home / "config.yaml").write_text(
|
||||
f"""
|
||||
moa:
|
||||
default_preset: review
|
||||
presets:
|
||||
review:
|
||||
degraded_reference_policy: {policy}
|
||||
reference_models:
|
||||
- provider: openrouter
|
||||
model: bad-model-a
|
||||
- provider: openrouter
|
||||
model: bad-model-b
|
||||
aggregator:
|
||||
provider: openrouter
|
||||
model: aggregator
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
outputs = [
|
||||
(
|
||||
"bad-model-a",
|
||||
"[failed: HTTP 401 key=super-secret]",
|
||||
moa_loop._RefAccounting(CanonicalUsage(input_tokens=5), 0.05),
|
||||
),
|
||||
(
|
||||
"bad-model-b",
|
||||
"[failed: timeout after 900s]",
|
||||
moa_loop._RefAccounting(CanonicalUsage(input_tokens=3), 0.03),
|
||||
),
|
||||
]
|
||||
aggregator_calls = []
|
||||
|
||||
def fake_call_llm(**kwargs):
|
||||
aggregator_calls.append(kwargs)
|
||||
return _response("aggregator acted alone")
|
||||
|
||||
monkeypatch.setattr(moa_loop, "_run_references_parallel", lambda *a, **k: outputs)
|
||||
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
|
||||
monkeypatch.setattr(
|
||||
moa_loop,
|
||||
"_slot_runtime",
|
||||
lambda slot: {"provider": slot["provider"], "model": slot["model"]},
|
||||
)
|
||||
return moa_loop, outputs, aggregator_calls
|
||||
|
||||
|
||||
def test_moa_facade_acts_aggregator_alone_when_all_references_fail_loud(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
"""Facade path (MoAChatCompletions.create): when every reference fails,
|
||||
the aggregator acts alone — no 'use the reference responses below'
|
||||
guidance wrapping a wall of failure sentinels. Under the loud policy the
|
||||
sanitized unavailability notice is still disclosed."""
|
||||
moa_loop, outputs, aggregator_calls = _facade_all_failed_fixture(
|
||||
monkeypatch, tmp_path, "loud"
|
||||
)
|
||||
|
||||
facade = moa_loop.MoAChatCompletions("review")
|
||||
response = facade.create(
|
||||
messages=[{"role": "user", "content": "review this"}], tools=[]
|
||||
)
|
||||
|
||||
# The aggregator still acted (it IS the acting model)…
|
||||
assert len(aggregator_calls) == 1
|
||||
prompt = str(aggregator_calls[0]["messages"])
|
||||
# …but got no failure sentinels or raw provider error text…
|
||||
assert "[failed:" not in prompt
|
||||
assert "super-secret" not in prompt
|
||||
assert "Use the reference responses below" not in prompt
|
||||
# …only the sanitized loud-policy notice.
|
||||
assert "Reference models unavailable" in prompt
|
||||
assert "bad-model-a" in prompt
|
||||
# Accounting for the failed fan-out is still folded into the turn.
|
||||
usage, cost = facade.consume_reference_usage()
|
||||
assert usage.input_tokens == 8
|
||||
assert cost == pytest.approx(0.08)
|
||||
assert response.choices[0].message.content == "aggregator acted alone"
|
||||
|
||||
|
||||
def test_moa_facade_acts_aggregator_alone_when_all_references_fail_silent(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
"""Silent policy: all-failed turns attach no reference guidance at all."""
|
||||
moa_loop, _outputs, aggregator_calls = _facade_all_failed_fixture(
|
||||
monkeypatch, tmp_path, "silent"
|
||||
)
|
||||
|
||||
facade = moa_loop.MoAChatCompletions("review")
|
||||
facade.create(messages=[{"role": "user", "content": "review this"}], tools=[])
|
||||
|
||||
assert len(aggregator_calls) == 1
|
||||
prompt = str(aggregator_calls[0]["messages"])
|
||||
assert "[failed:" not in prompt
|
||||
assert "Reference models unavailable" not in prompt
|
||||
assert "Mixture of Agents reference context" not in prompt
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue