fix(moa): contain failed reference details

This commit is contained in:
robbyczgw-cla 2026-07-15 08:21:13 +00:00 committed by Teknium
parent 3ef5229235
commit ccdf171bcd
10 changed files with 355 additions and 12 deletions

View file

@ -1171,6 +1171,10 @@ def run_conversation(
temperature=_preset_temperature(moa_config, "reference_temperature"),
aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"),
reference_max_tokens=moa_config.get("reference_max_tokens"),
reference_timeout=float(moa_config.get("reference_timeout") or 30.0),
degraded_reference_policy=str(
moa_config.get("degraded_reference_policy") or "loud"
),
)
if _moa_context:
for _msg in reversed(api_messages):

View file

@ -428,8 +428,9 @@ def _run_reference(
*,
temperature: float | None = None,
max_tokens: int | None = None,
reference_timeout: float = 30.0,
) -> tuple[str, str, Any]:
"""Call one reference model and return ``(label, text, usage)``.
"""Call one reference model and return ``(label, text, accounting)``.
The slot is resolved to its provider's real runtime (via ``_slot_runtime``)
and called through the same ``call_llm`` request-building path any model
@ -504,6 +505,7 @@ def _run_reference(
messages=messages,
temperature=temperature,
max_tokens=_effective_max_tokens,
timeout=reference_timeout,
reasoning_config=_slot_reasoning_config(slot),
extra_headers=extra_headers,
**runtime,
@ -571,6 +573,7 @@ def _run_references_parallel(
temperature: float | None = None,
max_tokens: int | None = None,
progress_callback: Any = None,
reference_timeout: float = 30.0,
) -> list[tuple[str, str, Any]]:
"""Fan out all reference models in parallel, returning outputs in order.
@ -586,8 +589,8 @@ def _run_references_parallel(
progress like ``MOA: 2/3 refs done``. Best-effort failures are logged
but never break the fan-out (display must never block a turn).
Each element is ``(label, text, usage)`` where usage is a
``CanonicalUsage`` (zeroed for skipped/failed references).
Each element is ``(label, text, accounting)`` where accounting is a
``_RefAccounting`` object (zeroed for skipped/failed references).
"""
from agent.usage_pricing import CanonicalUsage
@ -621,6 +624,7 @@ def _run_references_parallel(
ref_messages,
temperature=temperature,
max_tokens=max_tokens,
reference_timeout=reference_timeout,
)
] = idx
# Collect every reference before returning — the aggregator needs the
@ -862,6 +866,30 @@ def _preset_temperature(preset: dict[str, Any], key: str) -> float | None:
return None
def _is_failed_reference(text: str) -> bool:
"""Return whether a reference output is the internal failure sentinel."""
return text.lstrip().lower().startswith("[failed:")
def _successful_references(
reference_outputs: list[tuple[str, str, Any]],
) -> list[tuple[str, str, Any]]:
"""Filter failed advice while preserving each accounting payload."""
return [output for output in reference_outputs if not _is_failed_reference(output[1])]
def _failed_reference_labels(
reference_outputs: list[tuple[str, str, Any]],
) -> list[str]:
return [label for label, text, _accounting in reference_outputs if _is_failed_reference(text)]
def _degraded_notice(failed_labels: list[str], policy: str) -> str:
if not failed_labels or policy.strip().lower() == "silent":
return ""
return f"[Reference models unavailable: {', '.join(failed_labels)}]"
def aggregate_moa_context(
*,
user_prompt: str,
@ -871,6 +899,8 @@ def aggregate_moa_context(
temperature: float | None = None,
aggregator_temperature: float | None = None,
reference_max_tokens: int | None = None,
reference_timeout: float = 30.0,
degraded_reference_policy: str = "loud",
) -> str:
"""Run configured reference models and synthesize their advice.
@ -899,24 +929,33 @@ def aggregate_moa_context(
ref_messages,
temperature=temperature,
max_tokens=reference_max_tokens,
reference_timeout=reference_timeout,
)
successful_outputs = _successful_references(reference_outputs)
failed_labels = _failed_reference_labels(reference_outputs)
# 'full' privacy mode (moa.privacy_filter) also covers this one-shot /moa
# synthesis path: advisor text is redacted before it reaches the
# synthesizing aggregator. 'display' does not apply here — this path has
# no user-visible reference blocks or trace records of its own.
# no user-visible reference blocks or trace records of its own. Redaction
# runs on the successful outputs only (failed refs are already filtered
# into the degraded notice).
try:
from hermes_cli.config import load_config as _load_config
if _moa_privacy_mode((_load_config() or {}).get("moa")) == "full":
reference_outputs = _redact_reference_outputs(reference_outputs)
successful_outputs = _redact_reference_outputs(successful_outputs)
except Exception: # pragma: no cover - privacy filter must never break a turn
logger.debug("MoA privacy filter check failed", exc_info=True)
joined = "\n\n".join(
f"Reference {idx}{label}:\n{text}"
for idx, (label, text, _usage) in enumerate(reference_outputs, start=1)
for idx, (label, text, _accounting) in enumerate(successful_outputs, start=1)
)
degraded = _degraded_notice(failed_labels, degraded_reference_policy)
if degraded:
joined = f"{joined}\n\n{degraded}" if joined else degraded
synth_prompt = (
"You are the aggregator in a Mixture of Agents process. Synthesize the "
"reference responses into concise, actionable guidance for the main "
@ -1307,6 +1346,10 @@ class MoAChatCompletions:
# explicit values. See _preset_temperature.
temperature = _preset_temperature(preset, "reference_temperature")
aggregator_temperature = _preset_temperature(preset, "aggregator_temperature")
reference_timeout = float(preset.get("reference_timeout") or 30.0)
degraded_reference_policy = str(
preset.get("degraded_reference_policy") or "loud"
)
if aggregator_temperature is None and api_kwargs.get("temperature") is not None:
# The acting agent's own configured temperature (if any) still
# applies to the aggregator, which IS the acting model.
@ -1447,6 +1490,7 @@ class MoAChatCompletions:
temperature=temperature,
max_tokens=reference_max_tokens,
progress_callback=_progress,
reference_timeout=reference_timeout,
)
self._ref_cache_key = _cache_key
self._ref_cache_outputs = list(reference_outputs)
@ -1500,7 +1544,7 @@ class MoAChatCompletions:
# always happens at the consuming surface, so a mid-session mode
# change never leaks or double-redacts).
_ref_count = len(reference_outputs)
for _idx, (_label, _text, _usage) in enumerate(reference_outputs, start=1):
for _idx, (_label, _text, _accounting) in enumerate(reference_outputs, start=1):
self._emit(
"moa.reference",
index=_idx,
@ -1528,21 +1572,31 @@ class MoAChatCompletions:
guidance: str | None = None
agg_messages = [dict(m) for m in messages]
if reference_outputs:
successful_outputs = _successful_references(reference_outputs)
failed_labels = _failed_reference_labels(reference_outputs)
joined = ""
_agg_refs: list = []
if successful_outputs:
# 'full' privacy mode: redact the advisor text that reaches the
# AGGREGATOR too (issue #59959's literal ask). 'display' leaves
# the aggregator input raw so synthesis quality is unaffected.
# The redaction is applied to a per-call copy — the cache always
# holds raw advisor text (see the emit comment above).
# holds raw advisor text (see the emit comment above). Failed
# refs are already filtered out; only successful advisor text is
# joined (and redacted when requested).
_agg_refs = (
_redact_reference_outputs(reference_outputs)
_redact_reference_outputs(successful_outputs)
if privacy_mode == "full"
else reference_outputs
else successful_outputs
)
joined = "\n\n".join(
f"Reference {idx}{label}:\n{text}"
for idx, (label, text, _usage) in enumerate(_agg_refs, start=1)
)
degraded = _degraded_notice(failed_labels, degraded_reference_policy)
if degraded:
joined = f"{joined}\n\n{degraded}" if joined else degraded
if joined:
guidance = (
"[Mixture of Agents reference context]\n"
f"Preset: {self.preset_name}\n"

View file

@ -1010,6 +1010,7 @@ export interface MoaConfigResponse {
{
aggregator: MoaModelSlot
aggregator_temperature: number
degraded_reference_policy: 'loud' | 'silent'
enabled: boolean
max_tokens: number
reference_models: MoaModelSlot[]
@ -1018,14 +1019,17 @@ export interface MoaConfigResponse {
reference_max_tokens?: number | null
/** Fan-out cadence (per_iteration | user_turn) — round-tripped. */
fanout?: string
reference_timeout: number
}
>
aggregator: MoaModelSlot
aggregator_temperature: number
degraded_reference_policy: 'loud' | 'silent'
enabled: boolean
max_tokens: number
reference_models: MoaModelSlot[]
reference_temperature: number
reference_timeout: number
}
export interface ModelAssignmentRequest {

View file

@ -4,6 +4,7 @@ from __future__ import annotations
import base64
import json
import math
from copy import deepcopy
from typing import Any
@ -20,6 +21,9 @@ DEFAULT_MOA_AGGREGATOR: dict[str, str] = {
"model": "anthropic/claude-opus-4.8",
}
DEFAULT_MOA_REFERENCE_TIMEOUT = 30.0
MAX_MOA_REFERENCE_TIMEOUT = 300.0
def _default_reference_models() -> list[dict[str, Any]]:
return [{**slot, "enabled": True} for slot in deepcopy(DEFAULT_MOA_REFERENCE_MODELS)]
@ -41,6 +45,25 @@ def _coerce_float_or_none(value: Any) -> float | None:
return None
def _coerce_reference_timeout(value: Any) -> float:
"""Return a finite positive advisor timeout capped at five minutes."""
if isinstance(value, bool):
return DEFAULT_MOA_REFERENCE_TIMEOUT
try:
timeout = float(value)
except (TypeError, ValueError):
return DEFAULT_MOA_REFERENCE_TIMEOUT
if not math.isfinite(timeout) or timeout <= 0:
return DEFAULT_MOA_REFERENCE_TIMEOUT
return min(timeout, MAX_MOA_REFERENCE_TIMEOUT)
def _coerce_degraded_reference_policy(value: Any) -> str:
"""Normalize failed-advisor disclosure policy; unknown values fail loud."""
policy = str(value or "loud").strip().lower()
return policy if policy in {"loud", "silent"} else "loud"
def _coerce_int(value: Any, default: int) -> int:
if value is None or value == "":
return default
@ -266,6 +289,8 @@ def _default_preset() -> dict[str, Any]:
# matching single-model agent behavior.
"reference_temperature": None,
"aggregator_temperature": None,
"reference_timeout": DEFAULT_MOA_REFERENCE_TIMEOUT,
"degraded_reference_policy": "loud",
"max_tokens": 4096,
"reference_max_tokens": None,
"fanout": "per_iteration",
@ -302,6 +327,10 @@ def _normalize_preset(raw: Any) -> dict[str, Any]:
"aggregator": aggregator,
"reference_temperature": _coerce_float_or_none(raw.get("reference_temperature")),
"aggregator_temperature": _coerce_float_or_none(raw.get("aggregator_temperature")),
"reference_timeout": _coerce_reference_timeout(raw.get("reference_timeout")),
"degraded_reference_policy": _coerce_degraded_reference_policy(
raw.get("degraded_reference_policy")
),
"max_tokens": _coerce_int(raw.get("max_tokens"), 4096),
# Optional cap on how much each reference ADVISOR may generate per turn.
# None (default) = uncapped: advisors write full-length advice, matching
@ -367,6 +396,8 @@ def normalize_moa_config(raw: Any) -> dict[str, Any]:
"aggregator": deepcopy(active["aggregator"]),
"reference_temperature": active["reference_temperature"],
"aggregator_temperature": active["aggregator_temperature"],
"reference_timeout": active["reference_timeout"],
"degraded_reference_policy": active["degraded_reference_policy"],
"max_tokens": active["max_tokens"],
"reference_max_tokens": active.get("reference_max_tokens"),
"fanout": active.get("fanout", "per_iteration"),

View file

@ -1370,6 +1370,8 @@ class MoaPresetPayload(BaseModel):
# single-model agent behavior.
reference_temperature: Optional[float] = None
aggregator_temperature: Optional[float] = None
reference_timeout: float = 30.0
degraded_reference_policy: str = "loud"
max_tokens: int = 4096
# Newer per-preset knobs (see moa_config._normalize_preset). Optional so
# older clients that never send them keep working; declared so clients
@ -1389,6 +1391,8 @@ class MoaConfigPayload(BaseModel):
aggregator: MoaModelSlot = MoaModelSlot()
reference_temperature: Optional[float] = None
aggregator_temperature: Optional[float] = None
reference_timeout: float = 30.0
degraded_reference_policy: str = "loud"
max_tokens: int = 4096
reference_max_tokens: Optional[int] = None
fanout: Optional[str] = None
@ -6784,6 +6788,8 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
"aggregator": _slot_dict(preset.aggregator),
"reference_temperature": preset.reference_temperature,
"aggregator_temperature": preset.aggregator_temperature,
"reference_timeout": preset.reference_timeout,
"degraded_reference_policy": preset.degraded_reference_policy,
"max_tokens": preset.max_tokens,
"reference_max_tokens": preset.reference_max_tokens,
"fanout": preset.fanout,
@ -6805,6 +6811,8 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
aggregator=body.aggregator,
reference_temperature=body.reference_temperature,
aggregator_temperature=body.aggregator_temperature,
reference_timeout=body.reference_timeout,
degraded_reference_policy=body.degraded_reference_policy,
max_tokens=body.max_tokens,
reference_max_tokens=body.reference_max_tokens,
fanout=body.fanout,
@ -6824,7 +6832,6 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
status_code=422,
detail="Invalid MoA config: " + "; ".join(problems),
)
normalized = normalize_moa_config(raw)
# Merge instead of overwrite so that hand-edited keys not declared
# in MoaConfigPayload (e.g. save_traces, trace_dir) survive a GUI

View file

@ -738,3 +738,29 @@ def test_privacy_filter_round_trips_through_normalize():
assert normalize_moa_config(once)["privacy_filter"] == "display"
full = normalize_moa_config({"privacy_filter": "full"})
assert normalize_moa_config(full)["privacy_filter"] == "full"
def test_reference_failure_controls_are_normalized_per_preset_and_flattened():
cfg = normalize_moa_config(
_preset(reference_timeout="120.5", degraded_reference_policy="silent")
)
preset = cfg["presets"]["p"]
assert preset["reference_timeout"] == 120.5
assert preset["degraded_reference_policy"] == "silent"
assert cfg["reference_timeout"] == 120.5
assert cfg["degraded_reference_policy"] == "silent"
@pytest.mark.parametrize("value", [None, "", 0, -1, "bad"])
def test_reference_timeout_invalid_values_fall_back_to_default(value):
assert resolve_moa_preset(_preset(reference_timeout=value), "p")["reference_timeout"] == 30.0
def test_reference_timeout_is_capped_and_unknown_policy_is_loud():
preset = resolve_moa_preset(
_preset(reference_timeout=9999, degraded_reference_policy="wat"), "p"
)
assert preset["reference_timeout"] == 300.0
assert preset["degraded_reference_policy"] == "loud"

View file

@ -880,6 +880,8 @@ class TestWebServerEndpoints:
"aggregator": {"provider": "openrouter", "model": "anthropic/claude-opus-4.8"},
"reference_temperature": 0.6,
"aggregator_temperature": 0.4,
"reference_timeout": 44.5,
"degraded_reference_policy": "silent",
"max_tokens": 4096,
"enabled": True,
}
@ -897,6 +899,38 @@ class TestWebServerEndpoints:
assert all(s.get("enabled", True) is True for s in saved_refs)
agg = cfg["moa"]["aggregator"]
assert {"provider": agg["provider"], "model": agg["model"]} == payload["aggregator"]
assert cfg["moa"]["reference_timeout"] == 44.5
assert cfg["moa"]["degraded_reference_policy"] == "silent"
def test_put_moa_models_persists_reference_failure_controls_per_preset(self):
from hermes_cli.config import load_config
payload = {
"default_preset": "review",
"presets": {
"review": {
"reference_models": [
{"provider": "openai-codex", "model": "gpt-5.5"}
],
"aggregator": {
"provider": "openrouter",
"model": "anthropic/claude-opus-4.8",
},
"reference_timeout": 87.5,
"degraded_reference_policy": "silent",
}
},
}
resp = self.client.put("/api/model/moa", json=payload)
assert resp.status_code == 200
saved = load_config()["moa"]["presets"]["review"]
assert saved["reference_timeout"] == 87.5
assert saved["degraded_reference_policy"] == "silent"
returned = self.client.get("/api/model/moa").json()["presets"]["review"]
assert returned["reference_timeout"] == 87.5
assert returned["degraded_reference_policy"] == "silent"
def test_put_moa_models_rejects_half_filled_slot_with_422(self):
"""#64156: a mid-edit autosave (provider picked, model empty) used to be

View file

@ -1865,3 +1865,180 @@ def test_prepared_aggregator_preserves_reasoning_config(monkeypatch):
)
assert captured["reasoning_config"] == expected_reasoning
def test_reference_filtering_preserves_accounting_triples():
from agent.moa_loop import (
_RefAccounting,
_failed_reference_labels,
_successful_references,
)
from agent.usage_pricing import CanonicalUsage
good_accounting = _RefAccounting(CanonicalUsage(input_tokens=7), 0.07)
failed_accounting = _RefAccounting(CanonicalUsage(input_tokens=5), 0.05)
outputs = [
("good-model", "useful advice", good_accounting),
("bad-model", "[failed: raw provider secret]", failed_accounting),
]
successful = _successful_references(outputs)
assert successful == [outputs[0]]
assert successful[0][2] is good_accounting
assert _failed_reference_labels(outputs) == ["bad-model"]
def test_aggregate_moa_context_sanitizes_failed_reference_and_forwards_timeout(monkeypatch):
from agent import moa_loop
from agent.usage_pricing import CanonicalUsage
outputs = [
("good-model", "useful advice", moa_loop._RefAccounting(CanonicalUsage())),
(
"bad-model",
"[failed: HTTP 401 key=super-secret]",
moa_loop._RefAccounting(CanonicalUsage()),
),
]
fanout_kwargs = {}
aggregator_calls = []
def fake_fanout(*args, **kwargs):
fanout_kwargs.update(kwargs)
return outputs
def fake_call_llm(**kwargs):
aggregator_calls.append(kwargs)
return _response("synthesized guidance")
monkeypatch.setattr(moa_loop, "_run_references_parallel", fake_fanout)
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
monkeypatch.setattr(
moa_loop,
"_slot_runtime",
lambda slot: {"provider": slot["provider"], "model": slot["model"]},
)
result = moa_loop.aggregate_moa_context(
user_prompt="review this",
api_messages=[{"role": "user", "content": "review this"}],
reference_models=[
{"provider": "openrouter", "model": "good-model"},
{"provider": "openrouter", "model": "bad-model"},
],
aggregator={"provider": "openrouter", "model": "aggregator"},
reference_timeout=17.5,
degraded_reference_policy="loud",
)
assert fanout_kwargs["reference_timeout"] == 17.5
private_prompt = aggregator_calls[0]["messages"][0]["content"]
assert "useful advice" in private_prompt
assert "super-secret" not in private_prompt
assert "Reference models unavailable: bad-model" in private_prompt
assert "super-secret" not in result
def test_moa_facade_sanitizes_failures_without_breaking_accounting(monkeypatch, tmp_path):
from agent import moa_loop
from agent.usage_pricing import CanonicalUsage
home = tmp_path / ".hermes"
home.mkdir()
(home / "config.yaml").write_text(
"""
moa:
default_preset: review
presets:
review:
reference_timeout: 19
degraded_reference_policy: silent
reference_models:
- provider: openrouter
model: good-model
- provider: openrouter
model: bad-model
aggregator:
provider: openrouter
model: aggregator
""".strip(),
encoding="utf-8",
)
monkeypatch.setenv("HERMES_HOME", str(home))
outputs = [
(
"good-model",
"useful advice",
moa_loop._RefAccounting(CanonicalUsage(input_tokens=7), 0.07),
),
(
"bad-model",
"[failed: HTTP 401 key=super-secret]",
moa_loop._RefAccounting(CanonicalUsage(input_tokens=5), 0.05),
),
]
fanout_kwargs = {}
aggregator_calls = []
def fake_fanout(*args, **kwargs):
fanout_kwargs.update(kwargs)
return outputs
def fake_call_llm(**kwargs):
aggregator_calls.append(kwargs)
return _response("aggregator acted")
monkeypatch.setattr(moa_loop, "_run_references_parallel", fake_fanout)
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
monkeypatch.setattr(
moa_loop,
"_slot_runtime",
lambda slot: {"provider": slot["provider"], "model": slot["model"]},
)
facade = moa_loop.MoAChatCompletions("review")
facade.create(messages=[{"role": "user", "content": "review this"}], tools=[])
assert fanout_kwargs["reference_timeout"] == 19.0
private_prompt = str(aggregator_calls[0]["messages"])
assert "useful advice" in private_prompt
assert "super-secret" not in private_prompt
assert "Reference models unavailable" not in private_prompt
usage, cost = facade.consume_reference_usage()
assert usage.input_tokens == 12
assert cost == pytest.approx(0.12)
assert facade._pending_trace["reference_outputs"] == outputs
def test_run_reference_forwards_configured_timeout(monkeypatch):
from agent import moa_loop
calls = []
def fake_call_llm(**kwargs):
calls.append(kwargs)
return _response("advice")
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
monkeypatch.setattr(
moa_loop,
"_slot_runtime",
lambda slot: {"provider": "openrouter", "model": slot["model"]},
)
monkeypatch.setattr(
"agent.usage_pricing.estimate_usage_cost",
lambda *args, **kwargs: SimpleNamespace(
amount_usd=None, status="unavailable", source=None
),
)
label, text, accounting = moa_loop._run_reference(
{"provider": "openrouter", "model": "advisor"},
[{"role": "user", "content": "review"}],
reference_timeout=23.5,
)
assert (label, text) == ("openrouter:advisor", "advice")
assert calls[0]["timeout"] == 23.5
assert isinstance(accounting, moa_loop._RefAccounting)

View file

@ -2344,6 +2344,8 @@ export interface MoaConfigResponse {
aggregator: MoaModelSlot;
reference_temperature: number;
aggregator_temperature: number;
reference_timeout: number;
degraded_reference_policy: "loud" | "silent";
max_tokens: number;
/** Optional advisor output cap — round-tripped, not edited here. */
reference_max_tokens?: number | null;
@ -2355,6 +2357,8 @@ export interface MoaConfigResponse {
aggregator: MoaModelSlot;
reference_temperature: number;
aggregator_temperature: number;
reference_timeout: number;
degraded_reference_policy: "loud" | "silent";
max_tokens: number;
enabled: boolean;
}

View file

@ -765,6 +765,8 @@ function MoaModelsModal({
aggregator: draft.aggregator,
reference_temperature: draft.reference_temperature,
aggregator_temperature: draft.aggregator_temperature,
reference_timeout: draft.reference_timeout,
degraded_reference_policy: draft.degraded_reference_policy,
max_tokens: draft.max_tokens,
enabled: draft.enabled,
};