mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
fix(moa): contain failed reference details
This commit is contained in:
parent
3ef5229235
commit
ccdf171bcd
10 changed files with 355 additions and 12 deletions
|
|
@ -1171,6 +1171,10 @@ def run_conversation(
|
|||
temperature=_preset_temperature(moa_config, "reference_temperature"),
|
||||
aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"),
|
||||
reference_max_tokens=moa_config.get("reference_max_tokens"),
|
||||
reference_timeout=float(moa_config.get("reference_timeout") or 30.0),
|
||||
degraded_reference_policy=str(
|
||||
moa_config.get("degraded_reference_policy") or "loud"
|
||||
),
|
||||
)
|
||||
if _moa_context:
|
||||
for _msg in reversed(api_messages):
|
||||
|
|
|
|||
|
|
@ -428,8 +428,9 @@ def _run_reference(
|
|||
*,
|
||||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
reference_timeout: float = 30.0,
|
||||
) -> tuple[str, str, Any]:
|
||||
"""Call one reference model and return ``(label, text, usage)``.
|
||||
"""Call one reference model and return ``(label, text, accounting)``.
|
||||
|
||||
The slot is resolved to its provider's real runtime (via ``_slot_runtime``)
|
||||
and called through the same ``call_llm`` request-building path any model
|
||||
|
|
@ -504,6 +505,7 @@ def _run_reference(
|
|||
messages=messages,
|
||||
temperature=temperature,
|
||||
max_tokens=_effective_max_tokens,
|
||||
timeout=reference_timeout,
|
||||
reasoning_config=_slot_reasoning_config(slot),
|
||||
extra_headers=extra_headers,
|
||||
**runtime,
|
||||
|
|
@ -571,6 +573,7 @@ def _run_references_parallel(
|
|||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
progress_callback: Any = None,
|
||||
reference_timeout: float = 30.0,
|
||||
) -> list[tuple[str, str, Any]]:
|
||||
"""Fan out all reference models in parallel, returning outputs in order.
|
||||
|
||||
|
|
@ -586,8 +589,8 @@ def _run_references_parallel(
|
|||
progress like ``MOA: 2/3 refs done``. Best-effort — failures are logged
|
||||
but never break the fan-out (display must never block a turn).
|
||||
|
||||
Each element is ``(label, text, usage)`` where usage is a
|
||||
``CanonicalUsage`` (zeroed for skipped/failed references).
|
||||
Each element is ``(label, text, accounting)`` where accounting is a
|
||||
``_RefAccounting`` object (zeroed for skipped/failed references).
|
||||
"""
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
|
|
@ -621,6 +624,7 @@ def _run_references_parallel(
|
|||
ref_messages,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
reference_timeout=reference_timeout,
|
||||
)
|
||||
] = idx
|
||||
# Collect every reference before returning — the aggregator needs the
|
||||
|
|
@ -862,6 +866,30 @@ def _preset_temperature(preset: dict[str, Any], key: str) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
def _is_failed_reference(text: str) -> bool:
|
||||
"""Return whether a reference output is the internal failure sentinel."""
|
||||
return text.lstrip().lower().startswith("[failed:")
|
||||
|
||||
|
||||
def _successful_references(
|
||||
reference_outputs: list[tuple[str, str, Any]],
|
||||
) -> list[tuple[str, str, Any]]:
|
||||
"""Filter failed advice while preserving each accounting payload."""
|
||||
return [output for output in reference_outputs if not _is_failed_reference(output[1])]
|
||||
|
||||
|
||||
def _failed_reference_labels(
|
||||
reference_outputs: list[tuple[str, str, Any]],
|
||||
) -> list[str]:
|
||||
return [label for label, text, _accounting in reference_outputs if _is_failed_reference(text)]
|
||||
|
||||
|
||||
def _degraded_notice(failed_labels: list[str], policy: str) -> str:
|
||||
if not failed_labels or policy.strip().lower() == "silent":
|
||||
return ""
|
||||
return f"[Reference models unavailable: {', '.join(failed_labels)}]"
|
||||
|
||||
|
||||
def aggregate_moa_context(
|
||||
*,
|
||||
user_prompt: str,
|
||||
|
|
@ -871,6 +899,8 @@ def aggregate_moa_context(
|
|||
temperature: float | None = None,
|
||||
aggregator_temperature: float | None = None,
|
||||
reference_max_tokens: int | None = None,
|
||||
reference_timeout: float = 30.0,
|
||||
degraded_reference_policy: str = "loud",
|
||||
) -> str:
|
||||
"""Run configured reference models and synthesize their advice.
|
||||
|
||||
|
|
@ -899,24 +929,33 @@ def aggregate_moa_context(
|
|||
ref_messages,
|
||||
temperature=temperature,
|
||||
max_tokens=reference_max_tokens,
|
||||
reference_timeout=reference_timeout,
|
||||
)
|
||||
|
||||
successful_outputs = _successful_references(reference_outputs)
|
||||
failed_labels = _failed_reference_labels(reference_outputs)
|
||||
|
||||
# 'full' privacy mode (moa.privacy_filter) also covers this one-shot /moa
|
||||
# synthesis path: advisor text is redacted before it reaches the
|
||||
# synthesizing aggregator. 'display' does not apply here — this path has
|
||||
# no user-visible reference blocks or trace records of its own.
|
||||
# no user-visible reference blocks or trace records of its own. Redaction
|
||||
# runs on the successful outputs only (failed refs are already filtered
|
||||
# into the degraded notice).
|
||||
try:
|
||||
from hermes_cli.config import load_config as _load_config
|
||||
|
||||
if _moa_privacy_mode((_load_config() or {}).get("moa")) == "full":
|
||||
reference_outputs = _redact_reference_outputs(reference_outputs)
|
||||
successful_outputs = _redact_reference_outputs(successful_outputs)
|
||||
except Exception: # pragma: no cover - privacy filter must never break a turn
|
||||
logger.debug("MoA privacy filter check failed", exc_info=True)
|
||||
|
||||
joined = "\n\n".join(
|
||||
f"Reference {idx} — {label}:\n{text}"
|
||||
for idx, (label, text, _usage) in enumerate(reference_outputs, start=1)
|
||||
for idx, (label, text, _accounting) in enumerate(successful_outputs, start=1)
|
||||
)
|
||||
degraded = _degraded_notice(failed_labels, degraded_reference_policy)
|
||||
if degraded:
|
||||
joined = f"{joined}\n\n{degraded}" if joined else degraded
|
||||
synth_prompt = (
|
||||
"You are the aggregator in a Mixture of Agents process. Synthesize the "
|
||||
"reference responses into concise, actionable guidance for the main "
|
||||
|
|
@ -1307,6 +1346,10 @@ class MoAChatCompletions:
|
|||
# explicit values. See _preset_temperature.
|
||||
temperature = _preset_temperature(preset, "reference_temperature")
|
||||
aggregator_temperature = _preset_temperature(preset, "aggregator_temperature")
|
||||
reference_timeout = float(preset.get("reference_timeout") or 30.0)
|
||||
degraded_reference_policy = str(
|
||||
preset.get("degraded_reference_policy") or "loud"
|
||||
)
|
||||
if aggregator_temperature is None and api_kwargs.get("temperature") is not None:
|
||||
# The acting agent's own configured temperature (if any) still
|
||||
# applies to the aggregator, which IS the acting model.
|
||||
|
|
@ -1447,6 +1490,7 @@ class MoAChatCompletions:
|
|||
temperature=temperature,
|
||||
max_tokens=reference_max_tokens,
|
||||
progress_callback=_progress,
|
||||
reference_timeout=reference_timeout,
|
||||
)
|
||||
self._ref_cache_key = _cache_key
|
||||
self._ref_cache_outputs = list(reference_outputs)
|
||||
|
|
@ -1500,7 +1544,7 @@ class MoAChatCompletions:
|
|||
# always happens at the consuming surface, so a mid-session mode
|
||||
# change never leaks or double-redacts).
|
||||
_ref_count = len(reference_outputs)
|
||||
for _idx, (_label, _text, _usage) in enumerate(reference_outputs, start=1):
|
||||
for _idx, (_label, _text, _accounting) in enumerate(reference_outputs, start=1):
|
||||
self._emit(
|
||||
"moa.reference",
|
||||
index=_idx,
|
||||
|
|
@ -1528,21 +1572,31 @@ class MoAChatCompletions:
|
|||
|
||||
guidance: str | None = None
|
||||
agg_messages = [dict(m) for m in messages]
|
||||
if reference_outputs:
|
||||
successful_outputs = _successful_references(reference_outputs)
|
||||
failed_labels = _failed_reference_labels(reference_outputs)
|
||||
joined = ""
|
||||
_agg_refs: list = []
|
||||
if successful_outputs:
|
||||
# 'full' privacy mode: redact the advisor text that reaches the
|
||||
# AGGREGATOR too (issue #59959's literal ask). 'display' leaves
|
||||
# the aggregator input raw so synthesis quality is unaffected.
|
||||
# The redaction is applied to a per-call copy — the cache always
|
||||
# holds raw advisor text (see the emit comment above).
|
||||
# holds raw advisor text (see the emit comment above). Failed
|
||||
# refs are already filtered out; only successful advisor text is
|
||||
# joined (and redacted when requested).
|
||||
_agg_refs = (
|
||||
_redact_reference_outputs(reference_outputs)
|
||||
_redact_reference_outputs(successful_outputs)
|
||||
if privacy_mode == "full"
|
||||
else reference_outputs
|
||||
else successful_outputs
|
||||
)
|
||||
joined = "\n\n".join(
|
||||
f"Reference {idx} — {label}:\n{text}"
|
||||
for idx, (label, text, _usage) in enumerate(_agg_refs, start=1)
|
||||
)
|
||||
degraded = _degraded_notice(failed_labels, degraded_reference_policy)
|
||||
if degraded:
|
||||
joined = f"{joined}\n\n{degraded}" if joined else degraded
|
||||
if joined:
|
||||
guidance = (
|
||||
"[Mixture of Agents reference context]\n"
|
||||
f"Preset: {self.preset_name}\n"
|
||||
|
|
|
|||
|
|
@ -1010,6 +1010,7 @@ export interface MoaConfigResponse {
|
|||
{
|
||||
aggregator: MoaModelSlot
|
||||
aggregator_temperature: number
|
||||
degraded_reference_policy: 'loud' | 'silent'
|
||||
enabled: boolean
|
||||
max_tokens: number
|
||||
reference_models: MoaModelSlot[]
|
||||
|
|
@ -1018,14 +1019,17 @@ export interface MoaConfigResponse {
|
|||
reference_max_tokens?: number | null
|
||||
/** Fan-out cadence (per_iteration | user_turn) — round-tripped. */
|
||||
fanout?: string
|
||||
reference_timeout: number
|
||||
}
|
||||
>
|
||||
aggregator: MoaModelSlot
|
||||
aggregator_temperature: number
|
||||
degraded_reference_policy: 'loud' | 'silent'
|
||||
enabled: boolean
|
||||
max_tokens: number
|
||||
reference_models: MoaModelSlot[]
|
||||
reference_temperature: number
|
||||
reference_timeout: number
|
||||
}
|
||||
|
||||
export interface ModelAssignmentRequest {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -20,6 +21,9 @@ DEFAULT_MOA_AGGREGATOR: dict[str, str] = {
|
|||
"model": "anthropic/claude-opus-4.8",
|
||||
}
|
||||
|
||||
DEFAULT_MOA_REFERENCE_TIMEOUT = 30.0
|
||||
MAX_MOA_REFERENCE_TIMEOUT = 300.0
|
||||
|
||||
|
||||
def _default_reference_models() -> list[dict[str, Any]]:
|
||||
return [{**slot, "enabled": True} for slot in deepcopy(DEFAULT_MOA_REFERENCE_MODELS)]
|
||||
|
|
@ -41,6 +45,25 @@ def _coerce_float_or_none(value: Any) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
def _coerce_reference_timeout(value: Any) -> float:
|
||||
"""Return a finite positive advisor timeout capped at five minutes."""
|
||||
if isinstance(value, bool):
|
||||
return DEFAULT_MOA_REFERENCE_TIMEOUT
|
||||
try:
|
||||
timeout = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_MOA_REFERENCE_TIMEOUT
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
return DEFAULT_MOA_REFERENCE_TIMEOUT
|
||||
return min(timeout, MAX_MOA_REFERENCE_TIMEOUT)
|
||||
|
||||
|
||||
def _coerce_degraded_reference_policy(value: Any) -> str:
|
||||
"""Normalize failed-advisor disclosure policy; unknown values fail loud."""
|
||||
policy = str(value or "loud").strip().lower()
|
||||
return policy if policy in {"loud", "silent"} else "loud"
|
||||
|
||||
|
||||
def _coerce_int(value: Any, default: int) -> int:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
|
|
@ -266,6 +289,8 @@ def _default_preset() -> dict[str, Any]:
|
|||
# matching single-model agent behavior.
|
||||
"reference_temperature": None,
|
||||
"aggregator_temperature": None,
|
||||
"reference_timeout": DEFAULT_MOA_REFERENCE_TIMEOUT,
|
||||
"degraded_reference_policy": "loud",
|
||||
"max_tokens": 4096,
|
||||
"reference_max_tokens": None,
|
||||
"fanout": "per_iteration",
|
||||
|
|
@ -302,6 +327,10 @@ def _normalize_preset(raw: Any) -> dict[str, Any]:
|
|||
"aggregator": aggregator,
|
||||
"reference_temperature": _coerce_float_or_none(raw.get("reference_temperature")),
|
||||
"aggregator_temperature": _coerce_float_or_none(raw.get("aggregator_temperature")),
|
||||
"reference_timeout": _coerce_reference_timeout(raw.get("reference_timeout")),
|
||||
"degraded_reference_policy": _coerce_degraded_reference_policy(
|
||||
raw.get("degraded_reference_policy")
|
||||
),
|
||||
"max_tokens": _coerce_int(raw.get("max_tokens"), 4096),
|
||||
# Optional cap on how much each reference ADVISOR may generate per turn.
|
||||
# None (default) = uncapped: advisors write full-length advice, matching
|
||||
|
|
@ -367,6 +396,8 @@ def normalize_moa_config(raw: Any) -> dict[str, Any]:
|
|||
"aggregator": deepcopy(active["aggregator"]),
|
||||
"reference_temperature": active["reference_temperature"],
|
||||
"aggregator_temperature": active["aggregator_temperature"],
|
||||
"reference_timeout": active["reference_timeout"],
|
||||
"degraded_reference_policy": active["degraded_reference_policy"],
|
||||
"max_tokens": active["max_tokens"],
|
||||
"reference_max_tokens": active.get("reference_max_tokens"),
|
||||
"fanout": active.get("fanout", "per_iteration"),
|
||||
|
|
|
|||
|
|
@ -1370,6 +1370,8 @@ class MoaPresetPayload(BaseModel):
|
|||
# single-model agent behavior.
|
||||
reference_temperature: Optional[float] = None
|
||||
aggregator_temperature: Optional[float] = None
|
||||
reference_timeout: float = 30.0
|
||||
degraded_reference_policy: str = "loud"
|
||||
max_tokens: int = 4096
|
||||
# Newer per-preset knobs (see moa_config._normalize_preset). Optional so
|
||||
# older clients that never send them keep working; declared so clients
|
||||
|
|
@ -1389,6 +1391,8 @@ class MoaConfigPayload(BaseModel):
|
|||
aggregator: MoaModelSlot = MoaModelSlot()
|
||||
reference_temperature: Optional[float] = None
|
||||
aggregator_temperature: Optional[float] = None
|
||||
reference_timeout: float = 30.0
|
||||
degraded_reference_policy: str = "loud"
|
||||
max_tokens: int = 4096
|
||||
reference_max_tokens: Optional[int] = None
|
||||
fanout: Optional[str] = None
|
||||
|
|
@ -6784,6 +6788,8 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
|
|||
"aggregator": _slot_dict(preset.aggregator),
|
||||
"reference_temperature": preset.reference_temperature,
|
||||
"aggregator_temperature": preset.aggregator_temperature,
|
||||
"reference_timeout": preset.reference_timeout,
|
||||
"degraded_reference_policy": preset.degraded_reference_policy,
|
||||
"max_tokens": preset.max_tokens,
|
||||
"reference_max_tokens": preset.reference_max_tokens,
|
||||
"fanout": preset.fanout,
|
||||
|
|
@ -6805,6 +6811,8 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
|
|||
aggregator=body.aggregator,
|
||||
reference_temperature=body.reference_temperature,
|
||||
aggregator_temperature=body.aggregator_temperature,
|
||||
reference_timeout=body.reference_timeout,
|
||||
degraded_reference_policy=body.degraded_reference_policy,
|
||||
max_tokens=body.max_tokens,
|
||||
reference_max_tokens=body.reference_max_tokens,
|
||||
fanout=body.fanout,
|
||||
|
|
@ -6824,7 +6832,6 @@ def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
|
|||
status_code=422,
|
||||
detail="Invalid MoA config: " + "; ".join(problems),
|
||||
)
|
||||
|
||||
normalized = normalize_moa_config(raw)
|
||||
# Merge instead of overwrite so that hand-edited keys not declared
|
||||
# in MoaConfigPayload (e.g. save_traces, trace_dir) survive a GUI
|
||||
|
|
|
|||
|
|
@ -738,3 +738,29 @@ def test_privacy_filter_round_trips_through_normalize():
|
|||
assert normalize_moa_config(once)["privacy_filter"] == "display"
|
||||
full = normalize_moa_config({"privacy_filter": "full"})
|
||||
assert normalize_moa_config(full)["privacy_filter"] == "full"
|
||||
|
||||
|
||||
def test_reference_failure_controls_are_normalized_per_preset_and_flattened():
|
||||
cfg = normalize_moa_config(
|
||||
_preset(reference_timeout="120.5", degraded_reference_policy="silent")
|
||||
)
|
||||
|
||||
preset = cfg["presets"]["p"]
|
||||
assert preset["reference_timeout"] == 120.5
|
||||
assert preset["degraded_reference_policy"] == "silent"
|
||||
assert cfg["reference_timeout"] == 120.5
|
||||
assert cfg["degraded_reference_policy"] == "silent"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "", 0, -1, "bad"])
|
||||
def test_reference_timeout_invalid_values_fall_back_to_default(value):
|
||||
assert resolve_moa_preset(_preset(reference_timeout=value), "p")["reference_timeout"] == 30.0
|
||||
|
||||
|
||||
def test_reference_timeout_is_capped_and_unknown_policy_is_loud():
|
||||
preset = resolve_moa_preset(
|
||||
_preset(reference_timeout=9999, degraded_reference_policy="wat"), "p"
|
||||
)
|
||||
|
||||
assert preset["reference_timeout"] == 300.0
|
||||
assert preset["degraded_reference_policy"] == "loud"
|
||||
|
|
|
|||
|
|
@ -880,6 +880,8 @@ class TestWebServerEndpoints:
|
|||
"aggregator": {"provider": "openrouter", "model": "anthropic/claude-opus-4.8"},
|
||||
"reference_temperature": 0.6,
|
||||
"aggregator_temperature": 0.4,
|
||||
"reference_timeout": 44.5,
|
||||
"degraded_reference_policy": "silent",
|
||||
"max_tokens": 4096,
|
||||
"enabled": True,
|
||||
}
|
||||
|
|
@ -897,6 +899,38 @@ class TestWebServerEndpoints:
|
|||
assert all(s.get("enabled", True) is True for s in saved_refs)
|
||||
agg = cfg["moa"]["aggregator"]
|
||||
assert {"provider": agg["provider"], "model": agg["model"]} == payload["aggregator"]
|
||||
assert cfg["moa"]["reference_timeout"] == 44.5
|
||||
assert cfg["moa"]["degraded_reference_policy"] == "silent"
|
||||
|
||||
def test_put_moa_models_persists_reference_failure_controls_per_preset(self):
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
payload = {
|
||||
"default_preset": "review",
|
||||
"presets": {
|
||||
"review": {
|
||||
"reference_models": [
|
||||
{"provider": "openai-codex", "model": "gpt-5.5"}
|
||||
],
|
||||
"aggregator": {
|
||||
"provider": "openrouter",
|
||||
"model": "anthropic/claude-opus-4.8",
|
||||
},
|
||||
"reference_timeout": 87.5,
|
||||
"degraded_reference_policy": "silent",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
resp = self.client.put("/api/model/moa", json=payload)
|
||||
|
||||
assert resp.status_code == 200
|
||||
saved = load_config()["moa"]["presets"]["review"]
|
||||
assert saved["reference_timeout"] == 87.5
|
||||
assert saved["degraded_reference_policy"] == "silent"
|
||||
returned = self.client.get("/api/model/moa").json()["presets"]["review"]
|
||||
assert returned["reference_timeout"] == 87.5
|
||||
assert returned["degraded_reference_policy"] == "silent"
|
||||
|
||||
def test_put_moa_models_rejects_half_filled_slot_with_422(self):
|
||||
"""#64156: a mid-edit autosave (provider picked, model empty) used to be
|
||||
|
|
|
|||
|
|
@ -1865,3 +1865,180 @@ def test_prepared_aggregator_preserves_reasoning_config(monkeypatch):
|
|||
)
|
||||
|
||||
assert captured["reasoning_config"] == expected_reasoning
|
||||
|
||||
|
||||
|
||||
def test_reference_filtering_preserves_accounting_triples():
|
||||
from agent.moa_loop import (
|
||||
_RefAccounting,
|
||||
_failed_reference_labels,
|
||||
_successful_references,
|
||||
)
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
good_accounting = _RefAccounting(CanonicalUsage(input_tokens=7), 0.07)
|
||||
failed_accounting = _RefAccounting(CanonicalUsage(input_tokens=5), 0.05)
|
||||
outputs = [
|
||||
("good-model", "useful advice", good_accounting),
|
||||
("bad-model", "[failed: raw provider secret]", failed_accounting),
|
||||
]
|
||||
|
||||
successful = _successful_references(outputs)
|
||||
assert successful == [outputs[0]]
|
||||
assert successful[0][2] is good_accounting
|
||||
assert _failed_reference_labels(outputs) == ["bad-model"]
|
||||
|
||||
|
||||
def test_aggregate_moa_context_sanitizes_failed_reference_and_forwards_timeout(monkeypatch):
|
||||
from agent import moa_loop
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
outputs = [
|
||||
("good-model", "useful advice", moa_loop._RefAccounting(CanonicalUsage())),
|
||||
(
|
||||
"bad-model",
|
||||
"[failed: HTTP 401 key=super-secret]",
|
||||
moa_loop._RefAccounting(CanonicalUsage()),
|
||||
),
|
||||
]
|
||||
fanout_kwargs = {}
|
||||
aggregator_calls = []
|
||||
|
||||
def fake_fanout(*args, **kwargs):
|
||||
fanout_kwargs.update(kwargs)
|
||||
return outputs
|
||||
|
||||
def fake_call_llm(**kwargs):
|
||||
aggregator_calls.append(kwargs)
|
||||
return _response("synthesized guidance")
|
||||
|
||||
monkeypatch.setattr(moa_loop, "_run_references_parallel", fake_fanout)
|
||||
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
|
||||
monkeypatch.setattr(
|
||||
moa_loop,
|
||||
"_slot_runtime",
|
||||
lambda slot: {"provider": slot["provider"], "model": slot["model"]},
|
||||
)
|
||||
|
||||
result = moa_loop.aggregate_moa_context(
|
||||
user_prompt="review this",
|
||||
api_messages=[{"role": "user", "content": "review this"}],
|
||||
reference_models=[
|
||||
{"provider": "openrouter", "model": "good-model"},
|
||||
{"provider": "openrouter", "model": "bad-model"},
|
||||
],
|
||||
aggregator={"provider": "openrouter", "model": "aggregator"},
|
||||
reference_timeout=17.5,
|
||||
degraded_reference_policy="loud",
|
||||
)
|
||||
|
||||
assert fanout_kwargs["reference_timeout"] == 17.5
|
||||
private_prompt = aggregator_calls[0]["messages"][0]["content"]
|
||||
assert "useful advice" in private_prompt
|
||||
assert "super-secret" not in private_prompt
|
||||
assert "Reference models unavailable: bad-model" in private_prompt
|
||||
assert "super-secret" not in result
|
||||
|
||||
|
||||
def test_moa_facade_sanitizes_failures_without_breaking_accounting(monkeypatch, tmp_path):
|
||||
from agent import moa_loop
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
(home / "config.yaml").write_text(
|
||||
"""
|
||||
moa:
|
||||
default_preset: review
|
||||
presets:
|
||||
review:
|
||||
reference_timeout: 19
|
||||
degraded_reference_policy: silent
|
||||
reference_models:
|
||||
- provider: openrouter
|
||||
model: good-model
|
||||
- provider: openrouter
|
||||
model: bad-model
|
||||
aggregator:
|
||||
provider: openrouter
|
||||
model: aggregator
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
outputs = [
|
||||
(
|
||||
"good-model",
|
||||
"useful advice",
|
||||
moa_loop._RefAccounting(CanonicalUsage(input_tokens=7), 0.07),
|
||||
),
|
||||
(
|
||||
"bad-model",
|
||||
"[failed: HTTP 401 key=super-secret]",
|
||||
moa_loop._RefAccounting(CanonicalUsage(input_tokens=5), 0.05),
|
||||
),
|
||||
]
|
||||
fanout_kwargs = {}
|
||||
aggregator_calls = []
|
||||
|
||||
def fake_fanout(*args, **kwargs):
|
||||
fanout_kwargs.update(kwargs)
|
||||
return outputs
|
||||
|
||||
def fake_call_llm(**kwargs):
|
||||
aggregator_calls.append(kwargs)
|
||||
return _response("aggregator acted")
|
||||
|
||||
monkeypatch.setattr(moa_loop, "_run_references_parallel", fake_fanout)
|
||||
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
|
||||
monkeypatch.setattr(
|
||||
moa_loop,
|
||||
"_slot_runtime",
|
||||
lambda slot: {"provider": slot["provider"], "model": slot["model"]},
|
||||
)
|
||||
|
||||
facade = moa_loop.MoAChatCompletions("review")
|
||||
facade.create(messages=[{"role": "user", "content": "review this"}], tools=[])
|
||||
|
||||
assert fanout_kwargs["reference_timeout"] == 19.0
|
||||
private_prompt = str(aggregator_calls[0]["messages"])
|
||||
assert "useful advice" in private_prompt
|
||||
assert "super-secret" not in private_prompt
|
||||
assert "Reference models unavailable" not in private_prompt
|
||||
usage, cost = facade.consume_reference_usage()
|
||||
assert usage.input_tokens == 12
|
||||
assert cost == pytest.approx(0.12)
|
||||
assert facade._pending_trace["reference_outputs"] == outputs
|
||||
|
||||
|
||||
def test_run_reference_forwards_configured_timeout(monkeypatch):
|
||||
from agent import moa_loop
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_call_llm(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return _response("advice")
|
||||
|
||||
monkeypatch.setattr(moa_loop, "call_llm", fake_call_llm)
|
||||
monkeypatch.setattr(
|
||||
moa_loop,
|
||||
"_slot_runtime",
|
||||
lambda slot: {"provider": "openrouter", "model": slot["model"]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"agent.usage_pricing.estimate_usage_cost",
|
||||
lambda *args, **kwargs: SimpleNamespace(
|
||||
amount_usd=None, status="unavailable", source=None
|
||||
),
|
||||
)
|
||||
|
||||
label, text, accounting = moa_loop._run_reference(
|
||||
{"provider": "openrouter", "model": "advisor"},
|
||||
[{"role": "user", "content": "review"}],
|
||||
reference_timeout=23.5,
|
||||
)
|
||||
|
||||
assert (label, text) == ("openrouter:advisor", "advice")
|
||||
assert calls[0]["timeout"] == 23.5
|
||||
assert isinstance(accounting, moa_loop._RefAccounting)
|
||||
|
|
|
|||
|
|
@ -2344,6 +2344,8 @@ export interface MoaConfigResponse {
|
|||
aggregator: MoaModelSlot;
|
||||
reference_temperature: number;
|
||||
aggregator_temperature: number;
|
||||
reference_timeout: number;
|
||||
degraded_reference_policy: "loud" | "silent";
|
||||
max_tokens: number;
|
||||
/** Optional advisor output cap — round-tripped, not edited here. */
|
||||
reference_max_tokens?: number | null;
|
||||
|
|
@ -2355,6 +2357,8 @@ export interface MoaConfigResponse {
|
|||
aggregator: MoaModelSlot;
|
||||
reference_temperature: number;
|
||||
aggregator_temperature: number;
|
||||
reference_timeout: number;
|
||||
degraded_reference_policy: "loud" | "silent";
|
||||
max_tokens: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -765,6 +765,8 @@ function MoaModelsModal({
|
|||
aggregator: draft.aggregator,
|
||||
reference_temperature: draft.reference_temperature,
|
||||
aggregator_temperature: draft.aggregator_temperature,
|
||||
reference_timeout: draft.reference_timeout,
|
||||
degraded_reference_policy: draft.degraded_reference_policy,
|
||||
max_tokens: draft.max_tokens,
|
||||
enabled: draft.enabled,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue