From 033f6ea4e7a2390a756a4421b09ef7b0587eff6d Mon Sep 17 00:00:00 2001 From: 2001Y <15167896+2001Y@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:58:32 +0900 Subject: [PATCH] feat(moa): support reference role prompts --- agent/moa_loop.py | 22 +++- hermes_cli/moa_cmd.py | 12 +- hermes_cli/moa_config.py | 11 +- hermes_cli/web_server.py | 4 + tests/hermes_cli/test_moa_cmd.py | 59 +++++++++ tests/hermes_cli/test_moa_config.py | 49 +++++++ tests/hermes_cli/test_web_server.py | 34 +++++ tests/run_agent/test_moa_loop_mode.py | 121 ++++++++++++++++++ .../user-guide/features/mixture-of-agents.md | 13 ++ 9 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 tests/hermes_cli/test_moa_cmd.py diff --git a/agent/moa_loop.py b/agent/moa_loop.py index fcc76c2cf0f..a92d03aea02 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -69,6 +69,17 @@ def _slot_label(slot: dict[str, str]) -> str: return f"{slot.get('provider', '').strip()}:{slot.get('model', '').strip()}" +def _reference_system_prompt(slot: dict[str, str]) -> str: + role_prompt = str(slot.get("role_prompt") or "").strip() + if not role_prompt: + return _REFERENCE_SYSTEM_PROMPT + return ( + f"{_REFERENCE_SYSTEM_PROMPT}\n\n" + "Reference-specific role instruction for this model:\n" + f"{role_prompt}" + ) + + def _slot_runtime(slot: dict[str, str]) -> dict[str, Any]: """Resolve a reference/aggregator slot to real runtime call kwargs. @@ -145,7 +156,7 @@ def _run_reference( # it is analyzing state for an aggregator, not acting on the task. The # trimmed view (_reference_messages) already strips the agent's own # system prompt, so this is the only system message the reference sees. - messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] + messages = [{"role": "system", "content": _reference_system_prompt(slot)}, *ref_messages] response = call_llm( task="moa_reference", messages=messages, @@ -506,7 +517,14 @@ class MoAChatCompletions: f"{m.get('role')}:{m.get('content')}" for m in ref_messages ).encode("utf-8", "replace") ).hexdigest() - _cache_key = (self.preset_name, _sig, tuple(_slot_label(s) for s in reference_models)) + _cache_key = ( + self.preset_name, + _sig, + tuple( + (_slot_label(s), str(s.get("role_prompt") or "").strip()) + for s in reference_models + ), + ) _refs_from_cache = _cache_key == self._ref_cache_key and bool(self._ref_cache_outputs) if _refs_from_cache: diff --git a/hermes_cli/moa_cmd.py b/hermes_cli/moa_cmd.py index 938d6baeacb..e5724321a01 100644 --- a/hermes_cli/moa_cmd.py +++ b/hermes_cli/moa_cmd.py @@ -57,7 +57,14 @@ def _pick_slot(current: dict[str, str] | None = None) -> dict[str, str]: current_model = (current or {}).get("model", "") model_default = models.index(current_model) if current_model in models else 0 model = models[_prompt_choice(f"Select model for {provider.get('slug')}", models, model_default)] - return {"provider": str(provider.get("slug") or ""), "model": str(model)} + slot = {"provider": str(provider.get("slug") or ""), "model": str(model)} + # The interactive picker edits only provider/model today. Preserve an + # existing per-reference role prompt so a quick model re-save does not erase + # richer config.yaml/dashboard-authored MoA roles. + role_prompt = str((current or {}).get("role_prompt") or "").strip() + if role_prompt: + slot["role_prompt"] = role_prompt + return slot def _print_config(config: dict[str, Any]) -> None: @@ -72,6 +79,9 @@ def _print_config(config: dict[str, Any]) -> None: print(" Reference models:") for idx, slot in enumerate(preset["reference_models"], start=1): print(f" {idx}. {slot['provider']}:{slot['model']}") + role_prompt = str(slot.get("role_prompt") or "").strip() + if role_prompt: + print(f" role: {role_prompt}") agg = preset["aggregator"] print(f" Aggregator: {agg['provider']}:{agg['model']}") diff --git a/hermes_cli/moa_config.py b/hermes_cli/moa_config.py index b4e2619176a..b892ad2e909 100644 --- a/hermes_cli/moa_config.py +++ b/hermes_cli/moa_config.py @@ -42,7 +42,7 @@ def _coerce_int(value: Any, default: int) -> int: return default -def _clean_slot(slot: Any) -> dict[str, str] | None: +def _clean_slot(slot: Any, *, preserve_role_prompt: bool = False) -> dict[str, str] | None: if not isinstance(slot, dict): return None provider = str(slot.get("provider") or "").strip() @@ -56,7 +56,12 @@ def _clean_slot(slot: Any) -> dict[str, str] | None: # an invalid slot is dropped, falling back to the preset's defaults. if provider.lower() == "moa": return None - return {"provider": provider, "model": model} + clean = {"provider": provider, "model": model} + if preserve_role_prompt: + role_prompt = str(slot.get("role_prompt") or "").strip() + if role_prompt: + clean["role_prompt"] = role_prompt + return clean def _default_preset() -> dict[str, Any]: @@ -80,7 +85,7 @@ def _normalize_preset(raw: Any) -> dict[str, Any]: # defaults instead of crashing the iteration, mirroring the tolerance # for the scalar fields below (reference_temperature / max_tokens). raw_refs = [raw_refs] if isinstance(raw_refs, dict) else [] - refs = [_clean_slot(item) for item in raw_refs] + refs = [_clean_slot(item, preserve_role_prompt=True) for item in raw_refs] refs = [item for item in refs if item is not None] if not refs: refs = deepcopy(DEFAULT_MOA_REFERENCE_MODELS) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index a76cf42fc2e..9f536a09dec 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -868,6 +868,10 @@ class ModelAssignment(BaseModel): class MoaModelSlot(BaseModel): provider: str = "" model: str = "" + # Optional per-reference guidance. Aggregator slots accept the field in the + # API payload for round-trip compatibility, but normalize_moa_config drops + # it for aggregators because only reference calls consume role prompts. + role_prompt: str = "" class MoaPresetPayload(BaseModel): diff --git a/tests/hermes_cli/test_moa_cmd.py b/tests/hermes_cli/test_moa_cmd.py new file mode 100644 index 00000000000..d4f3a25095a --- /dev/null +++ b/tests/hermes_cli/test_moa_cmd.py @@ -0,0 +1,59 @@ +from types import SimpleNamespace + + +def test_cmd_moa_configure_preserves_reference_role_prompt(monkeypatch, tmp_path, capsys): + """Interactive configure edits provider/model but must not erase role prompts.""" + home = tmp_path / ".hermes" + home.mkdir() + (home / "config.yaml").write_text( + """ +moa: + default_preset: review + presets: + review: + reference_models: + - provider: openrouter + model: x-ai/grok-4.3 + role_prompt: Find real-world edge cases. + aggregator: + provider: openrouter + model: openai/gpt-5.5 +""".strip(), + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + + from hermes_cli import moa_cmd + from hermes_cli.config import load_config + + monkeypatch.setattr( + moa_cmd, + "_model_options", + lambda: [ + { + "slug": "openrouter", + "name": "OpenRouter", + "models": ["x-ai/grok-4.3", "openai/gpt-5.5"], + } + ], + ) + + def fake_prompt(title, rows, default=0): + if title == "Add another reference model?": + return 1 + return default + + monkeypatch.setattr(moa_cmd, "_prompt_choice", fake_prompt) + + moa_cmd.cmd_moa(SimpleNamespace(moa_command="configure", name="review")) + + cfg = load_config() + refs = cfg["moa"]["presets"]["review"]["reference_models"] + assert refs == [ + { + "provider": "openrouter", + "model": "x-ai/grok-4.3", + "role_prompt": "Find real-world edge cases.", + } + ] + assert "role: Find real-world edge cases." in capsys.readouterr().out diff --git a/tests/hermes_cli/test_moa_config.py b/tests/hermes_cli/test_moa_config.py index e04bc638921..dddb0dd4c85 100644 --- a/tests/hermes_cli/test_moa_config.py +++ b/tests/hermes_cli/test_moa_config.py @@ -42,6 +42,55 @@ def test_normalize_moa_config_preserves_named_presets(): assert cfg["reference_models"] == [{"provider": "openai-codex", "model": "gpt-5.5"}] +def test_normalize_moa_config_preserves_reference_role_prompt(): + """Reference slots may carry per-model role guidance for advisory calls.""" + cfg = normalize_moa_config( + { + "presets": { + "review": { + "reference_models": [ + { + "provider": "openrouter", + "model": "x-ai/grok-4.3", + "role_prompt": "Find real-world edge cases.", + } + ] + } + } + } + ) + + assert cfg["presets"]["review"]["reference_models"] == [ + { + "provider": "openrouter", + "model": "x-ai/grok-4.3", + "role_prompt": "Find real-world edge cases.", + } + ] + + +def test_normalize_moa_config_drops_aggregator_role_prompt(): + """role_prompt is reference-only; aggregator slots remain provider/model pairs.""" + cfg = normalize_moa_config( + { + "presets": { + "review": { + "aggregator": { + "provider": "openrouter", + "model": "openai/gpt-5.5", + "role_prompt": "ignored", + } + } + } + } + ) + + assert cfg["presets"]["review"]["aggregator"] == { + "provider": "openrouter", + "model": "openai/gpt-5.5", + } + + def test_legacy_flat_config_becomes_default_preset(): cfg = normalize_moa_config( { diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 1b2caf95291..db38ed377c3 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -492,6 +492,40 @@ class TestWebServerEndpoints: assert cfg["moa"]["reference_models"] == payload["reference_models"] assert cfg["moa"]["aggregator"] == payload["aggregator"] + def test_put_moa_models_preserves_reference_role_prompts(self): + from hermes_cli.config import load_config + + payload = { + "presets": { + "review": { + "reference_models": [ + { + "provider": "openrouter", + "model": "x-ai/grok-4.3", + "role_prompt": "Find real-world edge cases.", + } + ], + "aggregator": { + "provider": "openrouter", + "model": "openai/gpt-5.5", + "role_prompt": "ignored on aggregators", + }, + } + }, + "default_preset": "review", + } + + resp = self.client.put("/api/model/moa", json=payload) + assert resp.status_code == 200 + assert resp.json()["presets"]["review"]["reference_models"][0]["role_prompt"] == "Find real-world edge cases." + roundtrip = self.client.get("/api/model/moa").json() + resp2 = self.client.put("/api/model/moa", json=roundtrip) + assert resp2.status_code == 200 + cfg = load_config() + preset = cfg["moa"]["presets"]["review"] + assert preset["reference_models"][0]["role_prompt"] == "Find real-world edge cases." + assert "role_prompt" not in preset["aggregator"] + # ── GET /api/media (remote image display) ─────────────────────────── def test_get_media_serves_image_in_root(self): diff --git a/tests/run_agent/test_moa_loop_mode.py b/tests/run_agent/test_moa_loop_mode.py index c2dc887e605..18a3f3376f9 100644 --- a/tests/run_agent/test_moa_loop_mode.py +++ b/tests/run_agent/test_moa_loop_mode.py @@ -401,6 +401,34 @@ def test_run_reference_prepends_advisory_system_prompt(monkeypatch): assert msgs[-1]["role"] == "user" +def test_run_reference_appends_slot_role_prompt_to_system_prompt(monkeypatch): + """A reference slot's role_prompt reaches that reference, not just aggregator.""" + from agent.moa_loop import _REFERENCE_SYSTEM_PROMPT, _run_reference + + captured = {} + + def fake_call_llm(**kwargs): + captured.update(kwargs) + return _response("edge-case advice") + + monkeypatch.setattr("agent.moa_loop.call_llm", fake_call_llm) + + label, text = _run_reference( + { + "provider": "openrouter", + "model": "x-ai/grok-4.3", + "role_prompt": "Focus on real-world minority knockout requirements.", + }, + [{"role": "user", "content": "evaluate this plan"}], + ) + + assert text == "edge-case advice" + system = captured["messages"][0] + assert system["role"] == "system" + assert _REFERENCE_SYSTEM_PROMPT in system["content"] + assert "real-world minority knockout requirements" in system["content"] + + def test_moa_facade_references_get_trimmed_messages(monkeypatch, tmp_path): home = tmp_path / ".hermes" home.mkdir() @@ -466,6 +494,99 @@ moa: assert agg_call["tools"] is not None +def test_moa_facade_applies_distinct_reference_role_prompts(monkeypatch, tmp_path): + """YAML role_prompt values reach the matching reference calls via facade.""" + home = tmp_path / ".hermes" + home.mkdir() + (home / "config.yaml").write_text( + """ +moa: + default_preset: review + presets: + review: + reference_models: + - provider: openrouter + model: x-ai/grok-4.3 + role_prompt: Edge {case} critic. + - provider: openrouter + model: z-ai/glm-5.2 + role_prompt: Design presentation critic. + aggregator: + provider: openrouter + model: openai/gpt-5.5 +""".strip(), + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + reference_system_prompts = {} + + def fake_call_llm(**kwargs): + if kwargs["task"] == "moa_reference": + reference_system_prompts[kwargs["model"]] = kwargs["messages"][0]["content"] + return _response(f"advice from {kwargs['model']}") + return _response("aggregator acted") + + monkeypatch.setattr("agent.moa_loop.call_llm", fake_call_llm) + + from agent.moa_loop import MoAChatCompletions + + facade = MoAChatCompletions("review") + facade.create(messages=[{"role": "user", "content": "q"}], tools=[]) + + assert "Edge {case} critic." in reference_system_prompts["x-ai/grok-4.3"] + assert "Design presentation critic." in reference_system_prompts["z-ai/glm-5.2"] + assert "Design presentation critic." not in reference_system_prompts["x-ai/grok-4.3"] + + +def test_moa_reference_cache_key_includes_role_prompt(monkeypatch, tmp_path): + """Changing only role_prompt must invalidate the per-turn reference cache.""" + home = tmp_path / ".hermes" + home.mkdir() + config_path = home / "config.yaml" + + def write_config(role_prompt: str) -> None: + config_path.write_text( + f""" +moa: + default_preset: review + presets: + review: + reference_models: + - provider: openrouter + model: x-ai/grok-4.3 + role_prompt: {role_prompt} + aggregator: + provider: openrouter + model: openai/gpt-5.5 +""".strip(), + encoding="utf-8", + ) + + write_config("first critic") + monkeypatch.setenv("HERMES_HOME", str(home)) + ref_prompts = [] + + def fake_call_llm(**kwargs): + if kwargs["task"] == "moa_reference": + ref_prompts.append(kwargs["messages"][0]["content"]) + return _response("advice") + return _response("acted") + + monkeypatch.setattr("agent.moa_loop.call_llm", fake_call_llm) + + from agent.moa_loop import MoAChatCompletions + + facade = MoAChatCompletions("review") + messages = [{"role": "user", "content": "same turn"}] + facade.create(messages=messages, tools=[]) + write_config("second critic") + facade.create(messages=messages, tools=[]) + + assert len(ref_prompts) == 2 + assert "first critic" in ref_prompts[0] + assert "second critic" in ref_prompts[1] + + def test_moa_disabled_preset_skips_references(monkeypatch, tmp_path): home = tmp_path / ".hermes" home.mkdir() diff --git a/website/docs/user-guide/features/mixture-of-agents.md b/website/docs/user-guide/features/mixture-of-agents.md index ca60d2db357..7efe3d8b351 100644 --- a/website/docs/user-guide/features/mixture-of-agents.md +++ b/website/docs/user-guide/features/mixture-of-agents.md @@ -80,8 +80,14 @@ moa: reference_models: - provider: openai-codex model: gpt-5.5 + role_prompt: >- + Check the plan as a rigorous implementation critic. Surface hidden + assumptions, missed tests, and tool-use risks. - provider: openrouter model: deepseek/deepseek-v4-pro + role_prompt: >- + Check feasibility, edge cases, and whether the extra perspective is + worth its cost. aggregator: provider: openrouter model: anthropic/claude-opus-4.8 @@ -97,6 +103,13 @@ Default preset: - reference: `openrouter:deepseek/deepseek-v4-pro` - aggregator / acting model: `openrouter:anthropic/claude-opus-4.8` +`role_prompt` is optional and reference-only. When present, Hermes appends it +to that reference model's private advisory system prompt, so each reference can +specialize (for example: logic critic, real-world edge-case critic, design +critic, or implementation verifier). It is trusted operator configuration: +don't fill it from untrusted user content, and keep it short because it is sent +on every reference call. + ## Terminal preset management ```bash