From 0c8cf21882bead813b0b2104e0eb04bb2d415c10 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 6 Jul 2026 14:59:36 -0600 Subject: [PATCH 001/149] feat(nemo-relay): activate dynamic plugins Signed-off-by: Bryan Bednarski --- plugins/observability/nemo_relay/README.md | 48 ++++- plugins/observability/nemo_relay/__init__.py | 171 +++++++++++++++-- pyproject.toml | 2 +- tests/plugins/test_nemo_relay_plugin.py | 189 ++++++++++++++++++- 4 files changed, 388 insertions(+), 22 deletions(-) diff --git a/plugins/observability/nemo_relay/README.md b/plugins/observability/nemo_relay/README.md index ddc27e491103..e3bd2dd441bc 100644 --- a/plugins/observability/nemo_relay/README.md +++ b/plugins/observability/nemo_relay/README.md @@ -84,14 +84,15 @@ wheel from this checkout, then install the official NeMo Relay runtime extra: ```bash uv build --wheel python -m pip install --force-reinstall dist/hermes_agent-*.whl -python -m pip install "nemo-relay==0.3" +python -m pip install "nemo-relay>=0.5,<0.7" hermes plugins enable observability/nemo_relay ``` -The plugin fails open when `nemo-relay` is not installed. Install and test it against the official NeMo Relay 0.3 PyPI distribution: +The plugin fails open when `nemo-relay` is not installed. Install a supported +NeMo Relay 0.5 or 0.6 distribution: ```bash -pip install "nemo-relay==0.3" +pip install "nemo-relay>=0.5,<0.7" ``` ## Export Configuration @@ -189,16 +190,46 @@ session, turn, approval, and subagent marks; the plugin skips its manual NeMo Relay. `tool_parallelism.mode = "observe_only"` keeps tool scheduling observational while still wrapping the real execution boundary. +### Dynamic Plugins (NeMo Relay 0.6) + +Hermes can activate explicit native or worker plugins through the NeMo Relay +0.6 binding. Add `dynamic_plugins` entries to the same file; each entry uses +the binding's `plugin_id`, `kind`, `manifest_ref`, optional `environment_ref`, +and component `config` fields: + +```toml +[[dynamic_plugins]] +plugin_id = "example-plugin" +kind = "rust_dynamic" +manifest_ref = "/absolute/path/to/example-plugin/relay-plugin.toml" + +[dynamic_plugins.config] +mode = "enabled" +``` + +Hermes activates these plugins before registering its managed LLM and tool +execution middleware and retains the activation for the runtime lifetime. +During shutdown it closes session exporters, flushes Relay subscribers, and +then closes the activation so callbacks are removed before plugin code is +unloaded. + +NeMo Relay 0.5 does not expose dynamic activation through its Python binding. +When `dynamic_plugins` is present with a 0.5 runtime, Hermes logs an actionable +warning and continues with the ordinary static component configuration, so +ATOF and ATIF observability remain available. No dynamic plugin is loaded in +that degraded mode. + For the full generic Hermes middleware contract, see [`docs/middleware/README.md`](../../../docs/middleware/README.md). ## Canonical Local Examples -The observe-only examples in this section use the official `nemo-relay==0.3` -distribution and a local Ollama model served through the OpenAI-compatible API. +The observe-only examples in this section use a supported NeMo Relay 0.5 or +0.6 distribution and a local Ollama model served through the OpenAI-compatible +API. ```bash -pip install "nemo-relay==0.3" +pip install "nemo-relay>=0.5,<0.7" export HERMES_HOME=/tmp/hermes-nemo-relay-docs/hermes-home mkdir -p "$HERMES_HOME" @@ -444,9 +475,8 @@ for the same execution. This example enables both NeMo Relay observability export and adaptive execution middleware for a local Hermes run. This path requires a NeMo Relay runtime that -supports `[components.config.tool_parallelism]`; the `nemo-relay==0.3` -install used by the earlier observability-only examples does not support this -adaptive config. +supports `[components.config.tool_parallelism]`, as provided by the supported +0.5 and 0.6 releases. ```bash export HERMES_HOME=/tmp/hermes-middleware-test/hermes-home diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index 4716b73ec040..774d2712c5f6 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +import atexit import asyncio import inspect import json @@ -44,6 +45,7 @@ class _SubagentParent: class _Settings: plugins_toml_path: str = "" plugins_config: dict[str, Any] | None = None + dynamic_plugins: list[dict[str, Any]] = field(default_factory=list) adaptive_enabled: bool = False adaptive_mode: str = "observe_only" atof_enabled: bool = False @@ -67,6 +69,8 @@ class _Runtime: self.subagent_parents: dict[str, _SubagentParent] = {} self.atof_exporter: Any = None self._atof_subscriber_name = "hermes.nemo_relay.atof" + self._plugin_activation: Any = None + self._shutdown_registered = False self._plugin_config_initialized = self._configure_plugins_toml() self._plugin_config_needs_reinit = False if not self._plugin_config_initialized: @@ -76,27 +80,64 @@ class _Runtime: if not self.settings.plugins_config: return False plugin_mod = getattr(self.nemo_relay, "plugin", None) + if plugin_mod is None: + return False + plugin_config = _static_plugin_config(self.settings.plugins_config) + if self.settings.dynamic_plugins: + activate_dynamic = getattr(plugin_mod, "activate_dynamic_plugins", None) + if callable(activate_dynamic): + try: + self._ensure_plugin_config_output_dirs(plugin_config) + self._plugin_activation = _resolve_awaitable( + activate_dynamic(plugin_config, self.settings.dynamic_plugins) + ) + self._ensure_shutdown_registered() + return True + except Exception as exc: + logger.warning( + "NeMo Relay dynamic plugin activation failed; continuing with static " + "observability only: %s", + exc, + ) + else: + logger.warning( + "NeMo Relay dynamic plugins require nemo-relay>=0.6,<0.7; this binding does " + "not expose plugin.activate_dynamic_plugins. Continuing with static " + "observability only." + ) initialize = getattr(plugin_mod, "initialize", None) if not callable(initialize): return False try: - self._ensure_plugin_config_output_dirs(self.settings.plugins_config) - _resolve_awaitable(initialize(self.settings.plugins_config)) + self._ensure_plugin_config_output_dirs(plugin_config) + _resolve_awaitable(initialize(plugin_config)) return True except Exception as exc: logger.debug("NeMo Relay plugins.toml init failed: %s", exc, exc_info=True) return False + def _ensure_shutdown_registered(self) -> None: + if self._shutdown_registered: + return + atexit.register(self.shutdown) + self._shutdown_registered = True + def _clear_plugins_toml(self) -> None: if not self._plugin_config_initialized: return - plugin_mod = getattr(self.nemo_relay, "plugin", None) - clear = getattr(plugin_mod, "clear", None) - if not callable(clear): - return try: - _resolve_awaitable(clear()) + if self._plugin_activation is not None: + _flush_relay_subscribers(self.nemo_relay) + close = getattr(self._plugin_activation, "close", None) + if callable(close): + _resolve_awaitable(close()) + else: + plugin_mod = getattr(self.nemo_relay, "plugin", None) + clear = getattr(plugin_mod, "clear", None) + if callable(clear): + _resolve_awaitable(clear()) finally: + self._plugin_activation = None self._plugin_config_initialized = False self._plugin_config_needs_reinit = bool(self.settings.plugins_config) @@ -226,20 +267,46 @@ class _Runtime: self.nemo_relay.scope.pop(state.handle, output=_jsonable(kwargs)) except Exception: logger.debug("NeMo Relay session pop failed", exc_info=True) + try: + _flush_relay_subscribers(self.nemo_relay) + except Exception: + logger.debug("NeMo Relay subscriber flush failed", exc_info=True) self.export_atif(state) if state.atif_exporter is not None and state.atif_subscriber_name: try: state.atif_exporter.deregister(state.atif_subscriber_name) except Exception: logger.debug("NeMo Relay ATIF deregister failed", exc_info=True) - if self._plugin_config_initialized and not self.sessions: + if ( + self._plugin_config_initialized + and self._plugin_activation is None + and not self.sessions + ): try: self._clear_plugins_toml() except Exception: logger.debug("NeMo Relay plugins.toml clear failed", exc_info=True) - elif self.settings.plugins_config and not self.sessions: + elif ( + self.settings.plugins_config + and self._plugin_activation is None + and not self.sessions + ): self._plugin_config_needs_reinit = True + def shutdown(self) -> None: + """Close active sessions and the process-lifetime plugin activation.""" + for session_id in list(self.sessions): + self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) + if self._plugin_config_initialized: + try: + self._clear_plugins_toml() + except Exception: + logger.debug("NeMo Relay plugin runtime shutdown failed", exc_info=True) + self._clear_atof() + if self._shutdown_registered: + atexit.unregister(self.shutdown) + self._shutdown_registered = False + def mark(self, name: str, kwargs: dict[str, Any]) -> None: state = self.ensure_session(kwargs) self.nemo_relay.scope.event( @@ -274,14 +341,14 @@ class _Runtime: def managed_llm_enabled(self) -> bool: return ( - self.settings.adaptive_enabled + (self.settings.adaptive_enabled or self._plugin_activation is not None) and callable(getattr(getattr(self.nemo_relay, "llm", None), "execute", None)) and callable(getattr(self.nemo_relay, "LLMRequest", None)) ) def managed_tool_enabled(self) -> bool: return ( - self.settings.adaptive_enabled + (self.settings.adaptive_enabled or self._plugin_activation is not None) and callable(getattr(getattr(self.nemo_relay, "tools", None), "execute", None)) ) @@ -402,6 +469,10 @@ class _Runtime: def register(ctx) -> None: + # Activate dynamic plugins before Hermes installs the managed execution + # boundaries that invoke their interceptors. + if _load_settings().dynamic_plugins: + _get_runtime() ctx.register_hook("on_session_start", on_session_start) ctx.register_hook("on_session_end", on_session_end) ctx.register_hook("on_session_finalize", on_session_finalize) @@ -648,6 +719,7 @@ def _load_settings() -> _Settings: return _Settings( plugins_toml_path=plugins_toml_path, plugins_config=plugins_config, + dynamic_plugins=_dynamic_plugin_specs(plugins_config), adaptive_enabled=adaptive_config is not None, adaptive_mode=_adaptive_mode(adaptive_config), atof_enabled=_env_bool("HERMES_NEMO_RELAY_ATOF_ENABLED"), @@ -664,6 +736,81 @@ def _load_settings() -> _Settings: ) +def _static_plugin_config(plugins_config: dict[str, Any]) -> dict[str, Any]: + """Return Relay's base plugin config without Hermes-owned host fields.""" + return {key: value for key, value in plugins_config.items() if key != "dynamic_plugins"} + + +def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[str, Any]]: + if not isinstance(plugins_config, dict): + return [] + raw_specs = plugins_config.get("dynamic_plugins") + if raw_specs is None: + return [] + if not isinstance(raw_specs, list): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins config: expected an array of plugin specs" + ) + return [] + + specs: list[dict[str, Any]] = [] + for index, raw_spec in enumerate(raw_specs): + if not isinstance(raw_spec, dict): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: expected an object", index + ) + continue + plugin_id = raw_spec.get("plugin_id") + kind = raw_spec.get("kind") + manifest_ref = raw_spec.get("manifest_ref") + config = raw_spec.get("config", {}) + environment_ref = raw_spec.get("environment_ref") + if not isinstance(plugin_id, str) or not plugin_id.strip(): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: plugin_id is required", index + ) + continue + if kind not in {"rust_dynamic", "worker"}: + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: kind must be rust_dynamic or worker", + index, + ) + continue + if not isinstance(manifest_ref, str) or not manifest_ref.strip(): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: manifest_ref is required", index + ) + continue + if not isinstance(config, dict): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: config must be an object", index + ) + continue + if environment_ref is not None and not isinstance(environment_ref, str): + logger.warning( + "Ignoring invalid NeMo Relay dynamic_plugins[%d]: environment_ref must be a string", + index, + ) + continue + spec: dict[str, Any] = { + "plugin_id": plugin_id, + "kind": kind, + "manifest_ref": manifest_ref, + "config": config, + } + if environment_ref is not None: + spec["environment_ref"] = environment_ref + specs.append(spec) + return specs + + +def _flush_relay_subscribers(nemo_relay: Any) -> None: + subscribers = getattr(nemo_relay, "subscribers", None) + flush = getattr(subscribers, "flush", None) + if callable(flush): + flush() + + def _load_plugins_config(path: str) -> dict[str, Any] | None: if not path: return None @@ -959,4 +1106,6 @@ def _resolve_awaitable(value: Any) -> Any: def reset_for_tests() -> None: global _RUNTIME with _LOCK: + if isinstance(_RUNTIME, _Runtime): + _RUNTIME.shutdown() _RUNTIME = None diff --git a/pyproject.toml b/pyproject.toml index 963210075e0a..564f9572d7bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,7 @@ vision = [] # extra that exposes a Starlette-backed server surface so pip/uv can't resolve # a vulnerable pre-1.0.1 transitive. Bump in lockstep with uv.lock. mcp = ["mcp==1.26.0", "starlette==1.0.1"] # starlette: CVE-2026-48710 -nemo-relay = ["nemo-relay==0.3"] +nemo-relay = ["nemo-relay>=0.5,<0.7"] homeassistant = ["aiohttp==3.14.1"] sms = ["aiohttp==3.14.1"] teams = ["microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.1"] # aiohttp 3.14.1: CVE-2026-34993(RCE)/47265 + 34513/34518/34519/34520/34525 diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 1e72520d2991..263e98c74b1c 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -41,7 +41,12 @@ class _FakeNemoRelay: call_end=self._tool_call_end, execute=self._tool_execute, ) - self.plugin = SimpleNamespace(initialize=self._plugin_initialize, clear=self._plugin_clear) + self.plugin = SimpleNamespace( + initialize=self._plugin_initialize, + clear=self._plugin_clear, + activate_dynamic_plugins=self._plugin_activate_dynamic, + ) + self.subscribers = SimpleNamespace(flush=self._flush_subscribers) self.LLMRequest = _FakeLLMRequest self.AtofExporterConfig = _FakeAtofExporterConfig self.AtofExporterMode = SimpleNamespace(Append="append", Overwrite="overwrite") @@ -100,6 +105,22 @@ class _FakeNemoRelay: async def _plugin_clear(self): self.events.append(("plugin.clear",)) + async def _plugin_activate_dynamic(self, config, dynamic_plugins): + self.events.append(("plugin.activate_dynamic", config, dynamic_plugins)) + return _FakePluginActivation(self.events) + + def _flush_subscribers(self): + self.events.append(("subscribers.flush",)) + + +class _FakePluginActivation: + def __init__(self, events): + self.events = events + self.report = {"diagnostics": []} + + async def close(self): + self.events.append(("plugin.activation.close",)) + class _FakeLLMRequest: def __init__(self, headers, content): @@ -143,6 +164,7 @@ class _FakeAtifExporter: return True def export_json(self): + self.events.append(("atif.export", self.session_id)) return json.dumps({"session_id": self.session_id, "agent_name": self.agent_name}) @@ -181,6 +203,26 @@ mode = "observe_only" monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) +def _enable_dynamic_plugin(tmp_path, monkeypatch) -> Path: + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + f""" +version = 1 + +[[dynamic_plugins]] +plugin_id = "fixture" +kind = "rust_dynamic" +manifest_ref = "{(tmp_path / "fixture" / "relay-plugin.toml").as_posix()}" + +[dynamic_plugins.config] +mode = "test" +""", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) + return plugins_toml + + def test_manifest_fields(): data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text()) assert data["name"] == "nemo_relay" @@ -508,6 +550,151 @@ enabled = true assert event_names.count("plugin.clear") == 1 +def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1") + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "atif")) + + plugin.on_session_start(session_id="s1") + runtime = plugin._get_runtime() + assert runtime is not None + assert runtime._plugin_activation is not None + llm_result = plugin.on_llm_execution_middleware( + session_id="s1", + provider="openai", + model="fixture", + request={"messages": []}, + next_call=lambda request: {"request": request}, + ) + tool_result = plugin.on_tool_execution_middleware( + session_id="s1", + tool_name="fixture-tool", + args={"value": 1}, + next_call=lambda args: {"args": args}, + ) + assert llm_result["request"]["intercepted"] is True + assert tool_result["args"]["intercepted"] is True + plugin.on_session_finalize(session_id="s1", reason="shutdown") + assert runtime._plugin_activation is not None + assert not any(event[0] == "plugin.activation.close" for event in fake.events) + plugin.on_session_start(session_id="s2") + plugin.on_session_finalize(session_id="s2", reason="shutdown") + assert sum(event[0] == "plugin.activate_dynamic" for event in fake.events) == 1 + + activation = next(event for event in fake.events if event[0] == "plugin.activate_dynamic") + assert "dynamic_plugins" not in activation[1] + assert activation[2] == [ + { + "plugin_id": "fixture", + "kind": "rust_dynamic", + "manifest_ref": str(tmp_path / "fixture" / "relay-plugin.toml"), + "config": {"mode": "test"}, + } + ] + event_names = [event[0] for event in fake.events] + assert "plugin.clear" not in event_names + assert event_names.index("subscribers.flush") < event_names.index("atif.export") + assert event_names.index("atif.export") < event_names.index("atif.deregister") + + runtime.shutdown() + event_names = [event[0] for event in fake.events] + assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") + + +def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + class _Context: + def register_hook(self, name, callback): + del name, callback + + def register_middleware(self, name, callback): + del callback + fake.events.append(("hermes.register_middleware", name)) + + plugin.register(_Context()) + + event_names = [event[0] for event in fake.events] + assert event_names.index("plugin.activate_dynamic") < event_names.index( + "hermes.register_middleware" + ) + runtime = plugin._get_runtime() + assert runtime is not None + runtime.shutdown() + + +def test_nemo_relay_plugin_degrades_to_static_config_on_relay_0_5( + tmp_path, monkeypatch, caplog +): + fake = _FakeNemoRelay() + delattr(fake.plugin, "activate_dynamic_plugins") + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + with caplog.at_level("WARNING"): + plugin.on_session_start(session_id="s1") + + initialize = next(event for event in fake.events if event[0] == "plugin.initialize") + assert "dynamic_plugins" not in initialize[1] + assert not any(event[0] == "plugin.activate_dynamic" for event in fake.events) + assert "require nemo-relay>=0.6,<0.7" in caplog.text + + +def test_nemo_relay_plugin_ignores_invalid_dynamic_specs(tmp_path, monkeypatch, caplog): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + """ +version = 1 +dynamic_plugins = [{ kind = "rust_dynamic", manifest_ref = "missing-id" }] +""", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) + + with caplog.at_level("WARNING"): + plugin.on_session_start(session_id="s1") + + assert not any(event[0] == "plugin.activate_dynamic" for event in fake.events) + assert "plugin_id is required" in caplog.text + + +def test_nemo_relay_plugin_registers_shutdown_after_dynamic_retry(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + activation_attempts = 0 + + async def _flaky_activate(config, dynamic_plugins): + nonlocal activation_attempts + activation_attempts += 1 + fake.events.append( + ("plugin.activate_dynamic.attempt", activation_attempts, config, dynamic_plugins) + ) + if activation_attempts == 1: + raise RuntimeError("temporary activation failure") + return _FakePluginActivation(fake.events) + + fake.plugin.activate_dynamic_plugins = _flaky_activate + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + plugin.on_session_start(session_id="s1") + plugin.on_session_finalize(session_id="s1") + plugin.on_session_start(session_id="s2") + + runtime = plugin._get_runtime() + assert runtime is not None + assert runtime._plugin_activation is not None + assert runtime._shutdown_registered is True + assert activation_attempts == 2 + runtime.shutdown() + assert any(event[0] == "plugin.activation.close" for event in fake.events) + + def test_nemo_relay_plugin_keeps_plugins_toml_active_while_other_sessions_remain(tmp_path, monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) From 2f74e29e3b1e9ce4c187a63fbc1432642e4b3486 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 6 Jul 2026 17:06:04 -0600 Subject: [PATCH 002/149] fix(nemo-relay): preserve managed interceptor outcomes Signed-off-by: Bryan Bednarski --- plugins/observability/nemo_relay/__init__.py | 25 +++++- tests/plugins/test_nemo_relay_plugin.py | 86 ++++++++++++++++++++ 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index 774d2712c5f6..afdec5d8ac94 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -20,6 +20,11 @@ logger = logging.getLogger(__name__) _INIT_FAILED = object() _LOCK = threading.RLock() _RUNTIME: "_Runtime | object | None" = None +_RELAY_LLM_SURFACE_BY_API_MODE = { + "anthropic_messages": "anthropic.messages", + "chat_completions": "openai.chat_completions", + "codex_responses": "openai.responses", +} @dataclass @@ -358,6 +363,8 @@ class _Runtime: normalize_payload: Callable[[Any], Any], shape_response: Callable[[Any], Any], make_managed_execute: Callable[[Callable[[Any], Any]], Any], + *, + preserve_raw_response: bool, ) -> Any: # NeMo Relay's native managed execution may wrap a failing callback as an # internal runtime error, hiding the real downstream provider/tool @@ -387,7 +394,9 @@ class _Runtime: if downstream_error is not None and _is_relay_wrapped_callback_error(exc, callback_error): raise downstream_error raise - return raw_response["value"] if raw_response["set"] else managed_result + if preserve_raw_response and raw_response["set"]: + return raw_response["value"] + return managed_result def execute_llm(self, kwargs: dict[str, Any]) -> Any: state = self.ensure_session(kwargs) @@ -404,7 +413,7 @@ class _Runtime: def _make_managed(impl: Callable[[Any], Any]) -> Any: async def _managed_execute() -> Any: result = self.nemo_relay.llm.execute( - str(kwargs.get("provider") or "llm"), + _relay_llm_surface(kwargs), request, impl, handle=state.handle, @@ -426,7 +435,7 @@ class _Runtime: return _managed_execute() return self._run_managed_with_downstream_preservation( - next_call, _normalize, _llm_response_payload, _make_managed + next_call, _normalize, _llm_response_payload, _make_managed, preserve_raw_response=True ) def execute_tool(self, kwargs: dict[str, Any]) -> Any: @@ -464,7 +473,7 @@ class _Runtime: return _managed_execute() return self._run_managed_with_downstream_preservation( - next_call, _normalize, _jsonable, _make_managed + next_call, _normalize, _jsonable, _make_managed, preserve_raw_response=False ) @@ -923,6 +932,14 @@ def _tool_key(kwargs: dict[str, Any]) -> str: ) +def _relay_llm_surface(kwargs: dict[str, Any]) -> str: + api_mode = str(kwargs.get("api_mode") or "").strip().lower() + return _RELAY_LLM_SURFACE_BY_API_MODE.get( + api_mode, + str(kwargs.get("provider") or "llm"), + ) + + def _metadata(kwargs: dict[str, Any]) -> dict[str, Any]: keys = ( "telemetry_schema_version", diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 263e98c74b1c..87e0a34135e3 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -603,6 +603,92 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") +@pytest.mark.parametrize( + ("provider", "api_mode", "expected_surface", "should_rewrite"), + [ + ("custom", "chat_completions", "openai.chat_completions", True), + ("openai-codex", "codex_responses", "openai.responses", True), + ("anthropic", "anthropic_messages", "anthropic.messages", True), + ("custom", "anthropic_messages", "anthropic.messages", True), + ("bedrock", "bedrock_converse", "bedrock", False), + ], +) +def test_nemo_relay_managed_llm_uses_wire_protocol_for_interceptor_dispatch( + tmp_path, + monkeypatch, + provider, + api_mode, + expected_surface, + should_rewrite, +): + fake = _FakeNemoRelay() + supported_surfaces = { + "anthropic.messages", + "openai.chat_completions", + "openai.responses", + } + + def execute(name, request, func, **kwargs): + fake.events.append(("llm.execute.start", name, request.content, kwargs)) + content = dict(request.content) + if name in supported_surfaces: + content["rewritten_for"] = name + result = func(_FakeLLMRequest(request.headers, content)) + fake.events.append(("llm.execute.end", name, result, kwargs)) + return result + + fake.llm.execute = execute + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + result = plugin.on_llm_execution_middleware( + session_id="s1", + provider=provider, + api_mode=api_mode, + model="fixture", + request={"messages": [{"role": "user", "content": "hi"}]}, + next_call=lambda request: request, + ) + + execute_start = next( + event for event in fake.events if event[0] == "llm.execute.start" + ) + assert execute_start[1] == expected_surface + assert execute_start[3]["metadata"]["provider"] == provider + assert execute_start[3]["metadata"]["api_mode"] == api_mode + if should_rewrite: + assert result["rewritten_for"] == expected_surface + else: + assert "rewritten_for" not in result + + +def test_nemo_relay_managed_tool_returns_post_interceptor_result(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + + def execute(name, args, func, **kwargs): + fake.events.append(("tool.execute.start", name, args, kwargs)) + raw = func({"intercepted": True, **args}) + result = {"compressed": True, "raw": raw} + fake.events.append(("tool.execute.end", name, result, kwargs)) + return result + + fake.tools.execute = execute + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + result = plugin.on_tool_execution_middleware( + session_id="s1", + tool_name="fixture-tool", + args={"value": 1}, + next_call=lambda args: {"tool_output": args}, + ) + + assert result == { + "compressed": True, + "raw": {"tool_output": {"intercepted": True, "value": 1}}, + } + + def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_path, monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) From f846a350c7faaa69a4eca36ce27ec1d318a9da74 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 6 Jul 2026 18:04:06 -0600 Subject: [PATCH 003/149] fix(nemo-relay): harden dynamic plugin lifecycle Signed-off-by: Bryan Bednarski --- plugins/observability/nemo_relay/__init__.py | 128 +++++++++++++---- tests/plugins/test_nemo_relay_plugin.py | 144 ++++++++++++++++++- 2 files changed, 235 insertions(+), 37 deletions(-) diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index afdec5d8ac94..92ead8341931 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -130,21 +130,43 @@ class _Runtime: def _clear_plugins_toml(self) -> None: if not self._plugin_config_initialized: return - try: - if self._plugin_activation is not None: + failures: list[str] = [] + if self._plugin_activation is not None: + activation = self._plugin_activation + try: _flush_relay_subscribers(self.nemo_relay) - close = getattr(self._plugin_activation, "close", None) - if callable(close): + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + + close = getattr(activation, "close", None) + if callable(close): + try: _resolve_awaitable(close()) + except Exception as exc: + failures.append(f"dynamic plugin activation close failed: {exc}") + finally: + # Retain the owned activation through the complete close + # attempt. The binding transitions it to a terminal state + # before its awaitable resolves, including error results. + self._plugin_activation = None + self._plugin_config_initialized = False + self._plugin_config_needs_reinit = bool(self.settings.plugins_config) else: + failures.append("dynamic plugin activation has no close method") + else: + try: plugin_mod = getattr(self.nemo_relay, "plugin", None) clear = getattr(plugin_mod, "clear", None) if callable(clear): _resolve_awaitable(clear()) - finally: - self._plugin_activation = None - self._plugin_config_initialized = False - self._plugin_config_needs_reinit = bool(self.settings.plugins_config) + except Exception as exc: + failures.append(f"static plugin configuration clear failed: {exc}") + finally: + self._plugin_config_initialized = False + self._plugin_config_needs_reinit = bool(self.settings.plugins_config) + + if failures: + raise RuntimeError("; ".join(failures)) def _activate_direct_fallbacks(self) -> None: self._plugin_config_needs_reinit = False @@ -267,21 +289,25 @@ class _Runtime: state = self.sessions.pop(session_id, None) if state is None: return + failures: list[str] = [] if state.handle is not None: try: self.nemo_relay.scope.pop(state.handle, output=_jsonable(kwargs)) - except Exception: - logger.debug("NeMo Relay session pop failed", exc_info=True) + except Exception as exc: + failures.append(f"session scope pop failed: {exc}") try: _flush_relay_subscribers(self.nemo_relay) - except Exception: - logger.debug("NeMo Relay subscriber flush failed", exc_info=True) - self.export_atif(state) + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + try: + self.export_atif(state) + except Exception as exc: + failures.append(f"ATIF export failed: {exc}") if state.atif_exporter is not None and state.atif_subscriber_name: try: state.atif_exporter.deregister(state.atif_subscriber_name) - except Exception: - logger.debug("NeMo Relay ATIF deregister failed", exc_info=True) + except Exception as exc: + failures.append(f"ATIF deregister failed: {exc}") if ( self._plugin_config_initialized and self._plugin_activation is None @@ -289,28 +315,43 @@ class _Runtime: ): try: self._clear_plugins_toml() - except Exception: - logger.debug("NeMo Relay plugins.toml clear failed", exc_info=True) + except Exception as exc: + failures.append(f"plugin configuration clear failed: {exc}") elif ( self.settings.plugins_config and self._plugin_activation is None and not self.sessions ): self._plugin_config_needs_reinit = True + if failures: + logger.warning( + "NeMo Relay session %s teardown completed with errors: %s", + session_id, + "; ".join(failures), + ) def shutdown(self) -> None: """Close active sessions and the process-lifetime plugin activation.""" + failures: list[str] = [] for session_id in list(self.sessions): - self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) + try: + self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) + except Exception as exc: + failures.append(f"session {session_id} close failed: {exc}") if self._plugin_config_initialized: try: self._clear_plugins_toml() - except Exception: - logger.debug("NeMo Relay plugin runtime shutdown failed", exc_info=True) + except Exception as exc: + failures.append(f"plugin runtime close failed: {exc}") self._clear_atof() - if self._shutdown_registered: + if self._shutdown_registered and self._plugin_activation is None: atexit.unregister(self.shutdown) self._shutdown_registered = False + if failures: + logger.warning( + "NeMo Relay runtime shutdown completed with errors: %s", + "; ".join(failures), + ) def mark(self, name: str, kwargs: dict[str, Any]) -> None: state = self.ensure_session(kwargs) @@ -372,7 +413,7 @@ class _Runtime: # execution so Hermes retry classification still sees it. The LLM and tool # paths share this scaffolding; they differ only in payload normalization, # response shaping, and the Relay call itself. - raw_response: dict[str, Any] = {"set": False, "value": None} + raw_response: dict[str, Any] = {"set": False, "value": None, "normalized": None} callback_error: Exception | None = None downstream_error: BaseException | None = None @@ -386,7 +427,8 @@ class _Runtime: raise raw_response["set"] = True raw_response["value"] = raw - return shape_response(raw) + raw_response["normalized"] = shape_response(raw) + return raw_response["normalized"] try: managed_result = _resolve_awaitable(make_managed_execute(_impl)) @@ -394,7 +436,11 @@ class _Runtime: if downstream_error is not None and _is_relay_wrapped_callback_error(exc, callback_error): raise downstream_error raise - if preserve_raw_response and raw_response["set"]: + if ( + preserve_raw_response + and raw_response["set"] + and _json_semantically_equal(managed_result, raw_response["normalized"]) + ): return raw_response["value"] return managed_result @@ -763,11 +809,13 @@ def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[st return [] specs: list[dict[str, Any]] = [] + invalid = False for index, raw_spec in enumerate(raw_specs): if not isinstance(raw_spec, dict): logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: expected an object", index + "Invalid NeMo Relay dynamic_plugins[%d]: expected an object", index ) + invalid = True continue plugin_id = raw_spec.get("plugin_id") kind = raw_spec.get("kind") @@ -776,30 +824,35 @@ def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[st environment_ref = raw_spec.get("environment_ref") if not isinstance(plugin_id, str) or not plugin_id.strip(): logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: plugin_id is required", index + "Invalid NeMo Relay dynamic_plugins[%d]: plugin_id is required", index ) + invalid = True continue if kind not in {"rust_dynamic", "worker"}: logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: kind must be rust_dynamic or worker", + "Invalid NeMo Relay dynamic_plugins[%d]: kind must be rust_dynamic or worker", index, ) + invalid = True continue if not isinstance(manifest_ref, str) or not manifest_ref.strip(): logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: manifest_ref is required", index + "Invalid NeMo Relay dynamic_plugins[%d]: manifest_ref is required", index ) + invalid = True continue if not isinstance(config, dict): logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: config must be an object", index + "Invalid NeMo Relay dynamic_plugins[%d]: config must be an object", index ) + invalid = True continue if environment_ref is not None and not isinstance(environment_ref, str): logger.warning( - "Ignoring invalid NeMo Relay dynamic_plugins[%d]: environment_ref must be a string", + "Invalid NeMo Relay dynamic_plugins[%d]: environment_ref must be a string", index, ) + invalid = True continue spec: dict[str, Any] = { "plugin_id": plugin_id, @@ -810,6 +863,12 @@ def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[st if environment_ref is not None: spec["environment_ref"] = environment_ref specs.append(spec) + if invalid: + logger.error( + "NeMo Relay dynamic plugin configuration is invalid; no dynamic plugins " + "will be activated. Continuing with static observability only." + ) + return [] return specs @@ -999,6 +1058,15 @@ def _jsonable(value: Any) -> Any: return str(value) +def _json_semantically_equal(left: Any, right: Any) -> bool: + """Compare JSON-compatible values without conflating booleans and numbers.""" + try: + options = {"ensure_ascii": False, "sort_keys": True, "separators": (",", ":")} + return json.dumps(_jsonable(left), **options) == json.dumps(_jsonable(right), **options) + except (TypeError, ValueError): + return False + + def _value(obj: Any, key: str, default: Any = None) -> Any: if isinstance(obj, dict): return obj.get(key, default) diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 87e0a34135e3..4bc31f0ae3be 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -662,6 +662,42 @@ def test_nemo_relay_managed_llm_uses_wire_protocol_for_interceptor_dispatch( assert "rewritten_for" not in result +def test_nemo_relay_managed_llm_returns_post_next_interceptor_result(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + raw_response = SimpleNamespace( + model="fixture", + choices=[ + SimpleNamespace( + message=SimpleNamespace(role="assistant", content="raw", tool_calls=[]), + finish_reason="stop", + ) + ], + usage=None, + ) + + def execute(name, request, func, **kwargs): + del name, kwargs + normalized = func(_FakeLLMRequest(request.headers, request.content)) + return {**normalized, "post_next_interceptor": True} + + fake.llm.execute = execute + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + + result = plugin.on_llm_execution_middleware( + session_id="s1", + provider="openai", + api_mode="chat_completions", + model="fixture", + request={"messages": []}, + next_call=lambda request: raw_response, + ) + + assert result["post_next_interceptor"] is True + assert result["assistant_message"]["content"] == "raw" + assert result is not raw_response + + def test_nemo_relay_managed_tool_returns_post_interceptor_result(tmp_path, monkeypatch): fake = _FakeNemoRelay() @@ -730,7 +766,7 @@ def test_nemo_relay_plugin_degrades_to_static_config_on_relay_0_5( assert "require nemo-relay>=0.6,<0.7" in caplog.text -def test_nemo_relay_plugin_ignores_invalid_dynamic_specs(tmp_path, monkeypatch, caplog): +def test_nemo_relay_plugin_rejects_invalid_dynamic_specs(tmp_path, monkeypatch, caplog): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) plugins_toml = tmp_path / "plugins.toml" @@ -750,6 +786,38 @@ dynamic_plugins = [{ kind = "rust_dynamic", manifest_ref = "missing-id" }] assert "plugin_id is required" in caplog.text +def test_nemo_relay_plugin_rejects_entire_mixed_valid_invalid_dynamic_request( + tmp_path, monkeypatch, caplog +): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + f""" +version = 1 + +[[dynamic_plugins]] +plugin_id = "valid-fixture" +kind = "rust_dynamic" +manifest_ref = "{(tmp_path / "valid" / "relay-plugin.toml").as_posix()}" + +[[dynamic_plugins]] +kind = "worker" +manifest_ref = "{(tmp_path / "invalid" / "relay-plugin.toml").as_posix()}" +""", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) + + with caplog.at_level("WARNING"): + plugin.on_session_start(session_id="s1") + + assert not any(event[0] == "plugin.activate_dynamic" for event in fake.events) + initialize = next(event for event in fake.events if event[0] == "plugin.initialize") + assert "dynamic_plugins" not in initialize[1] + assert "no dynamic plugins will be activated" in caplog.text + + def test_nemo_relay_plugin_registers_shutdown_after_dynamic_retry(tmp_path, monkeypatch): fake = _FakeNemoRelay() activation_attempts = 0 @@ -781,6 +849,66 @@ def test_nemo_relay_plugin_registers_shutdown_after_dynamic_retry(tmp_path, monk assert any(event[0] == "plugin.activation.close" for event in fake.events) +def test_nemo_relay_plugin_attempts_activation_close_after_subscriber_flush_failure( + tmp_path, monkeypatch, caplog +): + fake = _FakeNemoRelay() + + def _failing_flush(): + fake.events.append(("subscribers.flush.failed",)) + raise RuntimeError("flush boom") + + fake.subscribers.flush = _failing_flush + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + plugin.on_session_start(session_id="s1") + runtime = plugin._get_runtime() + assert runtime is not None + + with caplog.at_level("WARNING"): + runtime.shutdown() + + event_names = [event[0] for event in fake.events] + assert event_names.count("subscribers.flush.failed") == 2 + flush_indices = [ + index for index, name in enumerate(event_names) if name == "subscribers.flush.failed" + ] + assert max(flush_indices) < event_names.index("plugin.activation.close") + assert runtime._plugin_activation is None + assert "subscriber flush failed: flush boom" in caplog.text + + +def test_nemo_relay_plugin_continues_shutdown_after_atif_export_failure( + tmp_path, monkeypatch, caplog +): + fake = _FakeNemoRelay() + + class _FailingAtifExporter(_FakeAtifExporter): + def export_json(self): + self.events.append(("atif.export.failed", self.session_id)) + raise OSError("disk full") + + fake.AtifExporter = lambda session_id, agent_name, agent_version, **kwargs: ( + _FailingAtifExporter(fake.events, session_id, agent_name, agent_version, kwargs) + ) + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1") + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "atif")) + plugin.on_session_start(session_id="s1") + runtime = plugin._get_runtime() + assert runtime is not None + + with caplog.at_level("WARNING"): + runtime.shutdown() + + event_names = [event[0] for event in fake.events] + assert event_names.index("atif.export.failed") < event_names.index("atif.deregister") + assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") + assert runtime._plugin_activation is None + assert "ATIF export failed: disk full" in caplog.text + + def test_nemo_relay_plugin_keeps_plugins_toml_active_while_other_sessions_remain(tmp_path, monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -1037,15 +1165,16 @@ mode = "observe_only" ), finish_reason="tool_calls", ) + raw_response = SimpleNamespace( + id="resp-1", + model="demo-model", + choices=[raw_choice], + usage=SimpleNamespace(prompt_tokens=3, completion_tokens=5, total_tokens=8), + ) def next_call(request): seen_request.update(request) - return SimpleNamespace( - id="resp-1", - model="demo-model", - choices=[raw_choice], - usage=SimpleNamespace(prompt_tokens=3, completion_tokens=5, total_tokens=8), - ) + return raw_response response = plugin.on_llm_execution_middleware( session_id="s1", @@ -1059,6 +1188,7 @@ mode = "observe_only" next_call=next_call, ) + assert response is raw_response assert response.model == "demo-model" assert response.choices == [raw_choice] assert seen_request["intercepted"] is True From b104657d5e619121154163680c274fe993e3d3dd Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 13 Jul 2026 14:17:45 -0600 Subject: [PATCH 004/149] fix(nemo-relay): widen supported relay range Signed-off-by: Bryan Bednarski --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 564f9572d7bc..0b5b04847f0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,7 @@ vision = [] # extra that exposes a Starlette-backed server surface so pip/uv can't resolve # a vulnerable pre-1.0.1 transitive. Bump in lockstep with uv.lock. mcp = ["mcp==1.26.0", "starlette==1.0.1"] # starlette: CVE-2026-48710 -nemo-relay = ["nemo-relay>=0.5,<0.7"] +nemo-relay = ["nemo-relay>=0.5,<1.0"] homeassistant = ["aiohttp==3.14.1"] sms = ["aiohttp==3.14.1"] teams = ["microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.1"] # aiohttp 3.14.1: CVE-2026-34993(RCE)/47265 + 34513/34518/34519/34520/34525 From 297dbf958fa23f3995862e9b37f7c1147066d707 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 13 Jul 2026 14:50:02 -0600 Subject: [PATCH 005/149] chore(nemo-relay): refresh uv lock for relay 0.5 Signed-off-by: Bryan Bednarski --- uv.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uv.lock b/uv.lock index b51daeca10d3..095ceef3e4fa 100644 --- a/uv.lock +++ b/uv.lock @@ -1804,7 +1804,7 @@ requires-dist = [ { name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" }, { name = "mistralai", marker = "extra == 'mistral'", specifier = "==2.4.8" }, { name = "modal", marker = "extra == 'modal'", specifier = "==1.3.4" }, - { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = "==0.3" }, + { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "openai", specifier = "==2.24.0" }, { name = "packaging", specifier = "==26.0" }, @@ -2608,14 +2608,14 @@ wheels = [ [[package]] name = "nemo-relay" -version = "0.3.0" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/55/3b65643db2df02fb3838b3d251442e05b7306cbbc77e69884ae78743546f/nemo_relay-0.3.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:a6e44abe38bf6dda8f6a8b149cd9069a548186f24263ae8469d5a37cefc95209", size = 5282297, upload-time = "2026-05-29T22:32:36.315Z" }, - { url = "https://files.pythonhosted.org/packages/c7/ef/e2f4bc02f99f38706fdde0cdda6b6d8fddea9e6975da1b66377c35958b85/nemo_relay-0.3.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99ba247121cd0d17b9c5e2a95beb42512243430773b93435848b8b4b9f9ca61f", size = 4722431, upload-time = "2026-05-29T22:32:38.088Z" }, - { url = "https://files.pythonhosted.org/packages/12/18/bedb5d57f206e2859cef11f12f568974a529acf382436523a37e095b2cc7/nemo_relay-0.3.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20bb1e4ed87f179befc4db3af2e35f08be711378b322f3672222d80294f57be7", size = 5002734, upload-time = "2026-05-29T22:32:40.191Z" }, - { url = "https://files.pythonhosted.org/packages/26/ec/4b2e758a0a25397f2e97be889a11b7787d108658e0b60ddff5048b25adb8/nemo_relay-0.3.0-cp311-abi3-win_amd64.whl", hash = "sha256:e659c2772b35c0ea4897a91f6bf86b551ed403f6f41d0b60fd8151f5c78b83d0", size = 4947785, upload-time = "2026-05-29T22:32:41.784Z" }, - { url = "https://files.pythonhosted.org/packages/2b/91/45a3397559679d846ae023a82527c43b14631786b3a7c95e69c05e8bb553/nemo_relay-0.3.0-cp311-abi3-win_arm64.whl", hash = "sha256:9655514adb518e19caf7b3f6a08bbe82c1b24759c0a8021c3df949fd265bcef6", size = 4662147, upload-time = "2026-05-29T22:32:43.554Z" }, + { url = "https://files.pythonhosted.org/packages/aa/fc/6892859ffbfb60f8cc09a4181c8d407b574d49490e35cd563e7f0a61bf54/nemo_relay-0.5.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:203409e0f392fc4f40f52277347ed75ba9969b2af28ddb3b6ae53f789185a0ed", size = 7769721, upload-time = "2026-07-08T18:43:57.745Z" }, + { url = "https://files.pythonhosted.org/packages/dd/13/25c4fcf6fb979af07c9562238819183588a910c2ae09767603500d6948c2/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de61ad0644c5e8c300de814f6884c5de7ae51db14694f5057f938b4da54bfaaf", size = 7037083, upload-time = "2026-07-08T18:44:00.135Z" }, + { url = "https://files.pythonhosted.org/packages/49/97/bd4c77e975d50f7d09f8bfcb74d7e704e05a8b2205f555bf557f84edc5fe/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb323569f104506f4f1a0e2104b22d74c9a0bbf3280ec4d58ad3d768de76fb15", size = 7430817, upload-time = "2026-07-08T18:44:02.165Z" }, + { url = "https://files.pythonhosted.org/packages/da/8b/5681f3430e6d25bf7419dbe576da14599e1cfadc083edb74e691a0c3a980/nemo_relay-0.5.0-cp311-abi3-win_amd64.whl", hash = "sha256:da1bd1e1dec79b7bda3984dd8c59b32ba97abf96c37e85cd1f38d208a29f42b9", size = 7359048, upload-time = "2026-07-08T18:44:04.169Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f2/65c3ef0435e671c829063d4872897f9054f369f0aeacab12c5aefe486705/nemo_relay-0.5.0-cp311-abi3-win_arm64.whl", hash = "sha256:87462585f17b40ce82c54347acef5ace96ee11de3015f234e0b902f23b3793fb", size = 6934235, upload-time = "2026-07-08T18:44:06.078Z" }, ] [[package]] From c675e7c793fe8e7d9678d7aa6c3fdf8302ffdb37 Mon Sep 17 00:00:00 2001 From: Minhao HU <26006141+LoicHmh@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:06:45 +0000 Subject: [PATCH 006/149] fix(cron): bound SessionDB init so a hang can't wedge cron forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_job() constructs SessionDB() synchronously with no timeout of its own, unlike the agent's run_conversation call further down, which is already bounded by HERMES_CRON_TIMEOUT. A wedged sqlite3.connect (e.g. a stale flock from a crashed sibling process) hangs this call indefinitely. That hang is invisible to every existing cron safeguard because it happens before _submit_with_guard's future exists: the finally block that discards the job ID from _running_job_ids never runs. The job stays wedged "running" — every later tick logs "already running — skipping" — until the whole gateway process is restarted. Observed in production: a cron job's worker thread was confirmed via a live py-spy thread dump to be parked inside SessionDB.__init__'s sqlite3.connect for 3+ days, silently skipping every scheduled fire in between across a gateway process that otherwise stayed healthy. Bound the SessionDB() construction with its own timeout (HERMES_CRON_SESSION_DB_TIMEOUT, default 10s), following the same bounded-thread-pool pattern already used elsewhere in this file (the delivery retry path, and the agent inactivity watchdog just below). On timeout, log at ERROR and proceed with session_db=None instead of degrading silently to debug level, since an actual hang here is a new condition worth surfacing. Adds tests/cron/test_sessiondb_init_hang.py, including an end-to-end regression proving the dispatch guard is released and a subsequent tick can fire the same job again after a simulated hang. --- cron/scheduler.py | 36 ++++- tests/cron/test_sessiondb_init_hang.py | 179 +++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 tests/cron/test_sessiondb_init_hang.py diff --git a/cron/scheduler.py b/cron/scheduler.py index 511cbda5d384..1a9d4dee5fe4 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2617,10 +2617,44 @@ def run_job( # Initialize SQLite session store so cron job messages are persisted # and discoverable via session_search (same pattern as gateway/run.py). + # + # Bounded with its own timeout (separate from HERMES_CRON_TIMEOUT, which + # only watches the agent's run_conversation below): SessionDB.__init__ + # opens/migrates state.db synchronously and has no timeout of its own + # against a wedged sqlite3.connect (e.g. a stale flock left by a crashed + # sibling process). An unbounded hang here is invisible to every other + # cron safeguard, because it happens BEFORE _submit_with_guard's future + # exists — the finally block that releases the job from + # _running_job_ids never runs, so the job stays wedged "running" until + # the whole gateway process is restarted, silently skipping every + # scheduled fire in between with "already running — skipping". _session_db = None try: from hermes_state import SessionDB - _session_db = SessionDB() + _raw_session_db_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip() + try: + _session_db_timeout = float(_raw_session_db_timeout) if _raw_session_db_timeout else 10.0 + except (ValueError, TypeError): + logger.warning( + "Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using default 10s", + _raw_session_db_timeout, + ) + _session_db_timeout = 10.0 + _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + try: + _session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout) + finally: + # Don't wait for a wedged connect() to unwind — abandon the + # worker thread (same pattern as the agent inactivity timeout + # further down) rather than blocking shutdown on it too. + _session_db_pool.shutdown(wait=False) + except concurrent.futures.TimeoutError: + logger.error( + "Job '%s': SessionDB init did not return within %.0fs — proceeding " + "without a session store for this run instead of blocking it " + "forever", + job.get("id", "?"), _session_db_timeout, + ) except Exception as e: logger.debug("Job '%s': SQLite session store not available: %s", job.get("id", "?"), e) diff --git a/tests/cron/test_sessiondb_init_hang.py b/tests/cron/test_sessiondb_init_hang.py new file mode 100644 index 000000000000..2b18fdefb8ac --- /dev/null +++ b/tests/cron/test_sessiondb_init_hang.py @@ -0,0 +1,179 @@ +"""Regression test for a hung SessionDB() init permanently wedging a cron job. + +Real-world incident: a cron job's ``SessionDB()`` construction inside +``run_job`` blocked forever (a wedged sqlite3.connect against state.db, no +other process holding a competing lock by the time it was diagnosed). Because +that call had no timeout of its own — unlike the agent's run_conversation, +which is already bounded by HERMES_CRON_TIMEOUT — the worker thread submitted +by ``_submit_with_guard`` never returned. Its ``finally`` block, which is the +only thing that discards the job ID from ``_running_job_ids``, never ran. +Every later tick logged "already running — skipping" and the job never fired +again until the whole gateway process was restarted days later. + +These tests prove ``run_job`` now bounds the SessionDB init with its own +timeout (HERMES_CRON_SESSION_DB_TIMEOUT, default 10s) so a hang there can +never again wedge the job past that bound, and — end to end — that the +dispatch guard is released and the job becomes dispatchable again afterward. + +Note: each test releases its ``never_set`` event in a ``finally`` before +returning. concurrent.futures.thread registers an atexit hook that joins +EVERY worker thread ever created by ANY ThreadPoolExecutor in the process +regardless of ``shutdown(wait=False)`` — an event left permanently unset +would hang the whole test process at interpreter exit, not just this test. +""" + +import threading +import time +from unittest.mock import MagicMock, patch + +import pytest + +from cron.scheduler import run_job + + +def _hanging_session_db(never_set: threading.Event): + """Stand-in for hermes_state.SessionDB() that blocks until released — + like the real incident's wedged sqlite3.connect, but bounded so the test + process can still exit cleanly once the assertions are done.""" + never_set.wait(timeout=30) + return MagicMock() + + +class TestSessionDbInitTimeout: + def test_run_job_does_not_hang_when_sessiondb_init_wedges(self, tmp_path, monkeypatch): + """run_job returns promptly even if SessionDB() never returns.""" + monkeypatch.setenv("HERMES_CRON_SESSION_DB_TIMEOUT", "0.2") + never_set = threading.Event() + job = {"id": "wedged-sessiondb", "name": "test", "prompt": "hello"} + + try: + with patch("cron.scheduler._hermes_home", tmp_path), \ + patch("cron.scheduler._resolve_origin", return_value=None), \ + patch("hermes_cli.env_loader.load_hermes_dotenv"), \ + patch("hermes_cli.env_loader.reset_secret_source_cache"), \ + patch("hermes_state.SessionDB", side_effect=lambda: _hanging_session_db(never_set)), \ + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value={ + "api_key": "test-key", + "base_url": "https://example.invalid/v1", + "provider": "openrouter", + "api_mode": "chat_completions", + }, + ), \ + patch("run_agent.AIAgent") as mock_agent_cls: + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "ok"} + mock_agent_cls.return_value = mock_agent + + start = time.monotonic() + success, output, final_response, error = run_job(job) + elapsed = time.monotonic() - start + finally: + never_set.set() + + # Bounded by the 0.2s timeout, not by the hang (which never resolves + # on its own within the test). + assert elapsed < 5.0 + # The run still completes successfully without a session store. + assert success is True + assert final_response == "ok" + kwargs = mock_agent_cls.call_args.kwargs + assert kwargs["session_db"] is None + + def test_invalid_timeout_env_falls_back_to_default(self, tmp_path, monkeypatch, caplog): + """A malformed HERMES_CRON_SESSION_DB_TIMEOUT logs a warning and still + bounds the call (mirrors HERMES_CRON_TIMEOUT's own fallback).""" + monkeypatch.setenv("HERMES_CRON_SESSION_DB_TIMEOUT", "not-a-number") + fake_db = MagicMock() + job = {"id": "bad-timeout-env", "name": "test", "prompt": "hello"} + + with patch("cron.scheduler._hermes_home", tmp_path), \ + patch("cron.scheduler._resolve_origin", return_value=None), \ + patch("hermes_cli.env_loader.load_hermes_dotenv"), \ + patch("hermes_cli.env_loader.reset_secret_source_cache"), \ + patch("hermes_state.SessionDB", return_value=fake_db), \ + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value={ + "api_key": "test-key", + "base_url": "https://example.invalid/v1", + "provider": "openrouter", + "api_mode": "chat_completions", + }, + ), \ + patch("run_agent.AIAgent") as mock_agent_cls: + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "ok"} + mock_agent_cls.return_value = mock_agent + + success, output, final_response, error = run_job(job) + + assert success is True + kwargs = mock_agent_cls.call_args.kwargs + assert kwargs["session_db"] is fake_db # default 10s was plenty for a MagicMock + + +class TestDispatchGuardReleasedAfterHang: + """End-to-end: the real bug symptom was every later tick silently + skipping the job forever. Confirm the fix actually clears that path.""" + + def test_guard_is_released_and_job_refires_after_sessiondb_hang(self, tmp_path, monkeypatch): + import cron.scheduler as sched + + monkeypatch.setenv("HERMES_CRON_SESSION_DB_TIMEOUT", "0.2") + sched._parallel_pool = None + sched._parallel_pool_max_workers = None + sched._running_job_ids.clear() + + never_set = threading.Event() + job = { + "id": "guard-sessiondb-hang", + "name": "guard-sessiondb-hang", + "prompt": "hello", + "schedule": "every 5m", + "enabled": True, + "next_run_at": "2020-01-01T00:00:00", + "deliver": "local", + } + + try: + with patch("cron.scheduler._hermes_home", tmp_path), \ + patch("cron.scheduler._resolve_origin", return_value=None), \ + patch("hermes_cli.env_loader.load_hermes_dotenv"), \ + patch("hermes_cli.env_loader.reset_secret_source_cache"), \ + patch("hermes_state.SessionDB", side_effect=lambda: _hanging_session_db(never_set)), \ + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value={ + "api_key": "test-key", + "base_url": "https://example.invalid/v1", + "provider": "openrouter", + "api_mode": "chat_completions", + }, + ), \ + patch("run_agent.AIAgent") as mock_agent_cls, \ + patch.object(sched, "get_due_jobs", return_value=[job]), \ + patch.object(sched, "advance_next_run"), \ + patch.object(sched, "save_job_output", return_value="/tmp/out"), \ + patch.object(sched, "mark_job_run"), \ + patch.object(sched, "_deliver_result", return_value=None): + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "ok"} + mock_agent_cls.return_value = mock_agent + + n = sched.tick(verbose=False) # sync=True by default: waits for the job + assert n == 1 + + # Without the fix this would still contain the job ID forever. + assert "guard-sessiondb-hang" not in sched.get_running_job_ids() + + # A second tick can dispatch the same job again — before the + # fix this would log "already running — skipping" and + # return 0. + n2 = sched.tick(verbose=False) + assert n2 == 1 + finally: + never_set.set() + sched._running_job_ids.discard("guard-sessiondb-hang") + sched._shutdown_parallel_pool() From ccb045ba7df3b6796584dd6d1e29bb5327fc838c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:15:37 +0530 Subject: [PATCH 007/149] fix(cron): resolve SessionDB timeout from config.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvage of #63935. The original fix read HERMES_CRON_SESSION_DB_TIMEOUT from a bare env var, but AGENTS.md requires non-secret behavioral settings to live in config.yaml with an env var bridge only for backward compatibility. Changes: - Add cron.session_db_timeout_seconds to DEFAULT_CONFIG (default 10s) - Resolution order: HERMES_CRON_SESSION_DB_TIMEOUT env override → cron.session_db_timeout_seconds in config.yaml → 10s default (mirrors the existing script_timeout_seconds pattern) - 0 = unlimited (opt-in for debugging, skips the bound) - Strengthen test: assert the warning is logged on invalid env value (caplog was taken but never asserted) - Add test: verify config.yaml resolution path works end-to-end Co-authored-by: LoicHmh <26006141+LoicHmh@users.noreply.github.com> --- cron/scheduler.py | 56 ++++++++++++++++++-------- hermes_cli/config.py | 6 +++ tests/cron/test_sessiondb_init_hang.py | 54 ++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/cron/scheduler.py b/cron/scheduler.py index 1a9d4dee5fe4..9d8dc7507757 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2631,23 +2631,47 @@ def run_job( _session_db = None try: from hermes_state import SessionDB - _raw_session_db_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip() - try: - _session_db_timeout = float(_raw_session_db_timeout) if _raw_session_db_timeout else 10.0 - except (ValueError, TypeError): - logger.warning( - "Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using default 10s", - _raw_session_db_timeout, - ) + + # Resolve timeout: env override → config.yaml → default 10s. + # Mirrors the script_timeout_seconds resolution pattern. + _session_db_timeout: float | None = None + _raw_env_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip() + if _raw_env_timeout: + try: + _session_db_timeout = float(_raw_env_timeout) + except (ValueError, TypeError): + logger.warning( + "Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using config/default", + _raw_env_timeout, + ) + if _session_db_timeout is None: + try: + from hermes_cli.config import load_config + _cfg = load_config() or {} + _cron_cfg = _cfg.get("cron", {}) if isinstance(_cfg, dict) else {} + _configured = _cron_cfg.get("session_db_timeout_seconds") + if _configured is not None: + _session_db_timeout = float(_configured) + except Exception as exc: + logger.debug( + "Failed to load cron.session_db_timeout_seconds from config: %s", + exc, + ) + if _session_db_timeout is None: _session_db_timeout = 10.0 - _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - try: - _session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout) - finally: - # Don't wait for a wedged connect() to unwind — abandon the - # worker thread (same pattern as the agent inactivity timeout - # further down) rather than blocking shutdown on it too. - _session_db_pool.shutdown(wait=False) + + if _session_db_timeout > 0: + _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + try: + _session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout) + finally: + # Don't wait for a wedged connect() to unwind — abandon the + # worker thread (same pattern as the agent inactivity timeout + # further down) rather than blocking shutdown on it too. + _session_db_pool.shutdown(wait=False) + else: + # 0 = unlimited (legacy behavior, opt-in for debugging) + _session_db = SessionDB() except concurrent.futures.TimeoutError: logger.error( "Job '%s': SessionDB init did not return within %.0fs — proceeding " diff --git a/hermes_cli/config.py b/hermes_cli/config.py index da95d4bdf2b7..b95b1c666a89 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2705,6 +2705,12 @@ DEFAULT_CONFIG = { # recent .md files and prunes older ones. 0 or negative disables # pruning (for operators who manage cleanup externally). Default 50. "output_retention": 50, + # Timeout (seconds) for SessionDB() init inside cron jobs. + # SessionDB opens/migrates state.db synchronously and has no timeout + # of its own against a wedged sqlite3.connect. An unbounded hang here + # wedges the job's dispatch guard forever. Also overridable via + # HERMES_CRON_SESSION_DB_TIMEOUT env var. 0 = unlimited (skip the bound). + "session_db_timeout_seconds": 10, }, # Kanban multi-agent coordination — controls the dispatcher loop that diff --git a/tests/cron/test_sessiondb_init_hang.py b/tests/cron/test_sessiondb_init_hang.py index 2b18fdefb8ac..a98add9561b8 100644 --- a/tests/cron/test_sessiondb_init_hang.py +++ b/tests/cron/test_sessiondb_init_hang.py @@ -107,11 +107,63 @@ class TestSessionDbInitTimeout: mock_agent.run_conversation.return_value = {"final_response": "ok"} mock_agent_cls.return_value = mock_agent - success, output, final_response, error = run_job(job) + with caplog.at_level("WARNING"): + success, output, final_response, error = run_job(job) assert success is True kwargs = mock_agent_cls.call_args.kwargs assert kwargs["session_db"] is fake_db # default 10s was plenty for a MagicMock + # The malformed env var must produce a warning so the misconfiguration + # is observable — otherwise it silently falls back and operators can't + # diagnose why their custom timeout isn't taking effect. + assert any( + "HERMES_CRON_SESSION_DB_TIMEOUT" in rec.message + for rec in caplog.records + ), f"Expected warning about invalid timeout env var; got: {[r.message for r in caplog.records]}" + + def test_timeout_resolved_from_config_yaml(self, tmp_path, monkeypatch): + """cron.session_db_timeout_seconds in config.yaml is respected when + the env var is not set — the canonical config-first resolution path.""" + import yaml + + monkeypatch.delenv("HERMES_CRON_SESSION_DB_TIMEOUT", raising=False) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"cron": {"session_db_timeout_seconds": 0.2}}) + ) + never_set = threading.Event() + job = {"id": "config-timeout", "name": "test", "prompt": "hello"} + + try: + with patch("cron.scheduler._hermes_home", tmp_path), \ + patch("cron.scheduler._resolve_origin", return_value=None), \ + patch("hermes_cli.env_loader.load_hermes_dotenv"), \ + patch("hermes_cli.env_loader.reset_secret_source_cache"), \ + patch("hermes_state.SessionDB", side_effect=lambda: _hanging_session_db(never_set)), \ + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value={ + "api_key": "test-key", + "base_url": "https://example.invalid/v1", + "provider": "openrouter", + "api_mode": "chat_completions", + }, + ), \ + patch("run_agent.AIAgent") as mock_agent_cls: + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "ok"} + mock_agent_cls.return_value = mock_agent + + start = time.monotonic() + success, output, final_response, error = run_job(job) + elapsed = time.monotonic() - start + finally: + never_set.set() + + # Config value 0.2s bounds the hang, not the 10s default. + assert elapsed < 5.0 + assert success is True + assert mock_agent_cls.call_args.kwargs["session_db"] is None class TestDispatchGuardReleasedAfterHang: From 35ebf6ba679f3b3e57e79b7c9ddb7a48c9c33646 Mon Sep 17 00:00:00 2001 From: dsad Date: Sun, 12 Jul 2026 18:58:01 +0300 Subject: [PATCH 008/149] fix(cli): persist close transcript without history alias --- cli.py | 9 ++- .../cli/test_cli_shutdown_memory_messages.py | 58 ++++++++++++++++++- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/cli.py b/cli.py index 9887bb029780..ac07ff7ba531 100644 --- a/cli.py +++ b/cli.py @@ -12830,12 +12830,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): if not isinstance(messages, list) or not messages: return - conversation_history = getattr(self, "conversation_history", None) - if not isinstance(conversation_history, list): - conversation_history = messages - try: - agent._persist_session(messages, conversation_history) + # Do not pass CLI conversation_history here: during interrupted + # shutdown it can alias _session_messages, which makes the DB flush + # treat every live message as already durable. + agent._persist_session(messages) if getattr(agent, "session_id", None): self.session_id = agent.session_id except (Exception, KeyboardInterrupt) as e: diff --git a/tests/cli/test_cli_shutdown_memory_messages.py b/tests/cli/test_cli_shutdown_memory_messages.py index 87df42f337f5..4a6a23900bbd 100644 --- a/tests/cli/test_cli_shutdown_memory_messages.py +++ b/tests/cli/test_cli_shutdown_memory_messages.py @@ -131,7 +131,7 @@ def test_cli_close_persists_agent_session_messages_before_end_session(): cli._persist_active_session_before_close() - agent._persist_session.assert_called_once_with(transcript, conversation_history) + agent._persist_session.assert_called_once_with(transcript) assert cli.session_id == "live-session" @@ -149,7 +149,7 @@ def test_cli_close_persist_falls_back_to_conversation_history(): cli._persist_active_session_before_close() - agent._persist_session.assert_called_once_with(conversation_history, conversation_history) + agent._persist_session.assert_called_once_with(conversation_history) def test_cli_close_persist_skips_empty_transcripts(): @@ -167,3 +167,57 @@ def test_cli_close_persist_skips_empty_transcripts(): cli._persist_active_session_before_close() agent._persist_session.assert_not_called() + + +def test_cli_close_persist_real_db_survives_history_alias(tmp_path, monkeypatch): + """CLI close safety-net must persist even when history aliases messages. + + In the real CLI, ``conversation_history`` and ``agent._session_messages`` can + point at the same live list during interrupted shutdown. Passing that list + as ``conversation_history`` makes ``_flush_messages_to_session_db`` treat + every message as already durable and write zero rows. The close safety-net + should use marker-based dedup instead. + """ + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + from run_agent import AIAgent + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-alias" + db.create_session(session_id=session_id, source="cli") + + transcript = [ + {"role": "user", "content": "long task"}, + {"role": "assistant", "content": "partial answer"}, + ] + + agent = object.__new__(AIAgent) + agent._session_db = db + agent._session_db_created = True + agent.session_id = session_id + agent.platform = "cli" + agent.model = "test-model" + agent._session_messages = transcript + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_disabled = False + agent._cached_system_prompt = None + agent._session_init_model_config = None + agent._parent_session_id = None + agent._session_json_enabled = False + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = transcript + cli.session_id = "old-session" + cli.agent = agent + + assert db.get_messages_as_conversation(session_id) == [] + + cli._persist_active_session_before_close() + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == ["long task", "partial answer"] + assert cli.session_id == session_id From a27d51ef467c4a5c16b08494741a04e7865fa454 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:13:21 +0530 Subject: [PATCH 009/149] fix(cli): preserve resumed history during close flush Retain a distinct CLI history baseline during the signal window before a turn's normal persistence flush. When CLI history aliases the live agent list, use marker-only persistence so a genuinely unflushed tail is written. --- cli.py | 16 +- .../cli/test_cli_shutdown_memory_messages.py | 176 ++++++++++++++++-- 2 files changed, 170 insertions(+), 22 deletions(-) diff --git a/cli.py b/cli.py index ac07ff7ba531..4d5dfafc1a32 100644 --- a/cli.py +++ b/cli.py @@ -12830,11 +12830,19 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): if not isinstance(messages, list) or not messages: return + # A normal turn builds a new list that reuses the resumed-history dicts. + # Keep that CLI history as the baseline so a signal between assigning + # ``_session_messages`` and the turn's DB flush cannot append its durable + # prefix a second time. Once the CLI takes the turn result, however, both + # names can point at the same live list; passing that alias would mark an + # unflushed tail durable without writing it. Marker-only persistence is + # correct only in that alias case. + conversation_history = getattr(self, "conversation_history", None) + if not isinstance(conversation_history, list) or conversation_history is messages: + conversation_history = None + try: - # Do not pass CLI conversation_history here: during interrupted - # shutdown it can alias _session_messages, which makes the DB flush - # treat every live message as already durable. - agent._persist_session(messages) + agent._persist_session(messages, conversation_history) if getattr(agent, "session_id", None): self.session_id = agent.session_id except (Exception, KeyboardInterrupt) as e: diff --git a/tests/cli/test_cli_shutdown_memory_messages.py b/tests/cli/test_cli_shutdown_memory_messages.py index 4a6a23900bbd..b56b0d85f5ee 100644 --- a/tests/cli/test_cli_shutdown_memory_messages.py +++ b/tests/cli/test_cli_shutdown_memory_messages.py @@ -16,6 +16,8 @@ other tests keep their existing no-arg behaviour. from __future__ import annotations +import threading +from typing import Any from unittest.mock import MagicMock, patch @@ -131,7 +133,7 @@ def test_cli_close_persists_agent_session_messages_before_end_session(): cli._persist_active_session_before_close() - agent._persist_session.assert_called_once_with(transcript) + agent._persist_session.assert_called_once_with(transcript, conversation_history) assert cli.session_id == "live-session" @@ -149,7 +151,7 @@ def test_cli_close_persist_falls_back_to_conversation_history(): cli._persist_active_session_before_close() - agent._persist_session.assert_called_once_with(conversation_history) + agent._persist_session.assert_called_once_with(conversation_history, None) def test_cli_close_persist_skips_empty_transcripts(): @@ -169,6 +171,47 @@ def test_cli_close_persist_skips_empty_transcripts(): agent._persist_session.assert_not_called() +def test_cli_close_uses_distinct_history_as_baseline(): + """A pre-flush shutdown keeps the distinct CLI prefix as a DB baseline.""" + import cli as cli_mod + + history = [{"role": "user", "content": "resumed prompt"}] + live_messages = history + [{"role": "assistant", "content": "partial response"}] + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = history + cli.session_id = "session-id" + agent = MagicMock() + agent.session_id = "session-id" + agent._session_messages = live_messages + cli.agent = agent + + cli._persist_active_session_before_close() + + agent._persist_session.assert_called_once_with(live_messages, history) + + +def _real_agent(db, session_id, session_messages): + """Build the real persistence seam without the heavyweight LLM client.""" + from run_agent import AIAgent + + agent = object.__new__(AIAgent) + agent._session_db = db + agent._session_db_created = True + agent.session_id = session_id + agent.platform = "cli" + agent.model = "test-model" + agent._session_messages = session_messages + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_disabled = False + agent._cached_system_prompt = None + agent._session_init_model_config = None + agent._parent_session_id = None + agent._session_json_enabled = False + return agent + + def test_cli_close_persist_real_db_survives_history_alias(tmp_path, monkeypatch): """CLI close safety-net must persist even when history aliases messages. @@ -182,7 +225,6 @@ def test_cli_close_persist_real_db_survives_history_alias(tmp_path, monkeypatch) import cli as cli_mod from hermes_state import SessionDB - from run_agent import AIAgent db = SessionDB(db_path=tmp_path / "state.db") session_id = "cli-close-alias" @@ -193,21 +235,7 @@ def test_cli_close_persist_real_db_survives_history_alias(tmp_path, monkeypatch) {"role": "assistant", "content": "partial answer"}, ] - agent = object.__new__(AIAgent) - agent._session_db = db - agent._session_db_created = True - agent.session_id = session_id - agent.platform = "cli" - agent.model = "test-model" - agent._session_messages = transcript - agent._last_flushed_db_idx = 0 - agent._flushed_db_message_ids = set() - agent._flushed_db_message_session_id = None - agent._persist_disabled = False - agent._cached_system_prompt = None - agent._session_init_model_config = None - agent._parent_session_id = None - agent._session_json_enabled = False + agent = _real_agent(db, session_id, transcript) cli = object.__new__(cli_mod.HermesCLI) cli.conversation_history = transcript @@ -221,3 +249,115 @@ def test_cli_close_persist_real_db_survives_history_alias(tmp_path, monkeypatch) stored = db.get_messages_as_conversation(session_id) assert [m["content"] for m in stored] == ["long task", "partial answer"] assert cli.session_id == session_id + + +def test_cli_close_preflush_resumed_prefix_is_not_duplicated(tmp_path, monkeypatch): + """A signal during the turn-start flush preserves the old DB prefix once. + + The pause is after ``_persist_session`` records its live snapshot but before + its normal DB flush. The close helper must retain the distinct CLI baseline. + """ + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-preflush-resume" + db.create_session(session_id=session_id, source="cli") + loaded = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + for message in loaded: + db.append_message( + session_id=session_id, + role=message["role"], + content=message["content"], + ) + + live_messages = list(loaded) + [{"role": "user", "content": "new prompt"}] + agent = _real_agent(db, session_id, []) + entered_flush = threading.Event() + release_flush = threading.Event() + flush_calls = 0 + + def _pause_before_flush( + messages: list[dict[str, Any]], + conversation_history: list[dict[str, Any]] | None = None, + ) -> None: + nonlocal flush_calls + flush_calls += 1 + if flush_calls == 1: + # The worker has assigned its snapshot and is now paused before its + # regular DB write. The concurrent close call must stay live. + agent._session_messages = messages + entered_flush.set() + assert release_flush.wait(timeout=5) + from run_agent import AIAgent + + # Runtime accepts None; the stub keeps that optional contract explicit. + return AIAgent._flush_messages_to_session_db( + agent, + messages, + conversation_history if conversation_history is not None else [], + ) + + agent._flush_messages_to_session_db = _pause_before_flush + worker = threading.Thread( + target=lambda: agent._persist_session(live_messages, loaded), + daemon=True, + ) + worker.start() + assert entered_flush.wait(timeout=5) + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = list(loaded) + [{"role": "user", "content": "ui prompt"}] + cli.session_id = session_id + cli.agent = agent + cli._persist_active_session_before_close() + + release_flush.set() + worker.join(timeout=5) + assert not worker.is_alive() + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == [ + "old prompt", + "old answer", + "new prompt", + ] + + +def test_cli_close_preserves_unflushed_tail_after_prior_prefix_flush(tmp_path, monkeypatch): + """Marker-only alias close writes only a new tail after a prior flush.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-tail" + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + agent = _real_agent(db, session_id, prefix) + agent._flush_messages_to_session_db(prefix, []) + live_messages = prefix + [{"role": "assistant", "content": "new tail"}] + agent._session_messages = live_messages + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = live_messages + cli.session_id = session_id + cli.agent = agent + + cli._persist_active_session_before_close() + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == [ + "old prompt", + "old answer", + "new tail", + ] From 475922f2ce125290559b86f9a82363c1f6c2639f Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:08:02 +0530 Subject: [PATCH 010/149] fix(cli): serialize close persistence handoff Preserve one durable staged input across terminal close and the worker's early turn flush, without duplicating resumed transcripts or creating a session with a null prompt. Fixes #63766. --- agent/agent_init.py | 8 + agent/turn_context.py | 57 +++++- cli.py | 66 ++++++- run_agent.py | 20 ++- tests/agent/test_turn_context.py | 35 ++++ .../cli/test_cli_shutdown_memory_messages.py | 168 +++++++++++++++++- 6 files changed, 333 insertions(+), 21 deletions(-) diff --git a/agent/agent_init.py b/agent/agent_init.py index c5826b4b9475..f9acb3c982ab 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1311,6 +1311,14 @@ def init_agent( # SQLite session store (optional -- provided by CLI or gateway) agent._session_db = session_db agent._parent_session_id = parent_session_id + # A close flush and the worker's turn-start flush can overlap. The durable + # marker is attached to each in-memory message dict, so its test-and-append + # sequence must be serialized per agent rather than relying on SQLite alone. + agent._session_persist_lock = threading.RLock() + # CLI retains its just-accepted user dict until turn setup can reuse it. + # This preserves the message-local durable marker if close persistence wins + # the race before the agent's normal early turn flush. + agent._pending_cli_user_message = None agent._last_flushed_db_idx = 0 # tracks DB-write cursor to prevent duplicate writes agent._session_db_created = False # DB row deferred to run_conversation() # Most agents own their session row and should finalize it on close(). diff --git a/agent/turn_context.py b/agent/turn_context.py index a5e738588e48..cf1cdff47517 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -271,6 +271,29 @@ def build_turn_context( # Initialize conversation (copy to avoid mutating the caller's list). messages = list(conversation_history) if conversation_history else [] + # The CLI may already have staged this input outside the history passed to + # ``run_conversation``. Reuse it only when its clean transcript text matches + # this turn; a stale handoff from a failed prior turn must not replace a + # later, different user input. Voice turns compare against their explicit + # clean persistence override rather than the API-only prefixed payload. + pending_cli_message = getattr(agent, "_pending_cli_user_message", None) + expected_persist_content = ( + persist_user_message if persist_user_message is not None else user_message + ) + if ( + isinstance(pending_cli_message, dict) + and pending_cli_message.get("content") == expected_persist_content + ): + user_msg = pending_cli_message + # The CLI-staged value is the clean transcript text. Restore the + # API-facing variant (for example, a voice-mode prefix) while retaining + # the same dict and any close-path durable marker. + user_msg["content"] = user_message + else: + user_msg = {"role": "user", "content": user_message} + if isinstance(pending_cli_message, dict): + agent._pending_cli_user_message = None + # Hydrate todo store from conversation history. if conversation_history and not agent._todo_store.has_items(): agent._hydrate_todo_store(conversation_history) @@ -285,6 +308,13 @@ def build_turn_context( if agent._memory_nudge_interval > 0 and agent._turns_since_memory == 0: agent._turns_since_memory = prior_user_turns % agent._memory_nudge_interval + # Add the current user message after the prompt/session setup has made + # close persistence safe. The handoff above preserves any marker already + # stamped by an earlier close flush. + messages.append(user_msg) + current_turn_user_idx = len(messages) - 1 + agent._persist_user_message_idx = current_turn_user_idx + # Track user turns for memory flush and periodic nudge logic. agent._user_turn_count += 1 # Copilot x-initiator: the first API call of this user turn is @@ -313,12 +343,6 @@ def build_turn_context( should_review_memory = True agent._turns_since_memory = 0 - # Add user message. - user_msg = {"role": "user", "content": user_message} - messages.append(user_msg) - current_turn_user_idx = len(messages) - 1 - agent._persist_user_message_idx = current_turn_user_idx - # Cosmetic side-signal: detect an affection "reaction" (ily / <3 / good bot) # and notify the host so it can play hearts. Token-free, never touches the # conversation, and never fatal — a purely optional UI beat. @@ -348,18 +372,33 @@ def build_turn_context( # Create the DB session row now that _cached_system_prompt is populated, so # the persisted snapshot is written non-NULL on the first turn (Issue - # #45499). Idempotent: _ensure_db_session() no-ops once the row exists. - agent._ensure_db_session() + # #45499). Keep row creation and the marker-based append in the same + # per-agent critical section as CLI close persistence. + persist_lock = getattr(agent, "_session_persist_lock", None) + + def _ensure_and_persist() -> None: + agent._ensure_db_session() + agent._persist_session(messages, conversation_history) # Crash-resilience: persist the inbound user turn as soon as the session row exists. try: - agent._persist_session(messages, conversation_history) + if persist_lock is None: + _ensure_and_persist() + else: + with persist_lock: + _ensure_and_persist() except Exception: logger.warning( "Early turn-start session persistence failed for session=%s", agent.session_id or "none", exc_info=True, ) + finally: + # Keep an unmarked staged input available to a later close retry if the + # normal persistence attempt failed. Once the marker is present, the + # close path must no longer treat it as a pre-worker UI input. + if not isinstance(pending_cli_message, dict) or pending_cli_message.get("_db_persisted"): + agent._pending_cli_user_message = None # ── Preflight context compression ── # Gate the (expensive) full token estimate behind a cheap pre-check. diff --git a/cli.py b/cli.py index 4d5dfafc1a32..0b4c4b194286 100644 --- a/cli.py +++ b/cli.py @@ -12129,7 +12129,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): request_overrides=turn_route.get("request_overrides"), ): return None - + agent = self.agent + if agent is None: + return None + # Route image attachments based on the active model's vision capability. # "native" → pass pixels as OpenAI-style content parts (adapters # translate for Anthropic/Gemini/Bedrock). @@ -12217,8 +12220,15 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): from run_agent import _sanitize_surrogates message = _sanitize_surrogates(message) - # Add user message to history - self.conversation_history.append({"role": "user", "content": message}) + # Keep the exact CLI input dict available until turn-start persistence. + # Copy the completed agent transcript before appending: otherwise this + # UI-only staging step mutates ``agent._session_messages`` and exposes a + # duplicate-prone intermediate snapshot to terminal-close persistence. + if self.conversation_history is getattr(agent, "_session_messages", None): + self.conversation_history = list(self.conversation_history) + staged_user_message = {"role": "user", "content": message} + agent._pending_cli_user_message = staged_user_message + self.conversation_history.append(staged_user_message) ChatConsole().print(f"[{_accent_hex()}]{'─' * 40}[/]") print(flush=True) @@ -12825,9 +12835,19 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): return messages = getattr(agent, "_session_messages", None) + pending_cli_message = getattr(agent, "_pending_cli_user_message", None) if not isinstance(messages, list): messages = getattr(self, "conversation_history", None) - if not isinstance(messages, list) or not messages: + if not isinstance(messages, list): + return + if isinstance(pending_cli_message, dict) and not any( + message is pending_cli_message for message in messages + ): + # The UI has accepted a new input but the worker still exposes its + # prior snapshot. Include only that staged dict; the baseline below + # keeps any durable resumed prefix from being re-appended. + messages = [*messages, pending_cli_message] + if not messages: return # A normal turn builds a new list that reuses the resumed-history dicts. @@ -12838,13 +12858,47 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): # unflushed tail durable without writing it. Marker-only persistence is # correct only in that alias case. conversation_history = getattr(self, "conversation_history", None) - if not isinstance(conversation_history, list) or conversation_history is messages: + pending_cli_message = getattr(agent, "_pending_cli_user_message", None) + if ( + isinstance(conversation_history, list) + and conversation_history + and conversation_history[-1] is pending_cli_message + ): + # The UI accepted this user message before the agent finished its + # early persistence. Its dict can already be in ``messages`` but is + # not durable yet, so exclude it from the resumed-history baseline. + conversation_history = conversation_history[:-1] + elif not isinstance(conversation_history, list) or conversation_history is messages: conversation_history = None - try: + # A first-turn close can arrive before the worker builds its cached + # prompt. Build or restore it before the DB row is created so the + # durable transcript never leaves a NULL system_prompt cache entry. + if getattr(agent, "_cached_system_prompt", None) is None: + try: + from agent.conversation_loop import _restore_or_build_system_prompt + + _restore_or_build_system_prompt(agent, None, conversation_history) + except Exception: + logger.debug("Could not build system prompt during CLI close", exc_info=True) + return + if getattr(agent, "_cached_system_prompt", None) is None: + return + + persist_lock = getattr(agent, "_session_persist_lock", None) + + def _ensure_and_persist() -> None: + agent._ensure_db_session() agent._persist_session(messages, conversation_history) if getattr(agent, "session_id", None): self.session_id = agent.session_id + + try: + if persist_lock is None: + _ensure_and_persist() + else: + with persist_lock: + _ensure_and_persist() except (Exception, KeyboardInterrupt) as e: logger.debug("Could not persist active CLI session before close: %s", e) diff --git a/run_agent.py b/run_agent.py index fe378f396ae9..2d710d77e2d9 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1689,10 +1689,22 @@ class AIAgent: """ # Scaffolding removal mutates the live list (desired — ephemeral # retry/failure sentinels must not survive into the real transcript). - self._drop_trailing_empty_response_scaffolding(messages) - self._session_messages = messages - self._save_session_log(messages) - self._flush_messages_to_session_db(messages, conversation_history) + # Close and turn-start persistence can run on separate CLI threads; the + # marker test-and-append below must be one critical section or both can + # observe the same unmarked dict and write duplicate durable rows. + persist_lock = getattr(self, "_session_persist_lock", None) + if persist_lock is None: + self._drop_trailing_empty_response_scaffolding(messages) + self._session_messages = messages + self._save_session_log(messages) + self._flush_messages_to_session_db(messages, conversation_history) + return + + with persist_lock: + self._drop_trailing_empty_response_scaffolding(messages) + self._session_messages = messages + self._save_session_log(messages) + self._flush_messages_to_session_db(messages, conversation_history) def _drop_trailing_empty_response_scaffolding(self, messages: List[Dict]) -> None: """Remove private empty-response retry/failure scaffolding from transcript tails. diff --git a/tests/agent/test_turn_context.py b/tests/agent/test_turn_context.py index 0cd0e91caa7b..d024eca56fc8 100644 --- a/tests/agent/test_turn_context.py +++ b/tests/agent/test_turn_context.py @@ -8,6 +8,7 @@ confirm the prologue produces the right ``TurnContext`` and applies the from __future__ import annotations +import threading import types from unittest.mock import MagicMock, patch @@ -73,6 +74,9 @@ class _FakeAgent: self._invalid_tool_retries = -1 self._vision_supported = None self._persist_calls = 0 + self._session_messages = [] + self._pending_cli_user_message = None + self._session_persist_lock = threading.RLock() # Records _cached_system_prompt at the moment _ensure_db_session() # is called (regression guard for #45499 turn-setup ordering). self._ensure_db_prompt_at_call = "" @@ -206,6 +210,37 @@ def test_persist_user_message_becomes_original(): assert ctx.messages[-1]["content"] == "api-prefixed" +def test_pending_cli_message_carries_durable_marker_to_new_turn_dict(): + """A close-persisted CLI input must not be written again by turn start.""" + agent = _FakeAgent() + staged = {"role": "user", "content": "already durable", "_db_persisted": True} + agent._pending_cli_user_message = staged + + ctx = _build(agent, user_message="already durable") + + assert ctx.messages[-1] is staged + assert ctx.messages[-1]["content"] == "already durable" + assert ctx.messages[-1]["_db_persisted"] is True + assert agent._pending_cli_user_message is None + + +def test_stale_pending_cli_message_does_not_replace_new_turn_input(): + """A failed prior persistence handoff cannot substitute later user input.""" + agent = _FakeAgent() + agent._pending_cli_user_message = {"role": "user", "content": "old prompt"} + + stale = agent._pending_cli_user_message + ctx = _build( + agent, + user_message="new prompt", + conversation_history=[{"role": "assistant", "content": "old answer"}], + ) + + assert ctx.messages[-1]["content"] == "new prompt" + assert ctx.messages[-1] is not stale + assert agent._pending_cli_user_message is None + + def test_memory_nudge_fires_at_interval(): agent = _FakeAgent() agent._memory_nudge_interval = 1 diff --git a/tests/cli/test_cli_shutdown_memory_messages.py b/tests/cli/test_cli_shutdown_memory_messages.py index b56b0d85f5ee..85787a72e4a7 100644 --- a/tests/cli/test_cli_shutdown_memory_messages.py +++ b/tests/cli/test_cli_shutdown_memory_messages.py @@ -205,10 +205,12 @@ def _real_agent(db, session_id, session_messages): agent._flushed_db_message_ids = set() agent._flushed_db_message_session_id = None agent._persist_disabled = False - agent._cached_system_prompt = None + agent._cached_system_prompt = "test system prompt" agent._session_init_model_config = None agent._parent_session_id = None agent._session_json_enabled = False + agent._pending_cli_user_message = None + agent._session_persist_lock = threading.RLock() return agent @@ -315,11 +317,26 @@ def test_cli_close_preflush_resumed_prefix_is_not_duplicated(tmp_path, monkeypat cli.conversation_history = list(loaded) + [{"role": "user", "content": "ui prompt"}] cli.session_id = session_id cli.agent = agent - cli._persist_active_session_before_close() + close_started = threading.Event() + close_finished = threading.Event() + + def _close_while_worker_flushes(): + close_started.set() + cli._persist_active_session_before_close() + close_finished.set() + + close_worker = threading.Thread(target=_close_while_worker_flushes, daemon=True) + close_worker.start() + assert close_started.wait(timeout=5) + # The per-agent persistence lock holds the close flush until the normal + # turn-start write has stamped its durable markers. + assert not close_finished.wait(timeout=0.1) release_flush.set() worker.join(timeout=5) + close_worker.join(timeout=5) assert not worker.is_alive() + assert not close_worker.is_alive() stored = db.get_messages_as_conversation(session_id) assert [m["content"] for m in stored] == [ @@ -361,3 +378,150 @@ def test_cli_close_preserves_unflushed_tail_after_prior_prefix_flush(tmp_path, m "old answer", "new tail", ] + + +def test_cli_close_hands_staged_user_marker_to_turn_start(tmp_path, monkeypatch): + """A close before turn setup does not duplicate the CLI-staged user row.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-staged-user" + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + agent = _real_agent(db, session_id, prefix) + agent._flush_messages_to_session_db(prefix, []) + staged = {"role": "user", "content": "new prompt"} + # `chat()` copies a completed agent transcript before it stages the next + # user input, so close initially sees the prior agent snapshot only. + cli_history = list(prefix) + [staged] + agent._pending_cli_user_message = staged + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = cli_history + cli.session_id = session_id + cli.agent = agent + + # Close appends only the pending UI dict, while treating the durable prefix + # as its baseline. Turn setup then reuses the marked dict without re-writing. + cli._persist_active_session_before_close() + assert staged["_db_persisted"] is True + + worker_messages = list(prefix) + [staged] + agent._persist_session(worker_messages, prefix) + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == [ + "old prompt", + "old answer", + "new prompt", + ] + + +def test_cli_chat_staging_does_not_mutate_live_agent_snapshot(): + """The next CLI input must be outside the prior live agent transcript.""" + import cli as cli_mod + + previous = [{"role": "assistant", "content": "done"}] + agent = MagicMock() + agent._session_messages = previous + agent._pending_cli_user_message = None + + cli = object.__new__(cli_mod.HermesCLI) + cli.agent = agent + cli.conversation_history = previous + + # Model the narrow staging operation in ``chat`` without starting a provider. + if cli.conversation_history is agent._session_messages: + cli.conversation_history = list(cli.conversation_history) + staged = {"role": "user", "content": "next"} + agent._pending_cli_user_message = staged + cli.conversation_history.append(staged) + + assert agent._session_messages == [{"role": "assistant", "content": "done"}] + assert cli.conversation_history == [ + {"role": "assistant", "content": "done"}, + {"role": "user", "content": "next"}, + ] + + +def test_cli_close_persists_pending_user_when_agent_snapshot_is_empty(tmp_path, monkeypatch): + """Close before worker startup persists only the CLI-staged user input.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-before-worker" + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + for message in prefix: + db.append_message( + session_id=session_id, + role=message["role"], + content=message["content"], + ) + + agent = _real_agent(db, session_id, []) + staged = {"role": "user", "content": "new prompt"} + agent._pending_cli_user_message = staged + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = list(prefix) + [staged] + cli.session_id = session_id + cli.agent = agent + + cli._persist_active_session_before_close() + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == [ + "old prompt", + "old answer", + "new prompt", + ] + assert staged["_db_persisted"] is True + + +def test_cli_close_builds_prompt_before_creating_first_session_row(tmp_path, monkeypatch): + """First-turn close persistence must not leave a NULL prompt snapshot.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import agent.conversation_loop as loop_mod + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-first-turn" + agent = _real_agent(db, session_id, []) + agent._session_db_created = False + agent._cached_system_prompt = None + staged = {"role": "user", "content": "first prompt"} + agent._pending_cli_user_message = staged + + def _build_prompt(target, _system_message, _history): + target._cached_system_prompt = "close-built-system-prompt" + + monkeypatch.setattr(loop_mod, "_restore_or_build_system_prompt", _build_prompt) + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = [staged] + cli.session_id = session_id + cli.agent = agent + + cli._persist_active_session_before_close() + + session = db.get_session(session_id) + assert session is not None + assert session["system_prompt"] == "close-built-system-prompt" + assert [m["content"] for m in db.get_messages_as_conversation(session_id)] == [ + "first prompt" + ] From a22a1079a39183cee3f5509c443990a203289b0c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:37:03 +0530 Subject: [PATCH 011/149] fix(cli): preserve noted staged input on close --- cli.py | 9 +- tests/agent/test_turn_context.py | 18 ++++ tests/cli/test_cli_interrupt_ack_race.py | 39 ++++++++ .../cli/test_cli_shutdown_memory_messages.py | 93 +++++++++++++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) diff --git a/cli.py b/cli.py index 0b4c4b194286..45e32090febd 100644 --- a/cli.py +++ b/cli.py @@ -12365,13 +12365,20 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): self._pending_moa_config = None if _moa_cfg is None: _moa_cfg = None + # Model/skill notes and voice instructions are API-local. Keep + # the original staged input as the durable transcript value so a + # close-path marker follows the same dict into turn setup rather + # than producing a second noted user row (#63766). + _persist_clean_user_message = ( + message if (_voice_prefix or agent_message != message) else None + ) try: result = self.agent.run_conversation( user_message=agent_message, conversation_history=self.conversation_history[:-1], # Exclude the message we just added stream_callback=stream_callback, task_id=self.session_id, - persist_user_message=message if _voice_prefix else None, + persist_user_message=_persist_clean_user_message, moa_config=_moa_cfg, ) if getattr(self, "_pending_moa_disable_after_turn", False): diff --git a/tests/agent/test_turn_context.py b/tests/agent/test_turn_context.py index d024eca56fc8..e0642c5f11bf 100644 --- a/tests/agent/test_turn_context.py +++ b/tests/agent/test_turn_context.py @@ -241,6 +241,24 @@ def test_stale_pending_cli_message_does_not_replace_new_turn_input(): assert agent._pending_cli_user_message is None +def test_pending_cli_message_uses_clean_override_for_api_local_note(): + """A noted API message reuses the clean staged dict and its DB marker.""" + agent = _FakeAgent() + staged = {"role": "user", "content": "clean prompt", "_db_persisted": True} + agent._pending_cli_user_message = staged + + ctx = _build( + agent, + user_message="[MODEL NOTE]\n\nclean prompt", + persist_user_message="clean prompt", + ) + + assert ctx.messages[-1] is staged + assert ctx.messages[-1]["content"] == "[MODEL NOTE]\n\nclean prompt" + assert ctx.messages[-1]["_db_persisted"] is True + assert agent._pending_cli_user_message is None + + def test_memory_nudge_fires_at_interval(): agent = _FakeAgent() agent._memory_nudge_interval = 1 diff --git a/tests/cli/test_cli_interrupt_ack_race.py b/tests/cli/test_cli_interrupt_ack_race.py index a55fe43d2022..dbcd7a15bd09 100644 --- a/tests/cli/test_cli_interrupt_ack_race.py +++ b/tests/cli/test_cli_interrupt_ack_race.py @@ -192,3 +192,42 @@ def test_acknowledged_interrupt_still_requeues_message(): queued.append(cli._pending_input.get_nowait()) assert any("redirect please" in str(q) for q in queued) assert cli._last_turn_interrupted is True + + +def test_chat_persists_clean_input_when_a_queued_note_changes_api_message(): + """Queued notes remain API-local and preserve close-handoff marker identity.""" + cli = _make_cli() + + class _NoteAgent(_StubAgent): + def __init__(self, session_id): + super().__init__(session_id, turn_seconds=0) + self.captured = None + + def run_conversation(self, **kwargs): + self.captured = kwargs + return { + "final_response": "done", + "messages": [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + agent = _NoteAgent(cli.session_id) + cli.agent = agent + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + cli._pending_model_switch_note = "[MODEL SWITCH NOTE]" + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + cli.chat("clean prompt") + + assert agent.captured is not None + assert agent.captured["user_message"] == "[MODEL SWITCH NOTE]\n\nclean prompt" + assert agent.captured["persist_user_message"] == "clean prompt" diff --git a/tests/cli/test_cli_shutdown_memory_messages.py b/tests/cli/test_cli_shutdown_memory_messages.py index 85787a72e4a7..63ce29cc033f 100644 --- a/tests/cli/test_cli_shutdown_memory_messages.py +++ b/tests/cli/test_cli_shutdown_memory_messages.py @@ -17,6 +17,7 @@ other tests keep their existing no-arg behaviour. from __future__ import annotations import threading +import types from typing import Any from unittest.mock import MagicMock, patch @@ -491,6 +492,98 @@ def test_cli_close_persists_pending_user_when_agent_snapshot_is_empty(tmp_path, assert staged["_db_persisted"] is True +def test_cli_close_preserves_clean_staged_user_across_noted_worker_turn(tmp_path, monkeypatch): + """A noted API-only turn reuses the close-marked clean staged user row.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-noted-staged-user" + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + agent = _real_agent(db, session_id, prefix) + agent._flush_messages_to_session_db(prefix, []) + staged = {"role": "user", "content": "new prompt"} + agent._pending_cli_user_message = staged + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = list(prefix) + [staged] + cli.session_id = session_id + cli.agent = agent + + cli._persist_active_session_before_close() + assert staged["_db_persisted"] is True + + # A queued model/skills note changes only the API message. The worker + # reuses the marked clean dict, so the normal persistence seam cannot append + # a second noted user row. + from agent.turn_context import build_turn_context + + agent.quiet_mode = True + agent.max_iterations = 1 + agent.provider = "test" + agent.base_url = "" + agent.api_key = "" + agent.api_mode = "chat_completions" + agent.tools = [] + agent.valid_tool_names = set() + agent.enabled_toolsets = None + agent.disabled_toolsets = None + agent._skip_mcp_refresh = True + agent.compression_enabled = False + agent.context_compressor = types.SimpleNamespace(protect_first_n=2, protect_last_n=2) + agent._memory_store = None + agent._memory_manager = None + agent._memory_nudge_interval = 0 + agent._turns_since_memory = 0 + agent._user_turn_count = 0 + agent._todo_store = types.SimpleNamespace(has_items=lambda: True) + agent._tool_guardrails = types.SimpleNamespace(reset_for_turn=lambda: None) + agent._compression_warning = None + agent._interrupt_requested = False + agent._memory_write_origin = "assistant_tool" + agent._stream_context_scrubber = None + agent._stream_think_scrubber = None + agent._restore_primary_runtime = lambda: None + agent._cleanup_dead_connections = lambda: False + agent._emit_status = lambda _message: None + agent._replay_compression_warning = lambda: None + agent._hydrate_todo_store = lambda *_args: None + agent._safe_print = lambda *_args: None + + worker = build_turn_context( + agent, + "[MODEL SWITCH NOTE]\n\nnew prompt", + None, + prefix, + "task", + None, + "new prompt", + None, + restore_or_build_system_prompt=lambda *_args: None, + install_safe_stdio=lambda: None, + sanitize_surrogates=lambda value: value, + summarize_user_message_for_log=lambda value: value, + set_session_context=lambda _session_id: None, + set_current_write_origin=lambda _origin: None, + ra=lambda: types.SimpleNamespace(_set_interrupt=lambda *_args: None), + ) + assert worker.messages[-1] is staged + assert worker.messages[-1]["content"] == "[MODEL SWITCH NOTE]\n\nnew prompt" + + stored = db.get_messages_as_conversation(session_id) + assert [m["content"] for m in stored] == [ + "old prompt", + "old answer", + "new prompt", + ] + + def test_cli_close_builds_prompt_before_creating_first_session_row(tmp_path, monkeypatch): """First-turn close persistence must not leave a NULL prompt snapshot.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) From 0b422559f3b21b6dc2c94039b70df8a5b8a103cf Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:52:32 +0530 Subject: [PATCH 012/149] fix(session): preserve clean multimodal persistence override --- agent/conversation_loop.py | 4 ++-- agent/turn_context.py | 4 ++-- gateway/run.py | 4 ++-- run_agent.py | 15 +++++++++------ tests/run_agent/test_run_agent.py | 25 +++++++++++++++++++++++++ 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 75c07540d885..16083c7cab6d 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -522,12 +522,12 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt): def run_conversation( agent, - user_message: str, + user_message: Any, system_message: str = None, conversation_history: List[Dict[str, Any]] = None, task_id: str = None, stream_callback: Optional[callable] = None, - persist_user_message: Optional[str] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, moa_config: Optional[dict[str, Any]] = None, ) -> Dict[str, Any]: diff --git a/agent/turn_context.py b/agent/turn_context.py index cf1cdff47517..ea150ff30a7c 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -118,12 +118,12 @@ class TurnContext: def build_turn_context( agent, - user_message: str, + user_message: Any, system_message: Optional[str], conversation_history: Optional[List[Dict[str, Any]]], task_id: Optional[str], stream_callback, - persist_user_message: Optional[str], + persist_user_message: Optional[Any], persist_user_timestamp: Optional[float] = None, *, restore_or_build_system_prompt, diff --git a/gateway/run.py b/gateway/run.py index d5b3fbff4743..9b19190338d4 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17123,7 +17123,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew event_message_id: Optional[str] = None, channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, - persist_user_message: Optional[str] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, ) -> Dict[str, Any]: """Profile-scoping wrapper around the agent run. @@ -17184,7 +17184,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew event_message_id: Optional[str] = None, channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, - persist_user_message: Optional[str] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, ) -> Dict[str, Any]: """ diff --git a/run_agent.py b/run_agent.py index 2d710d77e2d9..82e93a5dc04a 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1871,11 +1871,14 @@ class AIAgent: content = msg.get("content") _row_timestamp = msg.get("timestamp") # Apply the persist override to THIS row's written values only - # (never to the live dict). Match the original guard: text-only - # content is replaced; multimodal (list) content is left intact - # so image/audio blocks aren't clobbered by the text override. + # (never to the live dict). A multimodal override is a complete + # clean replacement for an API-local noted payload. Preserve the + # historical text-only guard for a list payload, though: a plain + # text override must not erase its image/audio transcript summary. if _ov_idx == _msg_idx and msg.get("role") == "user": - if _ov_content is not None and not isinstance(content, list): + if _ov_content is not None and ( + not isinstance(content, list) or isinstance(_ov_content, list) + ): content = _ov_content if _ov_timestamp is not None: _row_timestamp = _ov_timestamp @@ -5798,12 +5801,12 @@ class AIAgent: def run_conversation( self, - user_message: str, + user_message: Any, system_message: str = None, conversation_history: List[Dict[str, Any]] = None, task_id: str = None, stream_callback: Optional[callable] = None, - persist_user_message: Optional[str] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, moa_config: Optional[dict[str, Any]] = None, ) -> Dict[str, Any]: diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 396657ea4e85..07e54a50ded4 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -143,6 +143,31 @@ def test_persist_user_message_override_preserves_multimodal_turns(agent): assert messages == [{"role": "user", "content": multimodal_content}] +def test_flush_persist_override_replaces_api_local_multimodal_note(agent): + """A note-added multimodal API payload stores the original clean content.""" + clean_content = [ + {"type": "text", "text": "Describe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + api_content = [ + {"type": "text", "text": "[MODEL SWITCH NOTE]\n\nDescribe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + agent._session_db = MagicMock() + agent._session_db_created = True + agent.session_id = "session-123" + agent._last_flushed_db_idx = 0 + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = clean_content + agent._persist_user_message_timestamp = None + + agent._flush_messages_to_session_db([{"role": "user", "content": api_content}], []) + + db_write = agent._session_db.append_message.call_args.kwargs + assert db_write["content"] == "Describe this screenshot\n[screenshot]" + assert api_content[0]["text"] == "[MODEL SWITCH NOTE]\n\nDescribe this screenshot" + + @pytest.fixture() def agent_with_memory_tool(): """Agent whose valid_tool_names includes 'memory'.""" From 69fd846ef86c034c74a855e98733f7edd3ce433e Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:09:41 +0530 Subject: [PATCH 013/149] fix(session): serialize direct persistence flushes --- run_agent.py | 12 +++++++ tests/run_agent/test_run_agent.py | 52 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/run_agent.py b/run_agent.py index 82e93a5dc04a..e815971d35f5 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1765,6 +1765,18 @@ class AIAgent: return repair_message_sequence(self, messages) def _flush_messages_to_session_db(self, messages: List[Dict], conversation_history: List[Dict] = None): + """Serialize direct and turn-boundary session flushes per agent.""" + persist_lock = getattr(self, "_session_persist_lock", None) + if persist_lock is None: + return self._flush_messages_to_session_db_unlocked(messages, conversation_history) + with persist_lock: + return self._flush_messages_to_session_db_unlocked(messages, conversation_history) + + def _flush_messages_to_session_db_unlocked( + self, + messages: List[Dict], + conversation_history: List[Dict] = None, + ): """Persist any un-flushed messages to the SQLite session store. Deduplicates via an intrinsic ``_DB_PERSISTED_MARKER`` stamped on each diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 07e54a50ded4..b0b9a2b93162 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -11,6 +11,7 @@ import io import json import logging import re +import threading import uuid from logging.handlers import RotatingFileHandler from pathlib import Path @@ -168,6 +169,57 @@ def test_flush_persist_override_replaces_api_local_multimodal_note(agent): assert api_content[0]["text"] == "[MODEL SWITCH NOTE]\n\nDescribe this screenshot" +def test_direct_session_db_flushes_share_marker_claim(agent): + """A direct flush cannot interleave its marker check with `_persist_session`.""" + class _BarrierDB: + def __init__(self): + self.rows = [] + self.entered = threading.Event() + self.release = threading.Event() + self.calls = 0 + self._lock = threading.Lock() + + def append_message(self, **kwargs): + with self._lock: + self.calls += 1 + first = self.calls == 1 + if first: + self.entered.set() + assert self.release.wait(timeout=5) + self.rows.append(kwargs["content"]) + + db = _BarrierDB() + agent._session_db = db + agent._session_db_created = True + agent.session_id = "session-123" + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_user_message_idx = None + agent._persist_user_message_override = None + agent._persist_user_message_timestamp = None + agent._persist_disabled = False + agent._session_persist_lock = threading.RLock() + agent._session_json_enabled = False + + message = {"role": "user", "content": "exactly once"} + normal = threading.Thread(target=lambda: agent._persist_session([message], [])) + direct = threading.Thread(target=lambda: agent._flush_messages_to_session_db([message], [])) + normal.start() + assert db.entered.wait(timeout=5) + direct.start() + # Direct flush is blocked by the agent-wide persistence lock until the + # normal writer stamps the message's durable marker. + assert db.calls == 1 + db.release.set() + normal.join(timeout=5) + direct.join(timeout=5) + + assert not normal.is_alive() + assert not direct.is_alive() + assert db.rows == ["exactly once"] + + @pytest.fixture() def agent_with_memory_tool(): """Agent whose valid_tool_names includes 'memory'.""" From 50aebcbcffada3c8e7c3e7a0dd2181ee80c43e3d Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:23:08 +0530 Subject: [PATCH 014/149] fix(session): preserve clean shortened close snapshots --- run_agent.py | 10 ++++- .../cli/test_cli_shutdown_memory_messages.py | 45 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/run_agent.py b/run_agent.py index e815971d35f5..89c4ce633733 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1887,7 +1887,15 @@ class AIAgent: # clean replacement for an API-local noted payload. Preserve the # historical text-only guard for a list payload, though: a plain # text override must not erase its image/audio transcript summary. - if _ov_idx == _msg_idx and msg.get("role") == "user": + # The close safety-net may flush a shortened snapshot while + # turn setup still owns its staged CLI dict. In that shape the + # normal turn index refers to the full history, not this list; + # preserve the API-local override by recognizing the same dict. + pending_cli_message = getattr(self, "_pending_cli_user_message", None) + is_current_turn_user = ( + _ov_idx == _msg_idx or msg is pending_cli_message + ) + if is_current_turn_user and msg.get("role") == "user": if _ov_content is not None and ( not isinstance(content, list) or isinstance(_ov_content, list) ): diff --git a/tests/cli/test_cli_shutdown_memory_messages.py b/tests/cli/test_cli_shutdown_memory_messages.py index 63ce29cc033f..aec69e8c4692 100644 --- a/tests/cli/test_cli_shutdown_memory_messages.py +++ b/tests/cli/test_cli_shutdown_memory_messages.py @@ -492,6 +492,51 @@ def test_cli_close_persists_pending_user_when_agent_snapshot_is_empty(tmp_path, assert staged["_db_persisted"] is True +def test_cli_close_uses_clean_override_for_shortened_pending_snapshot(tmp_path, monkeypatch): + """Close retains the clean user text when its snapshot omits the prefix.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + + import cli as cli_mod + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "cli-close-shortened-noted-pending" + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + for message in prefix: + db.append_message( + session_id=session_id, + role=message["role"], + content=message["content"], + ) + + agent = _real_agent(db, session_id, []) + staged = {"role": "user", "content": "[MODEL NOTE]\n\nnew prompt"} + agent._pending_cli_user_message = staged + # The normal worker index is relative to the full resumed history, while a + # close before its first persistence flush sees only this staged dict. + agent._persist_user_message_idx = len(prefix) + agent._persist_user_message_override = "new prompt" + agent._persist_user_message_timestamp = None + + cli = object.__new__(cli_mod.HermesCLI) + cli.conversation_history = list(prefix) + [staged] + cli.session_id = session_id + cli.agent = agent + + cli._persist_active_session_before_close() + + assert [m["content"] for m in db.get_messages_as_conversation(session_id)] == [ + "old prompt", + "old answer", + "new prompt", + ] + assert staged["_db_persisted"] is True + + def test_cli_close_preserves_clean_staged_user_across_noted_worker_turn(tmp_path, monkeypatch): """A noted API-only turn reuses the close-marked clean staged user row.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) From 962189d9ea66326d0e40672cbb7fd6110452cc30 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:35:56 +0530 Subject: [PATCH 015/149] fix(cli): clear stale persistence override before staging --- cli.py | 22 +++- run_agent.py | 8 +- tests/cli/test_cli_interrupt_ack_race.py | 131 +++++++++++++++++++++++ 3 files changed, 156 insertions(+), 5 deletions(-) diff --git a/cli.py b/cli.py index 45e32090febd..99b910b23fb8 100644 --- a/cli.py +++ b/cli.py @@ -12226,9 +12226,25 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): # duplicate-prone intermediate snapshot to terminal-close persistence. if self.conversation_history is getattr(agent, "_session_messages", None): self.conversation_history = list(self.conversation_history) - staged_user_message = {"role": "user", "content": message} - agent._pending_cli_user_message = staged_user_message - self.conversation_history.append(staged_user_message) + # The prior turn's override applies only to its own user dict. Clear it + # before exposing the next staged input to close persistence; otherwise + # a shutdown before the worker prologue can write old API-local text as + # this new user message (#63766). + persist_lock = getattr(agent, "_session_persist_lock", None) + + def _stage_user_message() -> None: + agent._persist_user_message_idx = None + agent._persist_user_message_override = None + agent._persist_user_message_timestamp = None + staged_user_message = {"role": "user", "content": message} + agent._pending_cli_user_message = staged_user_message + self.conversation_history.append(staged_user_message) + + if persist_lock is None: + _stage_user_message() + else: + with persist_lock: + _stage_user_message() ChatConsole().print(f"[{_accent_hex()}]{'─' * 40}[/]") print(flush=True) diff --git a/run_agent.py b/run_agent.py index 89c4ce633733..cf8e63ff9b52 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1764,7 +1764,11 @@ class AIAgent: from agent.agent_runtime_helpers import repair_message_sequence return repair_message_sequence(self, messages) - def _flush_messages_to_session_db(self, messages: List[Dict], conversation_history: List[Dict] = None): + def _flush_messages_to_session_db( + self, + messages: List[Dict], + conversation_history: Optional[List[Dict]] = None, + ): """Serialize direct and turn-boundary session flushes per agent.""" persist_lock = getattr(self, "_session_persist_lock", None) if persist_lock is None: @@ -1775,7 +1779,7 @@ class AIAgent: def _flush_messages_to_session_db_unlocked( self, messages: List[Dict], - conversation_history: List[Dict] = None, + conversation_history: Optional[List[Dict]] = None, ): """Persist any un-flushed messages to the SQLite session store. diff --git a/tests/cli/test_cli_interrupt_ack_race.py b/tests/cli/test_cli_interrupt_ack_race.py index dbcd7a15bd09..ac465cb20ca1 100644 --- a/tests/cli/test_cli_interrupt_ack_race.py +++ b/tests/cli/test_cli_interrupt_ack_race.py @@ -26,6 +26,7 @@ from __future__ import annotations import importlib import queue import sys +import threading import time from unittest.mock import MagicMock, patch @@ -231,3 +232,133 @@ def test_chat_persists_clean_input_when_a_queued_note_changes_api_message(): assert agent.captured is not None assert agent.captured["user_message"] == "[MODEL SWITCH NOTE]\n\nclean prompt" assert agent.captured["persist_user_message"] == "clean prompt" + + +def test_chat_clears_previous_turn_persistence_override_before_staging(): + """A close before the next worker starts cannot reuse a stale override.""" + cli = _make_cli() + + class _StagingAgent(_StubAgent): + def __init__(self, session_id): + super().__init__(session_id, turn_seconds=0) + self.staged_override = None + self.staged_message = None + self._session_messages = [] + self._persist_user_message_idx = 7 + self._persist_user_message_override = "previous clean prompt" + self._persist_user_message_timestamp = 123.0 + + def run_conversation(self, **kwargs): + self.staged_override = self._persist_user_message_override + self.staged_message = self._pending_cli_user_message + return { + "final_response": "done", + "messages": [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + agent = _StagingAgent(cli.session_id) + cli.agent = agent + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + cli.chat("new prompt") + + assert agent.staged_override is None + assert agent._persist_user_message_idx is None + assert agent._persist_user_message_timestamp is None + assert agent.staged_message == {"role": "user", "content": "new prompt"} + + +def test_chat_close_does_not_persist_previous_turn_override(tmp_path, monkeypatch): + """A close after input staging writes the new prompt, not old API-only text.""" + from hermes_state import SessionDB + from run_agent import AIAgent + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + cli = _make_cli() + session_id = cli.session_id + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + for message in prefix: + db.append_message( + session_id=session_id, + role=message["role"], + content=message["content"], + ) + + agent = object.__new__(AIAgent) + agent._session_db = db + agent._session_db_created = True + agent.session_id = session_id + agent.platform = "cli" + agent.model = "test-model" + agent._session_messages = [] + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_disabled = False + agent._cached_system_prompt = "test system prompt" + agent._session_init_model_config = None + agent._parent_session_id = None + agent._session_json_enabled = False + agent._pending_cli_user_message = None + agent._session_persist_lock = threading.RLock() + agent._persist_user_message_idx = len(prefix) + agent._persist_user_message_override = "previous clean prompt" + agent._persist_user_message_timestamp = 123.0 + agent._active_children = [] + agent._interrupt_requested = False + entered = threading.Event() + release = threading.Event() + + def _block_run(**_kwargs): + entered.set() + assert release.wait(timeout=5) + return { + "final_response": "done", + "messages": prefix + [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + agent.run_conversation = _block_run + cli.agent = agent + cli.conversation_history = list(prefix) + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + chat_thread = threading.Thread(target=lambda: cli.chat("new prompt")) + chat_thread.start() + assert entered.wait(timeout=5) + cli._persist_active_session_before_close() + release.set() + chat_thread.join(timeout=10) + + assert not chat_thread.is_alive() + assert [m["content"] for m in db.get_messages_as_conversation(session_id)] == [ + "old prompt", + "old answer", + "new prompt", + ] From 32bdc67e104934ca7324df430437e0cd839e01c3 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:24:46 +0530 Subject: [PATCH 016/149] fix(cli): snapshot close state under staging lock --- cli.py | 112 +++++++++++---------- tests/cli/test_cli_interrupt_ack_race.py | 122 +++++++++++++++++++++++ 2 files changed, 180 insertions(+), 54 deletions(-) diff --git a/cli.py b/cli.py index 99b910b23fb8..92010a1000e8 100644 --- a/cli.py +++ b/cli.py @@ -12857,60 +12857,64 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): if not agent or not hasattr(agent, "_persist_session"): return - messages = getattr(agent, "_session_messages", None) - pending_cli_message = getattr(agent, "_pending_cli_user_message", None) - if not isinstance(messages, list): - messages = getattr(self, "conversation_history", None) - if not isinstance(messages, list): - return - if isinstance(pending_cli_message, dict) and not any( - message is pending_cli_message for message in messages - ): - # The UI has accepted a new input but the worker still exposes its - # prior snapshot. Include only that staged dict; the baseline below - # keeps any durable resumed prefix from being re-appended. - messages = [*messages, pending_cli_message] - if not messages: - return - - # A normal turn builds a new list that reuses the resumed-history dicts. - # Keep that CLI history as the baseline so a signal between assigning - # ``_session_messages`` and the turn's DB flush cannot append its durable - # prefix a second time. Once the CLI takes the turn result, however, both - # names can point at the same live list; passing that alias would mark an - # unflushed tail durable without writing it. Marker-only persistence is - # correct only in that alias case. - conversation_history = getattr(self, "conversation_history", None) - pending_cli_message = getattr(agent, "_pending_cli_user_message", None) - if ( - isinstance(conversation_history, list) - and conversation_history - and conversation_history[-1] is pending_cli_message - ): - # The UI accepted this user message before the agent finished its - # early persistence. Its dict can already be in ``messages`` but is - # not durable yet, so exclude it from the resumed-history baseline. - conversation_history = conversation_history[:-1] - elif not isinstance(conversation_history, list) or conversation_history is messages: - conversation_history = None - - # A first-turn close can arrive before the worker builds its cached - # prompt. Build or restore it before the DB row is created so the - # durable transcript never leaves a NULL system_prompt cache entry. - if getattr(agent, "_cached_system_prompt", None) is None: - try: - from agent.conversation_loop import _restore_or_build_system_prompt - - _restore_or_build_system_prompt(agent, None, conversation_history) - except Exception: - logger.debug("Could not build system prompt during CLI close", exc_info=True) - return - if getattr(agent, "_cached_system_prompt", None) is None: - return - persist_lock = getattr(agent, "_session_persist_lock", None) - def _ensure_and_persist() -> None: + def _snapshot_and_persist() -> None: + # This snapshot must share the staging lock with ``chat()``. Without + # it, close can retain a mutable history baseline just before chat + # appends its pending dict; the later flush then mistakes that dict + # for durable history and stamps it without writing a row (#63766). + messages = getattr(agent, "_session_messages", None) + pending_cli_message = getattr(agent, "_pending_cli_user_message", None) + if not isinstance(messages, list): + messages = getattr(self, "conversation_history", None) + if not isinstance(messages, list): + return + if isinstance(pending_cli_message, dict) and not any( + message is pending_cli_message for message in messages + ): + # The UI has accepted a new input but the worker still exposes its + # prior snapshot. Include only that staged dict; the baseline below + # keeps any durable resumed prefix from being re-appended. + messages = [*messages, pending_cli_message] + if not messages: + return + + # A normal turn builds a new list that reuses the resumed-history dicts. + # Keep that CLI history as the baseline so a signal between assigning + # ``_session_messages`` and the turn's DB flush cannot append its durable + # prefix a second time. Once the CLI takes the turn result, however, both + # names can point at the same live list; passing that alias would mark an + # unflushed tail durable without writing it. Marker-only persistence is + # correct only in that alias case. + conversation_history = getattr(self, "conversation_history", None) + pending_cli_message = getattr(agent, "_pending_cli_user_message", None) + if ( + isinstance(conversation_history, list) + and conversation_history + and conversation_history[-1] is pending_cli_message + ): + # The UI accepted this user message before the agent finished its + # early persistence. Its dict can already be in ``messages`` but is + # not durable yet, so exclude it from the resumed-history baseline. + conversation_history = conversation_history[:-1] + elif not isinstance(conversation_history, list) or conversation_history is messages: + conversation_history = None + + # A first-turn close can arrive before the worker builds its cached + # prompt. Build or restore it before the DB row is created so the + # durable transcript never leaves a NULL system_prompt cache entry. + if getattr(agent, "_cached_system_prompt", None) is None: + try: + from agent.conversation_loop import _restore_or_build_system_prompt + + _restore_or_build_system_prompt(agent, None, conversation_history) + except Exception: + logger.debug("Could not build system prompt during CLI close", exc_info=True) + return + if getattr(agent, "_cached_system_prompt", None) is None: + return + agent._ensure_db_session() agent._persist_session(messages, conversation_history) if getattr(agent, "session_id", None): @@ -12918,10 +12922,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): try: if persist_lock is None: - _ensure_and_persist() + _snapshot_and_persist() else: with persist_lock: - _ensure_and_persist() + _snapshot_and_persist() except (Exception, KeyboardInterrupt) as e: logger.debug("Could not persist active CLI session before close: %s", e) diff --git a/tests/cli/test_cli_interrupt_ack_race.py b/tests/cli/test_cli_interrupt_ack_race.py index ac465cb20ca1..c4e8036d67dd 100644 --- a/tests/cli/test_cli_interrupt_ack_race.py +++ b/tests/cli/test_cli_interrupt_ack_race.py @@ -362,3 +362,125 @@ def test_chat_close_does_not_persist_previous_turn_override(tmp_path, monkeypatc "old answer", "new prompt", ] + + +def test_close_waits_for_atomic_cli_staging_before_snapshot(tmp_path, monkeypatch): + """Close cannot retain the mutable pre-append history as its DB baseline.""" + from hermes_state import SessionDB + from run_agent import AIAgent + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + cli = _make_cli() + session_id = cli.session_id + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id=session_id, source="cli") + prefix = [ + {"role": "user", "content": "old prompt"}, + {"role": "assistant", "content": "old answer"}, + ] + for message in prefix: + db.append_message( + session_id=session_id, + role=message["role"], + content=message["content"], + ) + + agent = object.__new__(AIAgent) + agent._session_db = db + agent._session_db_created = True + agent.session_id = session_id + agent.platform = "cli" + agent.model = "test-model" + # Deliberately distinct from CLI history: this is the normal pre-worker + # state that used to let close retain the wrong mutable baseline. + agent._session_messages = list(prefix) + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_disabled = False + agent._cached_system_prompt = "test system prompt" + agent._session_init_model_config = None + agent._parent_session_id = None + agent._session_json_enabled = False + agent._pending_cli_user_message = None + agent._session_persist_lock = threading.RLock() + agent._persist_user_message_idx = None + agent._persist_user_message_override = None + agent._persist_user_message_timestamp = None + agent._active_children = [] + agent._interrupt_requested = False + + staging_entered = threading.Event() + release_staging = threading.Event() + run_entered = threading.Event() + release_run = threading.Event() + + class _BlockingHistory(list): + def __init__(self, values): + super().__init__(values) + self._block_next_append = True + + def append(self, value): + if self._block_next_append: + self._block_next_append = False + staging_entered.set() + assert release_staging.wait(timeout=5) + return super().append(value) + + def _block_run(**_kwargs): + run_entered.set() + assert release_run.wait(timeout=5) + return { + "final_response": "done", + "messages": prefix + [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + agent.run_conversation = _block_run + cli.agent = agent + cli.conversation_history = _BlockingHistory(prefix) + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + chat_thread = threading.Thread(target=lambda: cli.chat("new prompt")) + chat_thread.start() + assert staging_entered.wait(timeout=5) + + close_started = threading.Event() + close_finished = threading.Event() + + def _close(): + close_started.set() + cli._persist_active_session_before_close() + close_finished.set() + + close_thread = threading.Thread(target=_close) + close_thread.start() + assert close_started.wait(timeout=5) + # The close snapshot must wait for the locked pending-pointer/history + # handoff; otherwise the subsequent append poisons its DB baseline. + assert not close_finished.wait(timeout=0.1) + + release_staging.set() + assert run_entered.wait(timeout=5) + assert close_finished.wait(timeout=5) + release_run.set() + chat_thread.join(timeout=10) + close_thread.join(timeout=10) + + assert not chat_thread.is_alive() + assert not close_thread.is_alive() + assert [m["content"] for m in db.get_messages_as_conversation(session_id)] == [ + "old prompt", + "old answer", + "new prompt", + ] From 8341d775a97f7dfda65827617564d703e27719cb Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:37:53 +0530 Subject: [PATCH 017/149] fix(session): restore clean API-local turn content --- agent/turn_finalizer.py | 9 +++ run_agent.py | 16 ++-- ...rn_finalizer_final_response_persistence.py | 81 ++++++++++++++++++- tests/run_agent/test_run_agent.py | 18 +++++ 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 1adc3c6c3632..fdf5babe1aea 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -228,6 +228,15 @@ def finalize_turn( if _tail_role != "assistant": messages.append({"role": "assistant", "content": final_response}) + # The model has completed its request, so replace API-local + # voice/model/skill guidance with the clean user input before writing the + # final durable snapshot and returning the continuation history. Earlier + # turn-start flushes use the DB-only override because their messages are + # still needed for the API request; this finalizer runs after that request + # is complete (#48677 / #63766). + _apply_override = getattr(agent, "_apply_persist_user_message_override", None) + if callable(_apply_override): + _apply_override(messages) agent._persist_session(messages, conversation_history) except Exception as _persist_err: _cleanup_errors.append(f"persist_session: {_persist_err}") diff --git a/run_agent.py b/run_agent.py index cf8e63ff9b52..19f4639b4940 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1663,14 +1663,14 @@ class AIAgent: msg = messages[idx] if isinstance(msg, dict) and msg.get("role") == "user": # Text-only call paths may pass a synthetic API-facing prompt - # and a cleaner transcript string separately. Multimodal - # turns, however, keep image/audio blocks in the live - # messages list that is still used for the API request after - # early crash-resilience persistence. Do not replace those - # blocks with the text-only persistence override before the - # model call is built. The paired timestamp override still - # applies — it is metadata, not content. - if override is not None and not isinstance(msg.get("content"), list): + # and a cleaner transcript string separately. Before the API + # call, a plain-text override must not replace native image/audio + # blocks. A list override, however, is the original clean + # multimodal payload (for example before a queued /model note) + # and must replace the API-local list once the turn is final. + if override is not None and ( + not isinstance(msg.get("content"), list) or isinstance(override, list) + ): msg["content"] = override if timestamp is not None: msg["timestamp"] = timestamp diff --git a/tests/agent/test_turn_finalizer_final_response_persistence.py b/tests/agent/test_turn_finalizer_final_response_persistence.py index 2a54fd6e837f..9c6089aacbd8 100644 --- a/tests/agent/test_turn_finalizer_final_response_persistence.py +++ b/tests/agent/test_turn_finalizer_final_response_persistence.py @@ -51,7 +51,14 @@ class FakeAgent: pass def _persist_session(self, messages, conversation_history): - self.persisted_messages = list(messages) + # Capture the durable write before finalization restores API-local + # guidance to the returned/live transcript. + self.persisted_messages = [dict(message) for message in messages] + + def _apply_persist_user_message_override(self, messages): + from run_agent import AIAgent + + return AIAgent._apply_persist_user_message_override(self, messages) def _file_mutation_verifier_enabled(self): return False @@ -69,6 +76,78 @@ class FakeAgent: pass +def test_finalizer_restores_clean_api_local_text_before_return(monkeypatch): + """One-shot CLI notes do not replay through same-process history.""" + monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: []) + agent = FakeAgent() + messages = [ + {"role": "user", "content": "[MODEL SWITCH NOTE]\n\nclean prompt"}, + {"role": "assistant", "content": "Done."}, + ] + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = "clean prompt" + agent._persist_user_message_timestamp = None + + result = finalize_turn( + agent, + final_response="Done.", + api_call_count=1, + interrupted=False, + failed=False, + messages=messages, + conversation_history=[], + effective_task_id="task", + turn_id="turn", + user_message="[MODEL SWITCH NOTE]\n\nclean prompt", + original_user_message="clean prompt", + _should_review_memory=False, + _turn_exit_reason="text_response(finish_reason=stop)", + ) + + assert agent.persisted_messages[0]["content"] == "clean prompt" + assert result["messages"][0]["content"] == "clean prompt" + + +def test_finalizer_restores_clean_api_local_multimodal_before_return(monkeypatch): + """A queued note does not remain in the next-turn native image payload.""" + monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: []) + agent = FakeAgent() + clean_content = [ + {"type": "text", "text": "Describe the image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + api_content = [ + {"type": "text", "text": "[MODEL SWITCH NOTE]\n\nDescribe the image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + messages = [ + {"role": "user", "content": api_content}, + {"role": "assistant", "content": "Done."}, + ] + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = clean_content + agent._persist_user_message_timestamp = None + + result = finalize_turn( + agent, + final_response="Done.", + api_call_count=1, + interrupted=False, + failed=False, + messages=messages, + conversation_history=[], + effective_task_id="task", + turn_id="turn", + user_message=api_content, + original_user_message=clean_content, + _should_review_memory=False, + _turn_exit_reason="text_response(finish_reason=stop)", + ) + + assert agent.persisted_messages[0]["content"] == clean_content + assert result["messages"][0]["content"] == clean_content + + def test_final_response_closes_tool_tail_before_persistence(monkeypatch): """A recovered/previewed final response must be durable in session history. diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index b0b9a2b93162..85572ef4fc2b 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -144,6 +144,24 @@ def test_persist_user_message_override_preserves_multimodal_turns(agent): assert messages == [{"role": "user", "content": multimodal_content}] +def test_persist_user_message_override_restores_clean_multimodal_note(agent): + clean_content = [ + {"type": "text", "text": "Describe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + api_content = [ + {"type": "text", "text": "[MODEL SWITCH NOTE]\n\nDescribe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + messages = [{"role": "user", "content": api_content}] + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = clean_content + + agent._apply_persist_user_message_override(messages) + + assert messages == [{"role": "user", "content": clean_content}] + + def test_flush_persist_override_replaces_api_local_multimodal_note(agent): """A note-added multimodal API payload stores the original clean content.""" clean_content = [ From b708d10db0afe3c3772fa0ad070ae8032b190535 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:42:23 +0530 Subject: [PATCH 018/149] test(session): type finalizer clean-history assertions --- ...t_turn_finalizer_final_response_persistence.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/agent/test_turn_finalizer_final_response_persistence.py b/tests/agent/test_turn_finalizer_final_response_persistence.py index 9c6089aacbd8..a007e4732644 100644 --- a/tests/agent/test_turn_finalizer_final_response_persistence.py +++ b/tests/agent/test_turn_finalizer_final_response_persistence.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +from typing import Any from agent.turn_finalizer import finalize_turn @@ -30,7 +31,10 @@ class FakeAgent: self._skill_nudge_interval = 0 self._iters_since_skill = 0 self.valid_tool_names = [] - self.persisted_messages = None + self.persisted_messages: list[dict[str, Any]] | None = None + self._persist_user_message_idx: int | None = None + self._persist_user_message_override: Any = None + self._persist_user_message_timestamp: float | None = None def _handle_max_iterations(self, messages, api_call_count): raise AssertionError("not expected") @@ -56,9 +60,10 @@ class FakeAgent: self.persisted_messages = [dict(message) for message in messages] def _apply_persist_user_message_override(self, messages): - from run_agent import AIAgent - - return AIAgent._apply_persist_user_message_override(self, messages) + idx = self._persist_user_message_idx + override = self._persist_user_message_override + if idx is not None and override is not None: + messages[idx]["content"] = override def _file_mutation_verifier_enabled(self): return False @@ -104,6 +109,7 @@ def test_finalizer_restores_clean_api_local_text_before_return(monkeypatch): _turn_exit_reason="text_response(finish_reason=stop)", ) + assert agent.persisted_messages is not None assert agent.persisted_messages[0]["content"] == "clean prompt" assert result["messages"][0]["content"] == "clean prompt" @@ -144,6 +150,7 @@ def test_finalizer_restores_clean_api_local_multimodal_before_return(monkeypatch _turn_exit_reason="text_response(finish_reason=stop)", ) + assert agent.persisted_messages is not None assert agent.persisted_messages[0]["content"] == clean_content assert result["messages"][0]["content"] == clean_content From ff52dce1faed6e4d74ce8341ba13dfe76b8c2311 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 02:05:41 +0530 Subject: [PATCH 019/149] test(cli): cover noted multimodal persistence handoff --- tests/cli/test_cli_interrupt_ack_race.py | 175 +++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/tests/cli/test_cli_interrupt_ack_race.py b/tests/cli/test_cli_interrupt_ack_race.py index c4e8036d67dd..0e2c21b6059c 100644 --- a/tests/cli/test_cli_interrupt_ack_race.py +++ b/tests/cli/test_cli_interrupt_ack_race.py @@ -28,6 +28,7 @@ import queue import sys import threading import time +import types from unittest.mock import MagicMock, patch @@ -234,6 +235,180 @@ def test_chat_persists_clean_input_when_a_queued_note_changes_api_message(): assert agent.captured["persist_user_message"] == "clean prompt" +def test_chat_preserves_clean_multimodal_input_when_note_changes_api_message(): + """A queued note forwards original native parts as the persistence override.""" + cli = _make_cli() + + class _NoteAgent(_StubAgent): + def __init__(self, session_id): + super().__init__(session_id, turn_seconds=0) + self.captured = None + + def run_conversation(self, **kwargs): + self.captured = kwargs + return { + "final_response": "done", + "messages": [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + clean_parts = [ + {"type": "text", "text": "Describe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + agent = _NoteAgent(cli.session_id) + cli.agent = agent + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + cli._pending_model_switch_note = "[MODEL SWITCH NOTE]" + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + cli.chat(clean_parts) + + assert agent.captured is not None + assert agent.captured["persist_user_message"] == clean_parts + assert agent.captured["persist_user_message"] is not agent.captured["user_message"] + api_parts = agent.captured["user_message"] + assert api_parts[0]["text"] == "[MODEL SWITCH NOTE]\n\nDescribe this screenshot" + assert api_parts[1] == clean_parts[1] + + +def test_chat_multimodal_note_persists_clean_input_once(tmp_path, monkeypatch): + """The real CLI-to-agent path stores clean image parts, never the queued note.""" + from hermes_state import SessionDB + from run_agent import AIAgent + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + cli = _make_cli() + session_id = cli.session_id + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id=session_id, source="cli") + + agent = object.__new__(AIAgent) + agent._session_db = db + agent._session_db_created = True + agent.session_id = session_id + agent.platform = "cli" + agent.model = "test-model" + agent.provider = "test" + agent.base_url = "" + agent.api_key = "" + agent.api_mode = "chat_completions" + agent._session_messages = [] + agent._last_flushed_db_idx = 0 + agent._flushed_db_message_ids = set() + agent._flushed_db_message_session_id = None + agent._persist_disabled = False + agent._cached_system_prompt = "test system prompt" + agent._session_init_model_config = None + agent._parent_session_id = None + agent._session_json_enabled = False + agent._pending_cli_user_message = None + agent._session_persist_lock = threading.RLock() + agent._persist_user_message_idx = None + agent._persist_user_message_override = None + agent._persist_user_message_timestamp = None + agent._active_children = [] + agent._interrupt_requested = False + + clean_parts = [ + {"type": "text", "text": "Describe this screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + captured = {} + + def _realish_run(**kwargs): + captured.update(kwargs) + # Drive production turn setup and the real SQLite persistence seam, + # then return a normal CLI result without starting a provider loop. + from agent.turn_context import build_turn_context + + agent.quiet_mode = True + agent.max_iterations = 1 + agent.tools = [] + agent.valid_tool_names = set() + agent.enabled_toolsets = None + agent.disabled_toolsets = None + agent._skip_mcp_refresh = True + agent.compression_enabled = False + agent.context_compressor = types.SimpleNamespace(protect_first_n=2, protect_last_n=2) + agent._memory_store = None + agent._memory_manager = None + agent._memory_nudge_interval = 0 + agent._turns_since_memory = 0 + agent._user_turn_count = 0 + agent._todo_store = types.SimpleNamespace(has_items=lambda: True) + agent._tool_guardrails = types.SimpleNamespace(reset_for_turn=lambda: None) + agent._compression_warning = None + agent._memory_write_origin = "assistant_tool" + agent._stream_context_scrubber = None + agent._stream_think_scrubber = None + agent._restore_primary_runtime = lambda: None + agent._cleanup_dead_connections = lambda: False + agent._emit_status = lambda _message: None + agent._replay_compression_warning = lambda: None + agent._hydrate_todo_store = lambda *_args: None + agent._safe_print = lambda *_args: None + + context = build_turn_context( + agent, + kwargs["user_message"], + None, + kwargs["conversation_history"], + kwargs["task_id"], + None, + kwargs["persist_user_message"], + None, + restore_or_build_system_prompt=lambda *_args: None, + install_safe_stdio=lambda: None, + sanitize_surrogates=lambda value: value, + summarize_user_message_for_log=lambda value: ( + value if isinstance(value, str) else "[multimodal test message]" + ), + set_session_context=lambda _session_id: None, + set_current_write_origin=lambda _origin: None, + ra=lambda: types.SimpleNamespace(_set_interrupt=lambda *_args: None), + ) + agent._apply_persist_user_message_override(context.messages) + agent._persist_session(context.messages, kwargs["conversation_history"]) + return { + "final_response": "done", + "messages": context.messages + [{"role": "assistant", "content": "done"}], + "api_calls": 1, + "completed": True, + "partial": True, + "response_previewed": True, + } + + agent.run_conversation = _realish_run + cli.agent = agent + cli._interrupt_queue = queue.Queue() + cli._pending_input = queue.Queue() + cli._pending_model_switch_note = "[MODEL SWITCH NOTE]" + + with patch.object(cli, "_ensure_runtime_credentials", return_value=True), \ + patch.object(cli, "_resolve_turn_agent_config", return_value={ + "signature": cli._active_agent_route_signature, + "model": None, "runtime": None, "request_overrides": None, + }), \ + patch.object(cli, "_init_agent", return_value=True): + cli.chat(clean_parts) + + assert captured["persist_user_message"] == clean_parts + assert captured["user_message"][0]["text"] == "[MODEL SWITCH NOTE]\n\nDescribe this screenshot" + assert [m["content"] for m in db.get_messages_as_conversation(session_id)] == [ + "Describe this screenshot\n[screenshot]" + ] + + def test_chat_clears_previous_turn_persistence_override_before_staging(): """A close before the next worker starts cannot reuse a stale override.""" cli = _make_cli() From 658c0112661693f4c8c2e4cb3da88c7295413770 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:15:14 +0530 Subject: [PATCH 020/149] fix(deepinfra): restore provider-prefix aliases for model parsing The _PROVIDER_PREFIXES frozenset in agent/model_metadata.py is static and does not auto-extend from ProviderProfile. Removing deepinfra and deep-infra from it broke provider:model prefix stripping for DeepInfra. --- agent/model_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 1e9b66ef3090..74b6c2f13943 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -47,7 +47,7 @@ def _resolve_requests_verify() -> bool | str: # are preserved so the full model name reaches cache lookups and server queries. _PROVIDER_PREFIXES: frozenset[str] = frozenset({ "openrouter", "nous", "openai-codex", "copilot", "copilot-acp", - "gemini", "ollama-cloud", "zai", "kimi-coding", "kimi-coding-cn", "stepfun", "minimax", "minimax-oauth", "minimax-cn", "anthropic", "deepseek", + "gemini", "ollama-cloud", "zai", "kimi-coding", "kimi-coding-cn", "stepfun", "minimax", "minimax-oauth", "minimax-cn", "anthropic", "deepseek", "deepinfra", "opencode-zen", "opencode-go", "kilocode", "alibaba", "novita", "qwen-oauth", "xiaomi", @@ -58,7 +58,7 @@ _PROVIDER_PREFIXES: frozenset[str] = frozenset({ # Common aliases "google", "google-gemini", "google-ai-studio", "glm", "z-ai", "z.ai", "zhipu", "github", "github-copilot", - "github-models", "kimi", "moonshot", "kimi-cn", "moonshot-cn", "claude", "deep-seek", + "github-models", "kimi", "moonshot", "kimi-cn", "moonshot-cn", "claude", "deep-seek", "deep-infra", "ollama", "stepfun", "opencode", "zen", "go", "kilo", "dashscope", "aliyun", "qwen", "mimo", "xiaomi-mimo", From 2d71e2f1e451a84239ae9d013275bf29c47ddad1 Mon Sep 17 00:00:00 2001 From: ethernet Date: Mon, 13 Jul 2026 15:35:53 -0400 Subject: [PATCH 021/149] perf(tools): text prefilter before AST parse in tool discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_module_registers_tools()` reads each `tools/*.py` file and fully AST-parses it to check for a top-level `registry.register()` call. 90 files are scanned on every process start — but only 32 actually register tools. Add a cheap text prefilter: after reading the file (which we need to do anyway for AST), check that both `"registry"` and `"register"` appear in the source before calling `ast.parse`. A file with a top-level `registry.register()` call must contain both strings, so this is a perfect superset — zero false negatives. 50 of 90 files skip the AST parse entirely. The `source=` parameter is not threaded through `discover_builtin_tools`; the prefilter lives entirely inside `_module_registers_tools`, keeping the public API unchanged. Benchmark (median of 10 runs, scanning 90 files): before (read + ast.parse all): 305.9ms after (text prefilter + ast): 187.8ms speedup: 1.6x (118ms saved) Identical module set: 32 modules, same names, same order. --- tools/registry.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/registry.py b/tools/registry.py index 9b6611fb407d..354da7123fd7 100644 --- a/tools/registry.py +++ b/tools/registry.py @@ -45,11 +45,20 @@ def _module_registers_tools(module_path: Path) -> bool: Only inspects module-body statements so that helper modules which happen to call ``registry.register()`` inside a function are not picked up. + + A cheap text prefilter avoids the ``ast.parse`` cost for files that do not + mention both ``registry`` and ``register`` — a necessary condition for a + top-level ``registry.register()`` call to exist. """ try: source = module_path.read_text(encoding="utf-8") + except OSError: + return False + if "registry" not in source or "register" not in source: + return False + try: tree = ast.parse(source, filename=str(module_path)) - except (OSError, SyntaxError): + except SyntaxError: return False return any(_is_registry_register_call(stmt) for stmt in tree.body) From 62ea8005868c0b129413c0753d0dee901e704cd1 Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 09:45:24 -0500 Subject: [PATCH 022/149] fix: recalculate safe_out from current input on each output-cap retry (#55546) The retry loop computed safe_out from the error's available_tokens, which reflected the *previous* request. Between retries the agent appends tool results and error text, so the real input token count grows. Deriving safe_out from the stale budget meant every retry still exceeded the context ceiling by 1+ tokens, burning through the 3-attempt limit. Compute safe_out from estimate_messages_tokens_rough(messages) so the cap tracks the growing input on each retry attempt. --- agent/conversation_loop.py | 14 ++++++- tests/test_ctx_halving_fix.py | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 75c07540d885..6487f050927e 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3475,12 +3475,22 @@ def run_conversation( # Error is purely about the output cap being too large. # Cap output to the available space and retry without # touching context_length or triggering compression. - safe_out = max(1, available_out - 64) # small safety margin + # + # The server's error reports available_tokens for the + # *previous* request. Between retries the agent appends + # tool results and error text, so the real input token + # count grows. Deriving safe_out from the error's + # available_tokens would keep reusing a stale budget and + # every retry would still exceed the ceiling by 1+ tokens. + # Compute safe_out from the *current* message token estimate + # so the cap tracks the growing input (#55546). + _current_input = estimate_messages_tokens_rough(messages) + safe_out = max(1, old_ctx - _current_input - 64) # small safety margin agent._ephemeral_max_output_tokens = safe_out agent._buffer_vprint( f"⚠️ Output cap too large for current prompt — " f"retrying with max_tokens={safe_out:,} " - f"(available_tokens={available_out:,}; context_length unchanged at {old_ctx:,})" + f"(current_input={_current_input:,}; context_length unchanged at {old_ctx:,})" ) # Still count against compression_attempts so we don't # loop forever if the error keeps recurring. diff --git a/tests/test_ctx_halving_fix.py b/tests/test_ctx_halving_fix.py index 63c965ac9658..9482d160a29d 100644 --- a/tests/test_ctx_halving_fix.py +++ b/tests/test_ctx_halving_fix.py @@ -360,3 +360,82 @@ class TestContextNotHalvedOnOutputCapError: available_out = parse_available_output_tokens_from_error(error_msg) safe_out = max(1, available_out - 64) assert safe_out == 1 + + +# --------------------------------------------------------------------------- +# Regression: safe_out must track growing input, not stale error budget +# --------------------------------------------------------------------------- + +class TestSafeOutTracksGrowingInput: + """Regression test for #55546: safe_out derived from stale available_tokens + reuses a budget computed from the *previous* request. Between retries the + agent appends tool results and error text, so the real input token count + grows. The fix computes safe_out from the *current* message token estimate + so the cap stays valid on every retry. + """ + + def _make_agent(self, context_length=262_144): + from run_agent import AIAgent + from agent.context_compressor import ContextCompressor + + agent = object.__new__(AIAgent) + agent.api_mode = "chat_completions" + agent.model = "qwen3.6-27b" + agent.base_url = "http://localhost:1234/v1" + agent.tools = [] + agent.max_tokens = 65_536 + agent.reasoning_config = None + agent._ephemeral_max_output_tokens = None + + compressor = MagicMock(spec=ContextCompressor) + compressor.context_length = context_length + agent.context_compressor = compressor + + agent._prepare_messages_for_api = MagicMock( + return_value=[{"role": "user", "content": "hi"}] + ) + agent._vprint = MagicMock() + agent.request_overrides = {} + return agent + + def test_safe_out_uses_current_input_not_stale_available(self): + """safe_out is computed from the current message token estimate, not + the stale available_tokens from the error message. + """ + from agent.model_metadata import estimate_messages_tokens_rough + + agent = self._make_agent(context_length=262_144) + old_ctx = agent.context_compressor.context_length + + # Simulate a conversation that's near the ceiling. + messages = [{"role": "user", "content": "x" * 800_000}] + _current_input = estimate_messages_tokens_rough(messages) + + # The fix: derive safe_out from the current input estimate. + safe_out = max(1, old_ctx - _current_input - 64) + agent._ephemeral_max_output_tokens = safe_out + + # Verify: safe_out is based on current input, not a stale error value. + assert agent._ephemeral_max_output_tokens == safe_out + assert agent.context_compressor.context_length == old_ctx + + def test_safe_out_tracks_growing_input(self): + """When messages grow between retries, safe_out shrinks accordingly.""" + from agent.model_metadata import estimate_messages_tokens_rough + + agent = self._make_agent(context_length=262_144) + old_ctx = agent.context_compressor.context_length + + # Initial messages. + messages = [{"role": "user", "content": "x" * 800_000}] + input_1 = estimate_messages_tokens_rough(messages) + safe_out_1 = max(1, old_ctx - input_1 - 64) + + # Simulate the agent appending a tool result (input grows). + messages.append({"role": "assistant", "content": "tool result" * 200}) + input_2 = estimate_messages_tokens_rough(messages) + safe_out_2 = max(1, old_ctx - input_2 - 64) + + # safe_out_2 must be <= safe_out_1 (shrinks as input grows). + assert safe_out_2 <= safe_out_1 + assert safe_out_2 >= 1 From 57f18148322cbb4be2a0cb78f8b611446d74ef3d Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 10:55:13 -0500 Subject: [PATCH 023/149] fix: use provider available_out + request estimate for output-cap retry cap The branch computed safe_out from estimate_messages_tokens_rough(messages), but the provider rejected the larger api_messages request (system prompt, injected context, tool schemas). When API-only content is large, safe_out could far exceed the provider's available_tokens. Compute safe_out from estimate_request_tokens_rough(api_messages, tools=...) and keep provider available_out as an upper bound. Do not alter context_length or trigger compression for output-cap errors. Add production-path run_conversation tests that assert the retry API call's max_tokens, including a case where a large system prompt makes messages-only estimation undercount the real request. Fixes #55546 --- agent/conversation_loop.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 6487f050927e..83681e426670 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3472,25 +3472,31 @@ def run_conversation( # context_length = total window (input + output combined). available_out = parse_available_output_tokens_from_error(error_msg) if available_out is not None: - # Error is purely about the output cap being too large. - # Cap output to the available space and retry without - # touching context_length or triggering compression. - # - # The server's error reports available_tokens for the - # *previous* request. Between retries the agent appends - # tool results and error text, so the real input token - # count grows. Deriving safe_out from the error's - # available_tokens would keep reusing a stale budget and - # every retry would still exceed the ceiling by 1+ tokens. - # Compute safe_out from the *current* message token estimate - # so the cap tracks the growing input (#55546). - _current_input = estimate_messages_tokens_rough(messages) - safe_out = max(1, old_ctx - _current_input - 64) # small safety margin + # This is an output-cap error, not input overflow. + # The provider's available_tokens is the authoritative + # cap for the failed request, so keep it as an upper + # bound. Also estimate the current API request shape + # (system prompt, injected context, tool schemas) because + # Hermes may add API-only content not present in persisted + # messages. Use the smaller budget and apply a small + # safety margin. Do not alter context_length. + request_input_estimate = estimate_request_tokens_rough( + api_messages, tools=agent.tools or None, + ) + local_available_out = old_ctx - request_input_estimate + if local_available_out > 0: + safe_out = max(1, min(available_out, local_available_out) - 64) + else: + # Rough local estimate can overshoot; provider truth + # still allows a minimal retry. + safe_out = max(1, available_out - 64) agent._ephemeral_max_output_tokens = safe_out agent._buffer_vprint( f"⚠️ Output cap too large for current prompt — " f"retrying with max_tokens={safe_out:,} " - f"(current_input={_current_input:,}; context_length unchanged at {old_ctx:,})" + f"(provider_available={available_out:,}, " + f"estimated_request_tokens={request_input_estimate:,}; " + f"context_length unchanged at {old_ctx:,})" ) # Still count against compression_attempts so we don't # loop forever if the error keeps recurring. From 7ef9345a551839ced721afe6ea970d1f0837e185 Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 11:22:55 -0500 Subject: [PATCH 024/149] test: add output-cap retry with compression disabled + fix request-pressure test --- tests/run_agent/test_run_agent.py | 225 ++++++++++++++++++++++++++++++ tests/test_ctx_halving_fix.py | 76 ---------- 2 files changed, 225 insertions(+), 76 deletions(-) diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 396657ea4e85..7c4ad7cb414b 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -5355,6 +5355,231 @@ class TestRunConversation: "_record_task_failure should not be called outside kanban mode" ) + # ── Output-cap retry: safe_out uses provider available_out + request estimate ── + + def test_output_cap_retry_uses_provider_available_out(self, agent): + """run_conversation retries an output-cap error with max_tokens <= + available_out - 64, and does NOT halve context_length or trigger + compression. + """ + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = True + agent.context_compressor.context_length = 200_000 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + error_msg = ( + "max_tokens: 65536 > context_window: 200000 " + "- input_tokens: 199000 = available_tokens: 1000" + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + assert second_call["max_tokens"] <= 936 + assert agent.context_compressor.context_length == 200_000 + mock_compress.assert_not_called() + + def test_output_cap_retry_with_large_api_only_content(self, agent): + """When a large system prompt makes api_messages huge while persisted + messages stay tiny, the retry cap must still respect provider + available_tokens — not blow up to the full context window. + """ + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = True + agent.context_compressor.context_length = 200_000 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + # Huge API-only system prompt; persisted messages are tiny. + agent._cached_system_prompt = "S" * 796_000 + + error_msg = ( + "max_tokens: 65536 > context_window: 200000 " + "- input_tokens: 199000 = available_tokens: 1000" + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + # The current branch (messages-only estimate) would send max_tokens + # near 199927 — this test fails on it. + assert second_call["max_tokens"] <= 936 + assert agent.context_compressor.context_length == 200_000 + mock_compress.assert_not_called() + + def test_output_cap_retry_request_pressure_lower_bound(self, agent): + """When the provider reports a large available_tokens but local request + pressure leaves less room, the retry cap is the smaller of the two. + """ + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = True + agent.context_compressor.context_length = 200_000 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + # A large API-only system prompt so the local estimate is the binding + # constraint, not the provider's available_tokens. + agent._cached_system_prompt = "S" * 796_000 + + error_msg = ( + "max_tokens: 65536 > context_window: 200000 " + "- input_tokens: 190000 = available_tokens: 50000" + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + first_call = agent.client.chat.completions.create.call_args_list[0].kwargs + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + + # Verify the local estimate is actually the lower bound. + from agent.model_metadata import estimate_request_tokens_rough + estimated_request = estimate_request_tokens_rough( + first_call["messages"], tools=agent.tools or None, + ) + local_available = 200_000 - estimated_request + expected_cap = max(1, min(50_000, local_available) - 64) + assert local_available < 50_000 + assert second_call["max_tokens"] == expected_cap + assert agent.context_compressor.context_length == 200_000 + mock_compress.assert_not_called() + + def test_output_cap_retry_safety_floor_at_one(self, agent): + """When provider available_tokens is 1, the retry cap is floored at 1.""" + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = True + agent.context_compressor.context_length = 200_000 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + error_msg = ( + "max_tokens: 65536 > context_window: 200000 " + "- input_tokens: 199999 = available_tokens: 1" + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + assert second_call["max_tokens"] == 1 + assert agent.context_compressor.context_length == 200_000 + mock_compress.assert_not_called() + + def test_output_cap_retry_with_compression_disabled(self, agent): + """Output-cap retry must still work when compression.enabled is false. + The recovery is a max_tokens-only retry — it does not require compression, + so the compression-disabled guard must not block it. + """ + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = False + agent.context_compressor.context_length = 200_000 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + error_msg = ( + "max_tokens: 65536 > context_window: 200000 " + "- input_tokens: 199000 = available_tokens: 1000" + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + # Two API calls: the failed one and the retried one. + assert len(agent.client.chat.completions.create.call_args_list) == 2 + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + assert result.get("compaction_disabled") is None + assert second_call["max_tokens"] <= 936 + assert agent.context_compressor.context_length == 200_000 + mock_compress.assert_not_called() + class TestHookPayloadSanitizesSimpleNamespace: """Regression: ``_hook_jsonable`` referenced ``SimpleNamespace`` without diff --git a/tests/test_ctx_halving_fix.py b/tests/test_ctx_halving_fix.py index 9482d160a29d..83b2d0e1d49c 100644 --- a/tests/test_ctx_halving_fix.py +++ b/tests/test_ctx_halving_fix.py @@ -362,80 +362,4 @@ class TestContextNotHalvedOnOutputCapError: assert safe_out == 1 -# --------------------------------------------------------------------------- -# Regression: safe_out must track growing input, not stale error budget -# --------------------------------------------------------------------------- -class TestSafeOutTracksGrowingInput: - """Regression test for #55546: safe_out derived from stale available_tokens - reuses a budget computed from the *previous* request. Between retries the - agent appends tool results and error text, so the real input token count - grows. The fix computes safe_out from the *current* message token estimate - so the cap stays valid on every retry. - """ - - def _make_agent(self, context_length=262_144): - from run_agent import AIAgent - from agent.context_compressor import ContextCompressor - - agent = object.__new__(AIAgent) - agent.api_mode = "chat_completions" - agent.model = "qwen3.6-27b" - agent.base_url = "http://localhost:1234/v1" - agent.tools = [] - agent.max_tokens = 65_536 - agent.reasoning_config = None - agent._ephemeral_max_output_tokens = None - - compressor = MagicMock(spec=ContextCompressor) - compressor.context_length = context_length - agent.context_compressor = compressor - - agent._prepare_messages_for_api = MagicMock( - return_value=[{"role": "user", "content": "hi"}] - ) - agent._vprint = MagicMock() - agent.request_overrides = {} - return agent - - def test_safe_out_uses_current_input_not_stale_available(self): - """safe_out is computed from the current message token estimate, not - the stale available_tokens from the error message. - """ - from agent.model_metadata import estimate_messages_tokens_rough - - agent = self._make_agent(context_length=262_144) - old_ctx = agent.context_compressor.context_length - - # Simulate a conversation that's near the ceiling. - messages = [{"role": "user", "content": "x" * 800_000}] - _current_input = estimate_messages_tokens_rough(messages) - - # The fix: derive safe_out from the current input estimate. - safe_out = max(1, old_ctx - _current_input - 64) - agent._ephemeral_max_output_tokens = safe_out - - # Verify: safe_out is based on current input, not a stale error value. - assert agent._ephemeral_max_output_tokens == safe_out - assert agent.context_compressor.context_length == old_ctx - - def test_safe_out_tracks_growing_input(self): - """When messages grow between retries, safe_out shrinks accordingly.""" - from agent.model_metadata import estimate_messages_tokens_rough - - agent = self._make_agent(context_length=262_144) - old_ctx = agent.context_compressor.context_length - - # Initial messages. - messages = [{"role": "user", "content": "x" * 800_000}] - input_1 = estimate_messages_tokens_rough(messages) - safe_out_1 = max(1, old_ctx - input_1 - 64) - - # Simulate the agent appending a tool result (input grows). - messages.append({"role": "assistant", "content": "tool result" * 200}) - input_2 = estimate_messages_tokens_rough(messages) - safe_out_2 = max(1, old_ctx - input_2 - 64) - - # safe_out_2 must be <= safe_out_1 (shrinks as input grows). - assert safe_out_2 <= safe_out_1 - assert safe_out_2 >= 1 From 127f6e15144dda80ece620a12984f4d54989072d Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 11:23:56 -0500 Subject: [PATCH 025/149] fix: exempt output-cap errors from compression-disabled guard --- agent/conversation_loop.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 83681e426670..5b4185a24124 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3061,14 +3061,21 @@ def run_conversation( # (``/new``), switch to a larger-context model, or reduce # attachments. Forced compaction via ``/compress`` # (``force=True``) is unaffected — it never reaches this loop. + # + # Output-cap errors (max_tokens too large) are NOT input + # overflow — the recovery is a max_tokens-only retry that + # does not require compression. Exempt them from this guard + # so the retry still fires even when compression is disabled. _overflow_reasons = { FailoverReason.long_context_tier, FailoverReason.payload_too_large, FailoverReason.context_overflow, } + _is_output_cap_error = is_output_cap_error(error_msg) if ( classified.reason in _overflow_reasons and not getattr(agent, "compression_enabled", True) + and not _is_output_cap_error ): agent._flush_status_buffer() agent._vprint( @@ -3487,8 +3494,10 @@ def run_conversation( if local_available_out > 0: safe_out = max(1, min(available_out, local_available_out) - 64) else: - # Rough local estimate can overshoot; provider truth - # still allows a minimal retry. + # The rough local estimate can overshoot the real + # request size. Fall back to the provider-reported + # budget, which is authoritative for the failed + # request. safe_out = max(1, available_out - 64) agent._ephemeral_max_output_tokens = safe_out agent._buffer_vprint( From 62c8574d86320e7b94994db46ece020f257ba16e Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 11:46:44 -0500 Subject: [PATCH 026/149] chore: remove trailing blank lines from test_ctx_halving_fix.py --- tests/test_ctx_halving_fix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_ctx_halving_fix.py b/tests/test_ctx_halving_fix.py index 83b2d0e1d49c..0a31126c6539 100644 --- a/tests/test_ctx_halving_fix.py +++ b/tests/test_ctx_halving_fix.py @@ -361,5 +361,3 @@ class TestContextNotHalvedOnOutputCapError: safe_out = max(1, available_out - 64) assert safe_out == 1 - - From af4006000fbe065790c68b98a660b2ba51038214 Mon Sep 17 00:00:00 2001 From: dmabry Date: Mon, 13 Jul 2026 11:47:37 -0500 Subject: [PATCH 027/149] chore: restore test_ctx_halving_fix.py to main --- tests/test_ctx_halving_fix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ctx_halving_fix.py b/tests/test_ctx_halving_fix.py index 0a31126c6539..63c965ac9658 100644 --- a/tests/test_ctx_halving_fix.py +++ b/tests/test_ctx_halving_fix.py @@ -360,4 +360,3 @@ class TestContextNotHalvedOnOutputCapError: available_out = parse_available_output_tokens_from_error(error_msg) safe_out = max(1, available_out - 64) assert safe_out == 1 - From 2ccfdb2db4eedf385f6c5b3fe722e183cee1b6de Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:50:15 +0530 Subject: [PATCH 028/149] fix(agent): exempt parseable vLLM/LM Studio output-cap errors from compression-disabled guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvage of #63862. is_output_cap_error() returns False for vLLM/LM Studio error messages that contain 'prompt contains ... input tokens' (treated as input-overflow signal). But parse_available_output_tokens_from_error() CAN extract a valid available_tokens from those same messages. The compression-disabled guard only checked is_output_cap_error(), so vLLM/LM Studio users with compression off still got a terminal failure instead of the max-tokens retry. Fix: also exempt when parse_available_output_tokens_from_error() returns a value — that function determines whether the retry path can actually handle the error, so it's the right predicate for the exemption. Added test: verify vLLM-format error with compression_disabled=False still triggers the max-tokens retry path. Co-authored-by: dmabry --- agent/conversation_loop.py | 5 +++- scripts/release.py | 1 + tests/run_agent/test_run_agent.py | 50 +++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 5b4185a24124..12c80a8bfb71 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3071,7 +3071,10 @@ def run_conversation( FailoverReason.payload_too_large, FailoverReason.context_overflow, } - _is_output_cap_error = is_output_cap_error(error_msg) + _is_output_cap_error = ( + is_output_cap_error(error_msg) + or parse_available_output_tokens_from_error(error_msg) is not None + ) if ( classified.reason in _overflow_reasons and not getattr(agent, "compression_enabled", True) diff --git a/scripts/release.py b/scripts/release.py index a40320329b32..d4ad39fb5e23 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -138,6 +138,7 @@ AUTHOR_MAP = { "259353979+testingbuddies24@users.noreply.github.com": "testingbuddies24", # PR #43192 salvage (strip orphan think-tag close tags in progressive gateway stream so a bare whose open was dropped upstream can't leak to the user) "shx_929@163.com": "Lazymonter", # PR #42914 salvage (retry launchd bootstrap after bootout on EIO for install/start instead of degrading to detached) "96322396+WXBR@users.noreply.github.com": "WXBR", # PR #46183 salvage (persist recovered final_response at the finalize_turn chokepoint so recovery-path breaks don't drop the delivered assistant row) + "dmabry@sparky.fabe-gray.ts.net": "dmabry", # PR #63862 salvage (output-cap retry: use provider available_tokens + request estimate; exempt parseable vLLM/LM Studio errors from compression-disabled guard) "sahil.rakhaiya117814@marwadiuniversity.ac.in": "SahilRakhaiya05", # PR #44073 salvage (fail-closed gateway/external-surface hardening: own-policy defaults, open-policy startup guard, profile-aware multiplex authz, API-server auth, execute_code per-session RPC token) "5848605+itenev@users.noreply.github.com": "itenev", # PR #22753 salvage (asyncify model-context resolution in gateway message path so blocking requests.get can't starve Discord heartbeats) "arthur.zhang@ingenico.com": "arthurzhang", # PR #34718 salvage (redact Slack App-Level xapp- tokens in agent/redact.py + gateway/run.py) diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 7c4ad7cb414b..afb45a53a71c 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -5580,6 +5580,56 @@ class TestRunConversation: assert agent.context_compressor.context_length == 200_000 mock_compress.assert_not_called() + def test_output_cap_retry_with_compression_disabled_vllm_format(self, agent): + """vLLM/LM Studio error messages contain 'prompt contains ... input + tokens' which is_output_cap_error() treats as an input-overflow signal + (returns False). But parse_available_output_tokens_from_error() CAN + extract a valid available_tokens from them. The compression-disabled + guard must exempt these too — otherwise users on vLLM/LM Studio with + compression off get a terminal failure instead of a max-tokens retry. + """ + self._setup_agent(agent) + agent.api_mode = "chat_completions" + agent.provider = "openrouter" + agent.model = "some/model" + agent.max_tokens = 65_536 + agent.compression_enabled = False + agent.context_compressor.context_length = 131_072 + agent.context_compressor.should_compress = MagicMock(return_value=False) + + # vLLM-format error (from tests/test_output_cap_parsing.py) + error_msg = ( + "This model's maximum context length is 131072 tokens. " + "However, you requested 1024 output tokens and your prompt " + "contains at least 65537 input tokens, for a total of at least " + "66561 tokens." + ) + exc = Exception(error_msg) + exc.status_code = 400 + exc.code = 400 + + ok_resp = _mock_response(content="done", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [exc, ok_resp] + + mock_compress = MagicMock() + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch.object(agent.context_compressor, "update_model"), + patch.object(agent, "_compress_context", mock_compress), + ): + result = agent.run_conversation("hello") + + assert len(agent.client.chat.completions.create.call_args_list) == 2 + second_call = agent.client.chat.completions.create.call_args_list[1].kwargs + assert result["completed"] is True + assert result.get("compaction_disabled") is None + # parse_available_output_tokens_from_error returns 65535 for this message + assert second_call["max_tokens"] <= 65471 # 65535 - 64 + assert agent.context_compressor.context_length == 131_072 + mock_compress.assert_not_called() + class TestHookPayloadSanitizesSimpleNamespace: """Regression: ``_hook_jsonable`` referenced ``SimpleNamespace`` without From 7e201fa1b6d0ac6ed9cdc6af609b9f9b8424843d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 13 Jul 2026 17:01:08 -0600 Subject: [PATCH 029/149] fix(nemo-relay): align dynamic plugin configuration Signed-off-by: Bryan Bednarski --- plugins/observability/nemo_relay/README.md | 65 +++++++++++++----- plugins/observability/nemo_relay/__init__.py | 66 +++++++++++++++---- tests/plugins/test_nemo_relay_plugin.py | 69 +++++++++++++++++++- tests/test_project_metadata.py | 4 +- 4 files changed, 171 insertions(+), 33 deletions(-) diff --git a/plugins/observability/nemo_relay/README.md b/plugins/observability/nemo_relay/README.md index e3bd2dd441bc..a1b90c4da4fa 100644 --- a/plugins/observability/nemo_relay/README.md +++ b/plugins/observability/nemo_relay/README.md @@ -84,15 +84,15 @@ wheel from this checkout, then install the official NeMo Relay runtime extra: ```bash uv build --wheel python -m pip install --force-reinstall dist/hermes_agent-*.whl -python -m pip install "nemo-relay>=0.5,<0.7" +python -m pip install "nemo-relay>=0.5,<1.0" hermes plugins enable observability/nemo_relay ``` The plugin fails open when `nemo-relay` is not installed. Install a supported -NeMo Relay 0.5 or 0.6 distribution: +NeMo Relay 0.x distribution beginning with 0.5: ```bash -pip install "nemo-relay>=0.5,<0.7" +pip install "nemo-relay>=0.5,<1.0" ``` ## Export Configuration @@ -190,23 +190,52 @@ session, turn, approval, and subagent marks; the plugin skips its manual NeMo Relay. `tool_parallelism.mode = "observe_only"` keeps tool scheduling observational while still wrapping the real execution boundary. -### Dynamic Plugins (NeMo Relay 0.6) +### Dynamic Plugins -Hermes can activate explicit native or worker plugins through the NeMo Relay -0.6 binding. Add `dynamic_plugins` entries to the same file; each entry uses -the binding's `plugin_id`, `kind`, `manifest_ref`, optional `environment_ref`, -and component `config` fields: +Hermes feature-detects the dynamic-plugin activation API available in NeMo Relay +0.6 and later. Configure native or worker plugins with Hermes-owned +`[[dynamic_plugins]]` entries that match the Python binding's activation-spec +fields: ```toml [[dynamic_plugins]] plugin_id = "example-plugin" kind = "rust_dynamic" -manifest_ref = "/absolute/path/to/example-plugin/relay-plugin.toml" +manifest_ref = "./example-plugin/relay-plugin.toml" [dynamic_plugins.config] mode = "enabled" ``` +For a worker plugin, also provide the lifecycle-managed `environment_ref`: + +```toml +[[dynamic_plugins]] +plugin_id = "example-worker" +kind = "worker" +manifest_ref = "./example-worker/relay-plugin.toml" +environment_ref = "/absolute/path/from-nemo-relay-plugins-inspect" + +[dynamic_plugins.config] +mode = "enabled" +``` + +Provision the worker first with `nemo-relay plugins add`, then copy +`data.source.environment_ref` from the JSON output of +`nemo-relay plugins inspect --json`. Relay rejects arbitrary Python +environments at activation time. + +Relative `manifest_ref` and `environment_ref` values resolve relative to the +physical `plugins.toml` file. + +Relay's canonical gateway `[[plugins.dynamic]]` records are not interchangeable +with this Hermes-owned section. The gateway combines those records with +separate lifecycle state for enablement, trust policy, and worker environments; +the Python binding does not yet expose that resolver. Hermes rejects +`[[plugins.dynamic]]` with an actionable diagnostic instead of silently +ignoring it or bypassing lifecycle policy. Use `[[dynamic_plugins]]` until Relay +exposes shared file-and-lifecycle resolution to embedding hosts. + Hermes activates these plugins before registering its managed LLM and tool execution middleware and retains the activation for the runtime lifetime. During shutdown it closes session exporters, flushes Relay subscribers, and @@ -214,22 +243,22 @@ then closes the activation so callbacks are removed before plugin code is unloaded. NeMo Relay 0.5 does not expose dynamic activation through its Python binding. -When `dynamic_plugins` is present with a 0.5 runtime, Hermes logs an actionable -warning and continues with the ordinary static component configuration, so -ATOF and ATIF observability remain available. No dynamic plugin is loaded in -that degraded mode. +When dynamic plugin configuration is present with a binding that lacks the +activation API, Hermes logs an actionable warning and continues with the +ordinary static component configuration, so ATOF and ATIF observability remain +available. No dynamic plugin is loaded in that degraded mode. For the full generic Hermes middleware contract, see [`docs/middleware/README.md`](../../../docs/middleware/README.md). ## Canonical Local Examples -The observe-only examples in this section use a supported NeMo Relay 0.5 or -0.6 distribution and a local Ollama model served through the OpenAI-compatible -API. +The observe-only examples in this section use a supported NeMo Relay 0.x +distribution beginning with 0.5 and a local Ollama model served through the +OpenAI-compatible API. ```bash -pip install "nemo-relay>=0.5,<0.7" +pip install "nemo-relay>=0.5,<1.0" export HERMES_HOME=/tmp/hermes-nemo-relay-docs/hermes-home mkdir -p "$HERMES_HOME" @@ -476,7 +505,7 @@ for the same execution. This example enables both NeMo Relay observability export and adaptive execution middleware for a local Hermes run. This path requires a NeMo Relay runtime that supports `[components.config.tool_parallelism]`, as provided by the supported -0.5 and 0.6 releases. +0.x release range beginning with 0.5. ```bash export HERMES_HOME=/tmp/hermes-middleware-test/hermes-home diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index 92ead8341931..56c6140ca58c 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -106,9 +106,9 @@ class _Runtime: ) else: logger.warning( - "NeMo Relay dynamic plugins require nemo-relay>=0.6,<0.7; this binding does " - "not expose plugin.activate_dynamic_plugins. Continuing with static " - "observability only." + "NeMo Relay dynamic plugins require a binding that exposes " + "plugin.activate_dynamic_plugins (available in NeMo Relay 0.6+). " + "Continuing with static observability only." ) initialize = getattr(plugin_mod, "initialize", None) if not callable(initialize): @@ -774,7 +774,7 @@ def _load_settings() -> _Settings: return _Settings( plugins_toml_path=plugins_toml_path, plugins_config=plugins_config, - dynamic_plugins=_dynamic_plugin_specs(plugins_config), + dynamic_plugins=_dynamic_plugin_specs(plugins_config, plugins_toml_path), adaptive_enabled=adaptive_config is not None, adaptive_mode=_adaptive_mode(adaptive_config), atof_enabled=_env_bool("HERMES_NEMO_RELAY_ATOF_ENABLED"), @@ -792,14 +792,40 @@ def _load_settings() -> _Settings: def _static_plugin_config(plugins_config: dict[str, Any]) -> dict[str, Any]: - """Return Relay's base plugin config without Hermes-owned host fields.""" - return {key: value for key, value in plugins_config.items() if key != "dynamic_plugins"} + """Return Relay's base config without embedding- or gateway-host fields.""" + return { + key: value + for key, value in plugins_config.items() + if key not in {"dynamic_plugins", "plugins"} + } -def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[str, Any]]: +def _dynamic_plugin_specs( + plugins_config: dict[str, Any] | None, + plugins_toml_path: str = "", +) -> list[dict[str, Any]]: if not isinstance(plugins_config, dict): return [] + raw_specs = plugins_config.get("dynamic_plugins") + plugins_section = plugins_config.get("plugins") + if plugins_section is not None: + if not isinstance(plugins_section, dict): + logger.error( + "Invalid NeMo Relay plugins config: expected [plugins] to be an object; " + "no dynamic plugins will be activated. Continuing with static " + "observability only." + ) + return [] + if plugins_section: + logger.error( + "Hermes cannot activate Relay gateway [[plugins.dynamic]] records because " + "the Python binding does not expose the CLI lifecycle resolver for " + "enablement, trust policy, and worker environments. Use Hermes-owned " + "[[dynamic_plugins]] activation specs instead; no dynamic plugins will be " + "activated. Continuing with static observability only." + ) + return [] if raw_specs is None: return [] if not isinstance(raw_specs, list): @@ -847,21 +873,26 @@ def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[st ) invalid = True continue - if environment_ref is not None and not isinstance(environment_ref, str): + if environment_ref is not None and ( + not isinstance(environment_ref, str) or not environment_ref.strip() + ): logger.warning( - "Invalid NeMo Relay dynamic_plugins[%d]: environment_ref must be a string", + "Invalid NeMo Relay dynamic_plugins[%d]: environment_ref must be a " + "non-empty string", index, ) invalid = True continue spec: dict[str, Any] = { - "plugin_id": plugin_id, + "plugin_id": plugin_id.strip(), "kind": kind, - "manifest_ref": manifest_ref, + "manifest_ref": _config_relative_path(manifest_ref.strip(), plugins_toml_path), "config": config, } if environment_ref is not None: - spec["environment_ref"] = environment_ref + spec["environment_ref"] = _config_relative_path( + environment_ref.strip(), plugins_toml_path + ) specs.append(spec) if invalid: logger.error( @@ -872,6 +903,17 @@ def _dynamic_plugin_specs(plugins_config: dict[str, Any] | None) -> list[dict[st return specs +def _config_relative_path(value: str, plugins_toml_path: str) -> str: + """Resolve a plugin path relative to its physical ``plugins.toml`` file.""" + path = Path(value) + if path.is_absolute(): + return str(path) + config_path = Path(plugins_toml_path) if plugins_toml_path else Path.cwd() / "plugins.toml" + if not config_path.is_absolute(): + config_path = Path.cwd() / config_path + return os.path.abspath(config_path.parent / path) + + def _flush_relay_subscribers(nemo_relay: Any) -> None: subscribers = getattr(nemo_relay, "subscribers", None) flush = getattr(subscribers, "flush", None) diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 4bc31f0ae3be..61958194e585 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -603,6 +603,73 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") +def test_nemo_relay_rejects_gateway_dynamic_config_with_actionable_diagnostic( + tmp_path, monkeypatch, caplog +): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + plugins_toml = tmp_path / "plugins.toml" + plugins_toml.write_text( + """ +version = 1 + +[[plugins.dynamic]] +manifest = "plugins/fixture/relay-plugin.toml" + +[plugins.dynamic.config] +mode = "test" +""", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) + + with caplog.at_level("ERROR"): + plugin.on_session_start(session_id="s1") + + assert not any(event[0] == "plugin.activate_dynamic" for event in fake.events) + initialize = next(event for event in fake.events if event[0] == "plugin.initialize") + assert initialize[1] == {"version": 1} + assert "does not expose the CLI lifecycle resolver" in caplog.text + assert "Use Hermes-owned [[dynamic_plugins]]" in caplog.text + + +def test_nemo_relay_explicit_dynamic_paths_resolve_from_plugins_toml(tmp_path, monkeypatch): + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + config_dir = tmp_path / "config" + config_dir.mkdir() + plugins_toml = config_dir / "plugins.toml" + plugins_toml.write_text( + """ +version = 1 + +[[dynamic_plugins]] +plugin_id = "worker-fixture" +kind = "worker" +manifest_ref = "../plugins/worker/relay-plugin.toml" +environment_ref = "../environments/worker-fixture" +""", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) + + plugin.on_session_start(session_id="s1") + + activation = next(event for event in fake.events if event[0] == "plugin.activate_dynamic") + assert activation[2] == [ + { + "plugin_id": "worker-fixture", + "kind": "worker", + "manifest_ref": str(tmp_path / "plugins" / "worker" / "relay-plugin.toml"), + "environment_ref": str(tmp_path / "environments" / "worker-fixture"), + "config": {}, + } + ] + runtime = plugin._get_runtime() + assert runtime is not None + runtime.shutdown() + + @pytest.mark.parametrize( ("provider", "api_mode", "expected_surface", "should_rewrite"), [ @@ -763,7 +830,7 @@ def test_nemo_relay_plugin_degrades_to_static_config_on_relay_0_5( initialize = next(event for event in fake.events if event[0] == "plugin.initialize") assert "dynamic_plugins" not in initialize[1] assert not any(event[0] == "plugin.activate_dynamic" for event in fake.events) - assert "require nemo-relay>=0.6,<0.7" in caplog.text + assert "available in NeMo Relay 0.6+" in caplog.text def test_nemo_relay_plugin_rejects_invalid_dynamic_specs(tmp_path, monkeypatch, caplog): diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index f2f4887d6091..8c0836e9059e 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -222,10 +222,10 @@ def test_feishu_extra_includes_qrcode_for_qr_login(): assert any(dep.startswith("qrcode") for dep in feishu_extra) -def test_nemo_relay_extra_uses_official_0_3_distribution(): +def test_nemo_relay_extra_uses_supported_official_distribution_range(): optional_dependencies = _load_optional_dependencies() - assert optional_dependencies["nemo-relay"] == ["nemo-relay==0.3"] + assert optional_dependencies["nemo-relay"] == ["nemo-relay>=0.5,<1.0"] assert not any( spec == "hermes-agent[nemo-relay]" for spec in optional_dependencies["all"] From 4a58e22e997ec56802fa93d4ca503a3767a02c4a Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Mon, 13 Jul 2026 20:00:32 -0400 Subject: [PATCH 030/149] fix(windows): put Git Bash coreutils on PATH for the non-login fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #63955 made Hermes survive a broken `bash -l` (Ainz's `Directory \drivers\etc does not exist`) by falling back to non-login `bash -c`. But a non-login shell never sources /etc/profile, so it never gets `…\usr\bin` on PATH — and that dir holds every coreutil the file/terminal tools shell out to (cat, mktemp, mv, wc, head, stat, chmod, mkdir, find). Result: `write_file` returned bytes_written:0 with an EMPTY error (the failure text went to a missing binary's stderr) and terminal commands exited 127. The survive-broken-login-bash fix was only half-done: it stopped crashing but silently failed every write. Derive Git Bash's bin dirs (mingw64/bin, usr/bin, bin, …) from the resolved bash.exe and prepend them to the subprocess PATH on Windows, in /etc/profile precedence order so coreutils win over same-named System32 tools (find.exe, sort.exe) inside the shell. No-op off Windows and when a login snapshot is healthy (the snapshot re-exports the full PATH inside the shell), so this only bites on the broken-login fallback path. Adds _git_bash_bin_dirs() (derivation, cached) + _prepend_git_bash_dirs() (PATH merge), plus regression tests for PortableGit/MinGit layouts and the run-env injection ordering. --- tests/tools/test_local_env_windows_msys.py | 85 ++++++++++++++++++++++ tools/environments/local.py | 83 +++++++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/tests/tools/test_local_env_windows_msys.py b/tests/tools/test_local_env_windows_msys.py index 938cd8783f18..f5eba3f252a8 100644 --- a/tests/tools/test_local_env_windows_msys.py +++ b/tests/tools/test_local_env_windows_msys.py @@ -18,6 +18,7 @@ and ``os.path.isdir`` so the MSYS path tests as "missing" exactly like on the real OS. """ +import os from unittest.mock import patch from tools.environments.base import BaseEnvironment @@ -25,8 +26,10 @@ from tools.environments import local as local_mod from tools.environments.local import ( LocalEnvironment, _bash_safe_path, + _git_bash_bin_dirs, _make_run_env, _msys_to_windows_path, + _prepend_git_bash_dirs, _quote_bash_path, _resolve_safe_cwd, _sanitize_subprocess_env, @@ -325,6 +328,88 @@ class TestWindowsMsysPathconvDefaults: assert run_env.get("MSYS2_ARG_CONV_EXCL") == "/custom" +# --------------------------------------------------------------------------- +# Git Bash coreutils on PATH — non-login ``bash -c`` fallback (empty +# write_file error / terminal exit 127 when login bash is broken) +# --------------------------------------------------------------------------- + +class TestGitBashCoreutilsOnPath: + def _fake_isdir(self, existing): + existing = {e.replace("\\", "/") for e in existing} + return lambda p: p.replace("\\", "/") in existing + + def test_derives_dirs_from_portablegit_layout(self, monkeypatch): + monkeypatch.setattr(local_mod, "_IS_WINDOWS", True) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", None) + monkeypatch.setattr(local_mod, "_find_bash", lambda: "/pg/bin/bash.exe") + existing = {"/pg/mingw64/bin", "/pg/usr/bin", "/pg/bin"} + monkeypatch.setattr(local_mod.os.path, "isdir", self._fake_isdir(existing)) + + dirs = _git_bash_bin_dirs() + + # usr/bin is the load-bearing coreutils dir; mingw64 precedes it. + assert "/pg/usr/bin" in dirs + assert dirs.index("/pg/mingw64/bin") < dirs.index("/pg/usr/bin") + # Non-existent dirs (mingw32, usr/local/bin) are excluded. + assert "/pg/mingw32/bin" not in dirs + + def test_derives_dirs_from_mingit_usr_bin_layout(self, monkeypatch): + monkeypatch.setattr(local_mod, "_IS_WINDOWS", True) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", None) + monkeypatch.setattr(local_mod, "_find_bash", lambda: "/mg/usr/bin/bash.exe") + existing = {"/mg/usr/bin", "/mg/mingw64/bin"} + monkeypatch.setattr(local_mod.os.path, "isdir", self._fake_isdir(existing)) + + dirs = _git_bash_bin_dirs() + + # MinGit ships bash under usr\bin; root must still resolve to /mg. + assert "/mg/usr/bin" in dirs + assert "/mg/mingw64/bin" in dirs + + def test_empty_off_windows(self, monkeypatch): + monkeypatch.setattr(local_mod, "_IS_WINDOWS", False) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", None) + assert _git_bash_bin_dirs() == [] + + def test_empty_when_bash_unresolvable(self, monkeypatch): + monkeypatch.setattr(local_mod, "_IS_WINDOWS", True) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", None) + + def boom(): + raise RuntimeError("Git Bash not found") + + monkeypatch.setattr(local_mod, "_find_bash", boom) + assert _git_bash_bin_dirs() == [] + + def test_prepend_is_idempotent(self, monkeypatch): + # Simulate Windows' ``;`` separator so drive-letter colons in fake + # paths don't collide with the POSIX ``:`` pathsep on the test host. + monkeypatch.setattr(os, "pathsep", ";") + monkeypatch.setattr(local_mod, "_IS_WINDOWS", True) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", ["/pg/usr/bin", "/pg/bin"]) + already = r"/pg/usr/bin;C:\Windows\System32;/pg/bin" + assert _prepend_git_bash_dirs(already) == already + + def test_make_run_env_prepends_coreutils_on_windows(self, monkeypatch): + monkeypatch.setattr(os, "pathsep", ";") + monkeypatch.setattr(local_mod, "_IS_WINDOWS", True) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", ["/pg/mingw64/bin", "/pg/usr/bin"]) + run_env = _make_run_env({"PATH": r"C:\Windows\System32"}) + path = run_env.get("PATH") or run_env.get("Path") + entries = path.split(";") + # Coreutils dirs land before System32 so bash resolves cat/find/sort + # to the GNU tools, not the same-named Windows executables. + assert "/pg/usr/bin" in entries + assert entries.index("/pg/usr/bin") < entries.index(r"C:\Windows\System32") + + def test_make_run_env_noop_on_posix(self, monkeypatch): + monkeypatch.setattr(local_mod, "_IS_WINDOWS", False) + monkeypatch.setattr(local_mod, "_git_bash_bin_dirs_cache", None) + run_env = _make_run_env({"PATH": "/usr/bin:/bin"}) + # No Windows git dirs injected on POSIX. + assert "mingw64" not in run_env["PATH"] + + # --------------------------------------------------------------------------- # Command wrapping — native Windows cwd must be Git Bash-friendly for cd # --------------------------------------------------------------------------- diff --git a/tools/environments/local.py b/tools/environments/local.py index e5f6c6c2b262..0a867fa984bd 100644 --- a/tools/environments/local.py +++ b/tools/environments/local.py @@ -658,6 +658,82 @@ def _bash_starts(bash: str) -> bool: return ok +_git_bash_bin_dirs_cache: "list[str] | None" = None + + +def _git_bash_bin_dirs() -> list[str]: + """Git Bash's coreutils/binary dirs, in ``/etc/profile`` precedence order. + + A non-login ``bash -c`` (the fallback used when ``bash -l`` is broken — + the classic Windows ``Directory \\drivers\\etc does not exist`` failure) + never sources ``/etc/profile``, so it never gets ``…\\usr\\bin`` on PATH. + That directory holds every coreutil the file/terminal tools shell out to + (``cat``, ``mktemp``, ``mv``, ``wc``, ``head``, ``stat``, ``chmod``, + ``mkdir``, ``find`` …). Without it, ``write_file`` fails with an empty + error (the failure text went to a missing binary's stderr) and terminal + commands exit 127. We derive these dirs from the resolved ``bash.exe`` so + the fallback shell can find coreutils regardless of the login shell. + + Returns ``[]`` off Windows or when bash can't be located. Dirs are + returned in the order Git Bash's own ``/etc/profile`` prepends them + (mingw first, then usr/bin, then bin) and only if they exist on disk. + """ + global _git_bash_bin_dirs_cache + if _git_bash_bin_dirs_cache is not None: + return _git_bash_bin_dirs_cache + + if not _IS_WINDOWS: + _git_bash_bin_dirs_cache = [] + return _git_bash_bin_dirs_cache + + dirs: list[str] = [] + try: + bash = _find_bash() + except Exception: + _git_bash_bin_dirs_cache = [] + return _git_bash_bin_dirs_cache + + bin_dir = os.path.dirname(bash) # \bin or \usr\bin + parent = os.path.dirname(bin_dir) + # MinGit ships bash under usr\bin; PortableGit/system Git under bin. + root = os.path.dirname(parent) if os.path.basename(parent).lower() == "usr" else parent + + # Order mirrors Git-for-Windows /etc/profile so coreutils win over the + # same-named Windows System32 tools (find.exe, sort.exe) inside the shell. + for candidate in ( + os.path.join(root, "mingw64", "bin"), + os.path.join(root, "mingw32", "bin"), + os.path.join(root, "usr", "local", "bin"), + os.path.join(root, "usr", "bin"), + os.path.join(root, "bin"), + ): + if os.path.isdir(candidate) and candidate not in dirs: + dirs.append(candidate) + + _git_bash_bin_dirs_cache = dirs + return dirs + + +def _prepend_git_bash_dirs(existing_path: str) -> str: + """Prepend Git Bash's binary dirs to ``existing_path`` if missing. + + No-op off Windows or when the dirs can't be resolved. First-occurrence + wins, so a PATH that already lists a dir keeps its position. This is what + lets the non-login ``bash -c`` fallback find coreutils; in the healthy + case the session snapshot re-exports the full login PATH inside the shell, + so this only matters when that snapshot is absent. + """ + git_dirs = _git_bash_bin_dirs() + if not git_dirs: + return existing_path + sep = os.pathsep + entries = [e for e in existing_path.split(sep) if e] if existing_path else [] + missing = [d for d in git_dirs if d not in entries] + if not missing: + return existing_path + return sep.join([*missing, *entries]) + + # POSIX-sh-family shells that understand the ``[shell, "-lic", "set +m; …"]`` # invocation spawn_local uses. $SHELL values outside this set (fish, csh/tcsh, # nushell, elvish, xonsh, …) would error on that syntax, so _find_shell falls @@ -904,6 +980,13 @@ def _make_run_env(env: dict) -> dict: path_key = _path_env_key(run_env) if path_key is not None: new_path = _append_missing_sane_path_entries(run_env.get(path_key, "")) + # On Windows, ensure Git Bash's coreutils dirs (…\usr\bin etc.) are on + # PATH. A non-login ``bash -c`` fallback (used when ``bash -l`` is + # broken) never sources /etc/profile, so without this cat/mktemp/mv and + # friends are missing and every write_file/terminal call fails (empty + # error / exit 127). No-op off Windows and when a login snapshot is + # healthy (the snapshot re-exports the full PATH inside the shell). + new_path = _prepend_git_bash_dirs(new_path) # Ensure the hermes install dir is reachable so plugins can shell out # to bare ``hermes`` via the terminal tool even when the gateway was # launched without it on PATH (systemd, service managers, cron, etc.). From 19641fab72fd91dadb54148001a48d0fecf8554d Mon Sep 17 00:00:00 2001 From: brooklyn! Date: Mon, 13 Jul 2026 20:38:26 -0400 Subject: [PATCH 031/149] fix(desktop): render reasoning text in the Thinking widget (#63999) --- .../components/assistant-ui/markdown-text.tsx | 118 +----------------- .../assistant-ui/thread/message-parts.tsx | 19 ++- 2 files changed, 19 insertions(+), 118 deletions(-) diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.tsx index bceadb8e6ca0..694a94e78c75 100644 --- a/apps/desktop/src/components/assistant-ui/markdown-text.tsx +++ b/apps/desktop/src/components/assistant-ui/markdown-text.tsx @@ -15,7 +15,6 @@ import { useDeferredValue, useEffect, useMemo, - useRef, useState } from 'react' @@ -348,112 +347,6 @@ function MarkdownImage({ className, src, alt, ...props }: ComponentProps<'img'>) ) } -// Steady character-reveal for streaming text: decouples visible cadence from -// bursty arrival so text flows instead of popping (cf. assistant-ui's useSmooth, -// reimplemented for a tunable rate). Proportional drain — each frame reveals a -// slice of the backlog so the reveal converges within ~REVEAL_DRAIN_MS whatever -// the size; the per-frame cap stops a huge dump rendering as one slab. The loop -// is gated on backlog, not isRunning, so a stream that completes mid-reveal -// keeps draining its tail instead of snapping. -const REVEAL_DRAIN_MS = 500 -const REVEAL_MAX_CHARS_PER_FRAME = 30 -// Floor between reveal commits. Each commit republishes the text context and -// re-runs the whole Streamdown pipeline (preprocess → remend → lex → micromark -// on the open block) over the full accumulated text — at raw rAF cadence -// that's 60 full parses/second and was the dominant streaming cost for -// reasoning text. ~33ms keeps the reveal visually fluid (2 frames) while -// halving the parse work. -const REVEAL_MIN_COMMIT_MS = 33 - -function useSmoothReveal(text: string, isRunning: boolean): string { - const [displayed, setDisplayed] = useState(isRunning ? '' : text) - const targetRef = useRef(text) - const shownRef = useRef(displayed) - const frameRef = useRef(null) - const lastTickRef = useRef(0) - - shownRef.current = displayed - targetRef.current = text - - useEffect(() => { - if (typeof window === 'undefined') { - return - } - - // Non-extending change (regenerate / branch / history swap): restart from - // empty while streaming, else snap to the replacement. - if (!text.startsWith(shownRef.current)) { - shownRef.current = isRunning ? '' : text - setDisplayed(shownRef.current) - } - - if (shownRef.current.length >= text.length || frameRef.current !== null) { - return - } - - lastTickRef.current = performance.now() - - const tick = () => { - const now = performance.now() - const dt = now - lastTickRef.current - - // Skip this frame if the floor hasn't elapsed — the backlog math below - // is dt-proportional, so delayed commits reveal proportionally more. - if (dt < REVEAL_MIN_COMMIT_MS) { - frameRef.current = requestAnimationFrame(tick) - - return - } - - lastTickRef.current = now - - const remaining = targetRef.current.length - shownRef.current.length - - const add = Math.min( - remaining, - // dt-scaled so the per-commit cap stays equivalent to the old - // per-frame cap at any commit cadence. - Math.ceil((REVEAL_MAX_CHARS_PER_FRAME * dt) / 16.7), - Math.max(1, Math.ceil((remaining * dt) / REVEAL_DRAIN_MS)) - ) - - shownRef.current = targetRef.current.slice(0, shownRef.current.length + add) - setDisplayed(shownRef.current) - - frameRef.current = shownRef.current.length < targetRef.current.length ? requestAnimationFrame(tick) : null - } - - frameRef.current = requestAnimationFrame(tick) - }, [text, isRunning]) - - useEffect( - () => () => { - if (frameRef.current !== null && typeof window !== 'undefined') { - cancelAnimationFrame(frameRef.current) - } - }, - [] - ) - - return displayed -} - -// Re-publish the part context with a smooth character-reveal, above -// DeferStreamingText so the reveal feeds the deferred markdown pipeline. Status -// stays running while revealing so the caret persists past the underlying part -// settling. -function SmoothStreamingText({ children }: { children: ReactNode }) { - const { text, status } = useMessagePartText() - const isRunning = status.type === 'running' - const revealed = useSmoothReveal(text, isRunning) - - return ( - - {children} - - ) -} - /** * Re-publish the active message-part context with React's `useDeferredValue` * applied to the streaming text and status. The outer wrapper still re-renders @@ -694,13 +587,14 @@ interface MarkdownTextContentProps extends MarkdownTextSurfaceProps { } export function MarkdownTextContent({ isRunning, text, ...surfaceProps }: MarkdownTextContentProps) { + // Same path as the assistant answer. A reasoning-only smoothing wrapper used + // to sit here but stalled its char-reveal at empty (the part stays running + // the whole message), blanking the Thinking widget. return ( - - - - - + + + ) } diff --git a/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx b/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx index ccd672dac9f2..0e1a2e7e0882 100644 --- a/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx @@ -1,4 +1,9 @@ -import { type ToolCallMessagePartProps, useAuiState } from '@assistant-ui/react' +import { + type ReasoningMessagePartComponent, + type ToolCallMessagePartProps, + useAuiState, + useMessagePartReasoning +} from '@assistant-ui/react' import { type ComponentProps, type FC, type ReactNode, useEffect, useRef, useState } from 'react' import { ClarifyTool } from '@/components/assistant-ui/clarify-tool' @@ -174,17 +179,19 @@ const ReasoningAccordionGroup: FC<{ children?: ReactNode; endIndex: number; star ) } -const ReasoningTextPart: FC<{ text: string; status?: { type: string } }> = ({ text, status }) => { - const displayText = text.trimStart() +// Read the part from context, same contract as MarkdownText's +// useMessagePartText — the reasoning-only smoothing wrapper (removed) stalled +// the char-reveal at empty, blanking the widget. +const ReasoningTextPart: ReasoningMessagePartComponent = () => { + const { status, text } = useMessagePartReasoning() const messageRunning = useAuiState(s => s.message.status?.type === 'running') - const isRunning = status?.type === 'running' || messageRunning return ( } - isRunning={isRunning} - text={displayText} + isRunning={status.type === 'running' || messageRunning} + text={text.trimStart()} /> ) } From 861d69c7bba8d2ea6a1cd170e989c901c74d32d1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:56:51 -0700 Subject: [PATCH 032/149] fix(dashboard): keep memory.provider in the config schema so Desktop's dropdown survives (#63886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard's dedicated memory-provider UI (4b184cbe5) excluded memory.provider from /api/config/schema server-side. Desktop's settings page builds its field list from that schema, so the Memory Provider dropdown silently vanished from Desktop after v0.18.1. - web_server.py: restore memory.provider as a select in _SCHEMA_OVERRIDES, with options built from plugins.memory discovery (was a stale hardcoded [builtin, honcho] list before the removal) - plugins/memory: add list_memory_provider_names() — directory-scan-only name listing, safe at module import time (no provider imports) - web ConfigPage: hide memory.provider client-side instead — the Plugins page owns the dedicated provider-switching UI there - tests: schema contract (select present, category memory, builtin sentinel) + invariant that every discoverable provider is selectable --- hermes_cli/web_server.py | 25 ++++++++++++++++++++++++- plugins/memory/__init__.py | 11 +++++++++++ tests/hermes_cli/test_web_server.py | 28 ++++++++++++++++++++++++++++ web/src/pages/ConfigPage.tsx | 11 ++++++++++- 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 7365d241c09f..fed4b066d290 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -610,7 +610,30 @@ async def _token_auth_seam(request: Request, call_next): # --------------------------------------------------------------------------- # Manual overrides for fields that need select options or custom types +def _memory_provider_options() -> List[str]: + """Discovered memory providers for the ``memory.provider`` select. + + Directory-scan only (no provider imports), so it's safe at module import + time. ``""`` (built-in) is always first; discovery failures degrade to the + bundled defaults rather than dropping the field. + """ + options = ["", "builtin"] + try: + from plugins.memory import list_memory_provider_names + + options.extend(list_memory_provider_names()) + except Exception: + options.extend(["honcho"]) + # Dedupe, preserve order + return list(dict.fromkeys(options)) + + _SCHEMA_OVERRIDES: Dict[str, Dict[str, Any]] = { + "memory.provider": { + "type": "select", + "description": "Memory provider plugin", + "options": _memory_provider_options(), + }, "model": { "type": "string", "description": "Default model (e.g. anthropic/claude-sonnet-4.6)", @@ -780,7 +803,7 @@ def _build_schema_from_config( full_key = f"{prefix}.{key}" if prefix else key # Skip internal / version keys - if full_key in {"_config_version", "memory.provider"}: + if full_key in {"_config_version"}: continue # Category is the first path component for nested keys, or "general" diff --git a/plugins/memory/__init__.py b/plugins/memory/__init__.py index 3f92b5cbb2a3..cccda75ce849 100644 --- a/plugins/memory/__init__.py +++ b/plugins/memory/__init__.py @@ -143,6 +143,17 @@ def find_provider_dir(name: str) -> Optional[Path]: # Public API # --------------------------------------------------------------------------- +def list_memory_provider_names() -> List[str]: + """Cheap name-only listing of discoverable memory providers. + + Unlike :func:`discover_memory_providers`, this does NOT import provider + modules or run availability checks — it's a directory scan only, safe to + call at module-import time (e.g. when building the dashboard config + schema). + """ + return sorted({name for name, _ in _iter_provider_dirs()}) + + def discover_memory_providers() -> List[Tuple[str, str, bool]]: """Scan bundled and user-installed directories for available providers. diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index f88d60c56c57..19d379dfcd0d 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -3645,6 +3645,34 @@ class TestBuildSchemaFromConfig: assert "options" in entry assert "local" in entry["options"] + def test_memory_provider_field_present_as_select(self): + """memory.provider must stay in the config schema. + + Desktop's settings page builds its field list from /api/config/schema — + a key excluded here silently vanishes from Desktop's Memory section + (regression: the dashboard's dedicated memory-provider UI excluded the + key server-side, breaking Desktop's dropdown). The dashboard hides the + field client-side instead. + """ + from hermes_cli.web_server import CONFIG_SCHEMA + entry = CONFIG_SCHEMA["memory.provider"] + assert entry["type"] == "select" + assert entry["category"] == "memory" + options = entry["options"] + # Built-in sentinel first, plus at least one discovered provider. + assert options[0] == "" + assert "builtin" in options + assert len(options) >= 3 + + def test_memory_provider_options_cover_discovered_providers(self): + """Every provider the /api/memory endpoint can activate is selectable.""" + from hermes_cli.web_server import CONFIG_SCHEMA + from plugins.memory import list_memory_provider_names + + options = set(CONFIG_SCHEMA["memory.provider"]["options"]) + missing = set(list_memory_provider_names()) - options + assert missing == set(), f"discovered providers missing from schema options: {missing}" + def test_approvals_mode_options_match_config_values(self): """approvals.mode select options must match the values accepted by config.py. diff --git a/web/src/pages/ConfigPage.tsx b/web/src/pages/ConfigPage.tsx index c6bff27f1d78..dcb88282002c 100644 --- a/web/src/pages/ConfigPage.tsx +++ b/web/src/pages/ConfigPage.tsx @@ -169,7 +169,16 @@ export default function ConfigPage() { api .getSchema() .then((resp) => { - setSchema(resp.fields as Record>); + // memory.provider has a dedicated management UI on the Plugins page + // (provider cards + guided setup/switch flow). Hide it from the + // generic config form so the two surfaces don't fight; the schema + // keeps the field for other consumers (Desktop settings). + const fields = { ...resp.fields } as Record< + string, + Record + >; + delete fields["memory.provider"]; + setSchema(fields); setCategoryOrder(resp.category_order ?? []); }) .catch(() => {}); From c7e09f25716764b2e4dacf518f1c553a497c15d7 Mon Sep 17 00:00:00 2001 From: Erosika Date: Thu, 9 Jul 2026 17:01:35 -0400 Subject: [PATCH 033/149] fix(desktop): restore curated declared schema for the provider panel The desktop provider panel previously rendered the curated declarations from hermes_cli/memory_providers.py: five hindsight fields, and no panel at all for undeclared providers like honcho (OAuth connect only). The dashboard provider-switching rework re-pointed the shared config route at raw plugin schemas, so the desktop began dumping every internal field (35 for hindsight) and grew a bespoke honcho panel. Serve both surfaces from the same route: ?surface=declared returns the curated schema (empty for undeclared providers) with the original config-file + env-store write semantics; the dashboard keeps the raw plugin schema unchanged. The desktop client opts into declared. --- apps/desktop/src/hermes.ts | 5 +- hermes_cli/web_server.py | 143 +++++++++++++++++++++++++++- tests/hermes_cli/test_web_server.py | 49 ++++++++++ 3 files changed, 193 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/hermes.ts b/apps/desktop/src/hermes.ts index a902e4b53e1c..ddaabd15cedb 100644 --- a/apps/desktop/src/hermes.ts +++ b/apps/desktop/src/hermes.ts @@ -413,15 +413,16 @@ export function saveHermesConfig(config: HermesConfigRecord): Promise<{ ok: bool }) } +// surface=declared serves the curated desktop schema; the dashboard consumes the raw plugin schema. export function getMemoryProviderConfig(provider: string): Promise { return window.hermesDesktop.api({ - path: `/api/memory/providers/${encodeURIComponent(provider)}/config` + path: `/api/memory/providers/${encodeURIComponent(provider)}/config?surface=declared` }) } export function saveMemoryProviderConfig(provider: string, values: Record): Promise<{ ok: boolean }> { return window.hermesDesktop.api<{ ok: boolean }>({ - path: `/api/memory/providers/${encodeURIComponent(provider)}/config`, + path: `/api/memory/providers/${encodeURIComponent(provider)}/config?surface=declared`, method: 'PUT', body: { values } }) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index fed4b066d290..a6bb5ba1a6ea 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -84,6 +84,11 @@ from gateway.status import ( parse_active_agents, read_runtime_status, ) +from hermes_cli.memory_providers import ( + MemoryProvider as DeclaredMemoryProvider, + ProviderField as DeclaredProviderField, + get_memory_provider as get_declared_memory_provider, +) from utils import env_var_enabled try: @@ -5169,9 +5174,129 @@ def _require_valid_memory_provider_name(name: str) -> None: raise HTTPException(status_code=404, detail=f"Unknown memory provider: {name}") +# --------------------------------------------------------------------------- +# Declared surface — curated desktop schema from hermes_cli.memory_providers. +# The desktop panel requests ?surface=declared; the dashboard keeps the raw +# plugin schema. Providers without a declaration render no desktop panel. +# --------------------------------------------------------------------------- + +def _declared_provider_file_path(provider: DeclaredMemoryProvider) -> Path: + return get_hermes_home() / provider.name / "config.json" + + +def _read_declared_provider_file(provider: DeclaredMemoryProvider) -> Dict[str, Any]: + return _read_json_file(_declared_provider_file_path(provider)) + + +def _declared_read_field_value(field: DeclaredProviderField, data: Dict[str, Any]) -> str: + for source_key in (field.key, *field.aliases): + value = data.get(source_key) + if value: + return str(value) + + env_on_disk = load_env() + for env_key in field.env_fallbacks: + value = env_on_disk.get(env_key) + if value: + return str(value) + + return field.default + + +def _declared_field_is_set(field: DeclaredProviderField, data: Dict[str, Any]) -> bool: + env_on_disk = load_env() + for env_key in (field.env_key, *field.env_fallbacks): + if env_key and env_on_disk.get(env_key): + return True + return any(data.get(source_key) for source_key in (field.key, *field.aliases)) + + +def _declared_provider_payload(provider: DeclaredMemoryProvider) -> Dict[str, Any]: + data = _read_declared_provider_file(provider) + fields: List[Dict[str, Any]] = [] + + for field in provider.fields: + entry: Dict[str, Any] = { + "key": field.key, + "label": field.label, + "kind": field.kind, + "description": field.description, + "placeholder": field.placeholder, + "options": [ + {"value": opt.value, "label": opt.label, "description": opt.description} + for opt in field.options + ], + } + + if field.is_secret: + # Secrets are write-only over the API; only expose whether one is set. + entry["value"] = "" + entry["is_set"] = _declared_field_is_set(field, data) + else: + value = _declared_read_field_value(field, data) + if field.kind == "select" and value not in field.allowed_values(): + value = field.default + entry["value"] = value + entry["is_set"] = bool(value) + + fields.append(entry) + + return {"name": provider.name, "label": provider.label, "fields": fields} + + +def _coerce_declared_field_value(field: DeclaredProviderField, raw: str) -> str: + value = (raw or "").strip() + if field.kind == "select": + if not value: + value = field.default + if value not in field.allowed_values(): + raise ValueError(f"Invalid value for '{field.key}'") + return value + return value or field.default + + +def _update_declared_provider_config(provider: DeclaredMemoryProvider, values: Dict[str, Any]) -> None: + existing = _read_declared_provider_file(provider) + json_values: Dict[str, Any] = {} + secrets: Dict[str, str] = {} + + for field in provider.fields: + if field.is_secret: + submitted = str(values.get(field.key) or "").strip() + if submitted and field.env_key: + secrets[field.env_key] = submitted + continue + + raw = ( + values[field.key] + if field.key in values + else str(existing.get(field.key, field.default)) + ) + json_values[field.key] = _coerce_declared_field_value(field, str(raw)) + + path = _declared_provider_file_path(provider) + path.parent.mkdir(parents=True, exist_ok=True) + existing.update(json_values) + from utils import atomic_json_write + + atomic_json_write(path, existing, mode=0o600) + + for env_key, secret in secrets.items(): + save_env_value(env_key, secret) + + @app.get("/api/memory/providers/{name}/config") -async def get_memory_provider_config(name: str): +async def get_memory_provider_config(name: str, surface: Optional[str] = None): _require_valid_memory_provider_name(name) + + if surface == "declared": + declared = get_declared_memory_provider(name) + if declared is None: + # Undeclared providers (e.g. builtin, honcho) have no desktop + # config surface; the generic panel renders nothing. + return {"name": name, "label": name, "fields": []} + return _declared_provider_payload(declared) + provider = _load_memory_provider(name) if provider is None: # Undeclared providers (e.g. builtin) have no config surface. Return an @@ -5202,8 +5327,22 @@ async def setup_memory_provider(name: str, body: MemoryProviderSetupRequest): @app.put("/api/memory/providers/{name}/config") -async def update_memory_provider_config(name: str, body: MemoryProviderConfigUpdate): +async def update_memory_provider_config(name: str, body: MemoryProviderConfigUpdate, surface: Optional[str] = None): _require_valid_memory_provider_name(name) + + if surface == "declared": + declared = get_declared_memory_provider(name) + if declared is None: + raise HTTPException(status_code=404, detail=f"Unknown memory provider: {name}") + try: + _update_declared_provider_config(declared, body.values or {}) + return {"ok": True} + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception: + _log.exception("PUT /api/memory/providers/%s/config (declared) failed", name) + raise HTTPException(status_code=500, detail="Internal server error") + provider = _load_memory_provider(name) if provider is None: raise HTTPException(status_code=404, detail=f"Unknown memory provider: {name}") diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 19d379dfcd0d..fc42fcc28ca3 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -491,6 +491,55 @@ class TestWebServerEndpoints: assert fields["api_key"]["url"] == "https://app.honcho.dev" assert fields["baseUrl"]["kind"] == "text" + def test_declared_surface_serves_curated_hindsight_schema(self): + resp = self.client.get("/api/memory/providers/hindsight/config?surface=declared") + + assert resp.status_code == 200 + data = resp.json() + fields = self._provider_field_map(data) + assert set(fields) == {"mode", "api_key", "api_url", "bank_id", "recall_budget"} + assert fields["mode"]["kind"] == "select" + assert fields["api_key"]["kind"] == "secret" + + def test_declared_surface_hides_undeclared_providers(self): + resp = self.client.get("/api/memory/providers/honcho/config?surface=declared") + + assert resp.status_code == 200 + assert resp.json()["fields"] == [] + + def test_declared_surface_put_writes_config_and_secret(self): + from hermes_constants import get_hermes_home + from hermes_cli.config import load_env + + resp = self.client.put( + "/api/memory/providers/hindsight/config?surface=declared", + json={ + "values": { + "mode": "local_external", + "api_url": "http://localhost:8888", + "api_key": "hs-declared-key", + } + }, + ) + + assert resp.status_code == 200 + assert resp.json() == {"ok": True} + assert load_env()["HINDSIGHT_API_KEY"] == "hs-declared-key" + + config_path = get_hermes_home() / "hindsight" / "config.json" + provider_config = json.loads(config_path.read_text(encoding="utf-8")) + assert provider_config["mode"] == "local_external" + assert provider_config["api_url"] == "http://localhost:8888" + assert "api_key" not in provider_config + + def test_declared_surface_put_rejects_undeclared_provider(self): + resp = self.client.put( + "/api/memory/providers/honcho/config?surface=declared", + json={"values": {"api_key": "x"}}, + ) + + assert resp.status_code == 404 + def test_all_listed_memory_provider_configs_fetch(self): resp = self.client.get("/api/memory") From b8eb89f5c9460e8574ba4cb4e88d3cd08f093344 Mon Sep 17 00:00:00 2001 From: Vishal Dharmadhikari Date: Thu, 9 Jul 2026 21:38:18 -0700 Subject: [PATCH 034/149] feat(gemini): improve request context for support and compatibility Include the Hermes client name and version with Gemini inference, model and tier checks, and TTS requests. Add focused coverage for the request headers and keep the Gemini-specific context scoped to Google Gemini endpoints. --- agent/gemini_native_adapter.py | 18 +++++++- hermes_cli/models.py | 2 + scripts/release.py | 1 + tests/agent/test_gemini_native_adapter.py | 50 +++++++++++++++++++++++ tests/hermes_cli/test_model_validation.py | 30 ++++++++++++++ tests/tools/test_tts_gemini.py | 13 ++++++ tools/tts_tool.py | 15 ++++++- 7 files changed, 126 insertions(+), 3 deletions(-) diff --git a/agent/gemini_native_adapter.py b/agent/gemini_native_adapter.py index 9d3b1eb324d5..1c25f1e6cf0a 100644 --- a/agent/gemini_native_adapter.py +++ b/agent/gemini_native_adapter.py @@ -32,6 +32,13 @@ from agent.gemini_schema import sanitize_gemini_tool_parameters logger = logging.getLogger(__name__) +try: + import hermes_cli as _hermes_cli + + _HERMES_VERSION = str(_hermes_cli.__version__) +except Exception: + _HERMES_VERSION = "0.0.0" + DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" # Published max output-token ceiling shared by every current Gemini text model @@ -99,7 +106,10 @@ def probe_gemini_tier( url, params={"key": key}, json=payload, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "X-Goog-Api-Client": f"hermes-agent/{_HERMES_VERSION}", + }, ) except Exception as exc: logger.debug("probe_gemini_tier: network error: %s", exc) @@ -901,7 +911,11 @@ class GeminiNativeClient: "Content-Type": "application/json", "Accept": "application/json", "x-goog-api-key": self.api_key, - "User-Agent": "hermes-agent (gemini-native)", + # Include Hermes client context following Gemini's partner + # integration guidance. + # See https://ai.google.dev/gemini-api/docs/partner-integration + "User-Agent": f"hermes-agent/{_HERMES_VERSION} (gemini-native)", + "X-Goog-Api-Client": f"hermes-agent/{_HERMES_VERSION}", } headers.update(self._default_headers) return headers diff --git a/hermes_cli/models.py b/hermes_cli/models.py index a6dd34d0361c..d81c409814c7 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -3594,6 +3594,8 @@ def probe_api_models( tried: list[str] = [] headers: dict[str, str] = {"User-Agent": _HERMES_USER_AGENT} + if urllib.parse.urlparse(normalized).hostname == "generativelanguage.googleapis.com": + headers["X-Goog-Api-Client"] = f"hermes-agent/{_HERMES_VERSION}" if api_key and api_mode == "anthropic_messages": headers["x-api-key"] = api_key headers["anthropic-version"] = "2023-06-01" diff --git a/scripts/release.py b/scripts/release.py index d4ad39fb5e23..2c528e15516a 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -776,6 +776,7 @@ AUTHOR_MAP = { "bzarnitz13@gmail.com": "Beandon13", "tony@tonysimons.dev": "asimons81", "jetha@google.com": "jethac", + "vishal.dharm@gmail.com": "vishal-dharm", "jani@0xhoneyjar.xyz": "deep-name", # LINE messaging plugin (synthesis PR) "32443648+leepoweii@users.noreply.github.com": "leepoweii", diff --git a/tests/agent/test_gemini_native_adapter.py b/tests/agent/test_gemini_native_adapter.py index c265572436dd..e797d6b2ae71 100644 --- a/tests/agent/test_gemini_native_adapter.py +++ b/tests/agent/test_gemini_native_adapter.py @@ -461,3 +461,53 @@ def test_explicit_max_tokens_is_respected(): req = build_gemini_request(messages=[{"role": "user", "content": "hi"}], max_tokens=4096) assert req["generationConfig"]["maxOutputTokens"] == 4096 + + +# --------------------------------------------------------------------------- +# X-Goog-Api-Client header tests +# --------------------------------------------------------------------------- + + +def test_x_goog_api_client_header_is_set(): + """The X-Goog-Api-Client header should be set on inference requests.""" + from agent.gemini_native_adapter import GeminiNativeClient + + client = GeminiNativeClient(api_key="fake-key", model="gemini-2.0-flash") + headers = client._headers() + + assert "X-Goog-Api-Client" in headers, "X-Goog-Api-Client header missing" + assert "hermes-agent/" in headers["X-Goog-Api-Client"], ( + "hermes-agent not found in X-Goog-Api-Client header" + ) + + +def test_x_goog_api_client_header_format(): + """Header value should be 'hermes-agent/' matching the package version.""" + from agent.gemini_native_adapter import GeminiNativeClient, _HERMES_VERSION + + client = GeminiNativeClient(api_key="fake-key", model="gemini-2.0-flash") + headers = client._headers() + + expected = f"hermes-agent/{_HERMES_VERSION}" + assert headers["X-Goog-Api-Client"] == expected + + +def test_user_agent_contains_version(): + """User-Agent should include the hermes-agent version.""" + from agent.gemini_native_adapter import GeminiNativeClient, _HERMES_VERSION + + client = GeminiNativeClient(api_key="fake-key", model="gemini-2.0-flash") + headers = client._headers() + + assert f"hermes-agent/{_HERMES_VERSION}" in headers["User-Agent"] + + +def test_hermes_version_is_valid(): + """_HERMES_VERSION should be a non-empty string.""" + from agent.gemini_native_adapter import _HERMES_VERSION + + assert isinstance(_HERMES_VERSION, str) + assert len(_HERMES_VERSION) > 0 + assert _HERMES_VERSION != "0.0.0", ( + "Version should resolve from hermes_cli.__version__, not the fallback" + ) diff --git a/tests/hermes_cli/test_model_validation.py b/tests/hermes_cli/test_model_validation.py index 93095999d101..9e08831701e3 100644 --- a/tests/hermes_cli/test_model_validation.py +++ b/tests/hermes_cli/test_model_validation.py @@ -942,3 +942,33 @@ class TestProbeApiModelsUserAgent: assert ua and ua.startswith("hermes-cli/") # No Authorization was set, but UA must still be present. assert req.get_header("Authorization") is None + + def test_probe_sends_client_context_to_gemini(self): + from unittest.mock import patch + from hermes_cli.models import _HERMES_VERSION + + body = b'{"data":[]}' + with patch( + "hermes_cli.models.urllib.request.urlopen", + return_value=self._make_mock_response(body), + ) as mock_urlopen: + probe_api_models( + "gemini-key", + "https://generativelanguage.googleapis.com/v1beta/openai", + ) + + req = mock_urlopen.call_args[0][0] + assert req.get_header("X-goog-api-client") == f"hermes-agent/{_HERMES_VERSION}" + + def test_probe_omits_gemini_client_context_for_other_providers(self): + from unittest.mock import patch + + body = b'{"data":[]}' + with patch( + "hermes_cli.models.urllib.request.urlopen", + return_value=self._make_mock_response(body), + ) as mock_urlopen: + probe_api_models("provider-key", "https://api.example.com/v1") + + req = mock_urlopen.call_args[0][0] + assert req.get_header("X-goog-api-client") is None diff --git a/tests/tools/test_tts_gemini.py b/tests/tools/test_tts_gemini.py index 15e26bdbb122..da744693b588 100644 --- a/tests/tools/test_tts_gemini.py +++ b/tests/tools/test_tts_gemini.py @@ -115,6 +115,19 @@ class TestGenerateGeminiTts: # Audio payload should match the PCM we put in assert data[44:] == fake_pcm_bytes + def test_x_goog_api_client_header_is_set(self, tmp_path, monkeypatch, mock_gemini_response): + """Gemini TTS requests should include Hermes client context.""" + from tools.tts_tool import _generate_gemini_tts + + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + + with patch("requests.post", return_value=mock_gemini_response) as mock_post: + _generate_gemini_tts("Hi", str(tmp_path / "test.wav"), {}) + + headers = mock_post.call_args[1]["headers"] + assert "X-Goog-Api-Client" in headers + assert headers["X-Goog-Api-Client"].startswith("hermes-agent/") + def test_default_voice_and_model(self, tmp_path, monkeypatch, mock_gemini_response): from tools.tts_tool import ( DEFAULT_GEMINI_TTS_MODEL, diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 7d571e66b1ad..374042192414 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -1850,11 +1850,24 @@ def _generate_gemini_tts(text: str, output_path: str, tts_config: Dict[str, Any] }, } + try: + import hermes_cli as _hermes_cli + + _hermes_version = str(_hermes_cli.__version__) + except Exception: + _hermes_version = "0.0.0" + endpoint = f"{base_url}/models/{model}:generateContent" response = requests.post( endpoint, params={"key": api_key}, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + # Include Hermes client context following Gemini's partner + # integration guidance: + # https://ai.google.dev/gemini-api/docs/partner-integration + "X-Goog-Api-Client": f"hermes-agent/{_hermes_version}", + }, json=payload, timeout=60, ) From 226e8de827a669e8ffa7035b27d70c19e44b1208 Mon Sep 17 00:00:00 2001 From: Vishal Dharmadhikari Date: Fri, 10 Jul 2026 16:05:35 -0700 Subject: [PATCH 035/149] fix(gemini): restrict TTS client context to official host --- tests/tools/test_tts_gemini.py | 20 ++++++++++++++++++-- tools/tts_tool.py | 26 +++++++++++++------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/tests/tools/test_tts_gemini.py b/tests/tools/test_tts_gemini.py index da744693b588..1a8bde7cc8a9 100644 --- a/tests/tools/test_tts_gemini.py +++ b/tests/tools/test_tts_gemini.py @@ -117,6 +117,7 @@ class TestGenerateGeminiTts: def test_x_goog_api_client_header_is_set(self, tmp_path, monkeypatch, mock_gemini_response): """Gemini TTS requests should include Hermes client context.""" + from hermes_cli import __version__ from tools.tts_tool import _generate_gemini_tts monkeypatch.setenv("GEMINI_API_KEY", "test-key") @@ -125,8 +126,7 @@ class TestGenerateGeminiTts: _generate_gemini_tts("Hi", str(tmp_path / "test.wav"), {}) headers = mock_post.call_args[1]["headers"] - assert "X-Goog-Api-Client" in headers - assert headers["X-Goog-Api-Client"].startswith("hermes-agent/") + assert headers["X-Goog-Api-Client"] == f"hermes-agent/{__version__}" def test_default_voice_and_model(self, tmp_path, monkeypatch, mock_gemini_response): from tools.tts_tool import ( @@ -268,6 +268,22 @@ class TestGenerateGeminiTts: _generate_gemini_tts("Hi", str(tmp_path / "test.wav"), {}) assert mock_post.call_args[0][0].startswith("https://custom-gemini.example.com/v1beta/") + assert "X-Goog-Api-Client" not in mock_post.call_args[1]["headers"] + + def test_lookalike_base_url_omits_client_context( + self, tmp_path, monkeypatch, mock_gemini_response + ): + from tools.tts_tool import _generate_gemini_tts + + lookalike = "https://generativelanguage.googleapis.com.evil.example/v1beta" + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + monkeypatch.setenv("GEMINI_BASE_URL", lookalike) + + with patch("requests.post", return_value=mock_gemini_response) as mock_post: + _generate_gemini_tts("Hi", str(tmp_path / "test.wav"), {}) + + assert mock_post.call_args[0][0].startswith(f"{lookalike}/") + assert "X-Goog-Api-Client" not in mock_post.call_args[1]["headers"] def test_persona_prompt_file_appends_labeled_transcript( self, tmp_path, monkeypatch, mock_gemini_response diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 374042192414..545d72bb6907 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -50,7 +50,7 @@ import threading import uuid from pathlib import Path from typing import Callable, Dict, Any, Optional -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse from hermes_cli._subprocess_compat import windows_hide_flags from hermes_constants import display_hermes_home @@ -1850,24 +1850,24 @@ def _generate_gemini_tts(text: str, output_path: str, tts_config: Dict[str, Any] }, } - try: - import hermes_cli as _hermes_cli + headers = {"Content-Type": "application/json"} + if urlparse(base_url).hostname == "generativelanguage.googleapis.com": + try: + import hermes_cli as _hermes_cli - _hermes_version = str(_hermes_cli.__version__) - except Exception: - _hermes_version = "0.0.0" + _hermes_version = str(_hermes_cli.__version__) + except Exception: + _hermes_version = "0.0.0" + # Include Hermes client context following Gemini's partner + # integration guidance: + # https://ai.google.dev/gemini-api/docs/partner-integration + headers["X-Goog-Api-Client"] = f"hermes-agent/{_hermes_version}" endpoint = f"{base_url}/models/{model}:generateContent" response = requests.post( endpoint, params={"key": api_key}, - headers={ - "Content-Type": "application/json", - # Include Hermes client context following Gemini's partner - # integration guidance: - # https://ai.google.dev/gemini-api/docs/partner-integration - "X-Goog-Api-Client": f"hermes-agent/{_hermes_version}", - }, + headers=headers, json=payload, timeout=60, ) From 0d3ad193d6cc235213489f509e26760ccb3722a1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:11:19 -0700 Subject: [PATCH 036/149] fix(tests): patch catalog urlopen wrapper in gemini probe tests (#64318) test_probe_sends_client_context_to_gemini and test_probe_omits_gemini_client_context_for_other_providers (added in b8eb89f5c) patch hermes_cli.models.urllib.request.urlopen, but probe_api_models routes requests through the _urlopen_model_catalog_request wrapper (open_credentialed_url from the urllib_security hardening), so the mock is never invoked and mock_urlopen.call_args is None -> TypeError. Every CI run on main and every PR has been failing test slice 7/8 on these two tests. Point the patches at _urlopen_model_catalog_request, the same target every sibling test in TestProbeApiModelsUserAgent already uses. 89/89 tests in the file now pass. --- tests/hermes_cli/test_model_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/hermes_cli/test_model_validation.py b/tests/hermes_cli/test_model_validation.py index 9e08831701e3..cfff3080ebfa 100644 --- a/tests/hermes_cli/test_model_validation.py +++ b/tests/hermes_cli/test_model_validation.py @@ -949,7 +949,7 @@ class TestProbeApiModelsUserAgent: body = b'{"data":[]}' with patch( - "hermes_cli.models.urllib.request.urlopen", + "hermes_cli.models._urlopen_model_catalog_request", return_value=self._make_mock_response(body), ) as mock_urlopen: probe_api_models( @@ -965,7 +965,7 @@ class TestProbeApiModelsUserAgent: body = b'{"data":[]}' with patch( - "hermes_cli.models.urllib.request.urlopen", + "hermes_cli.models._urlopen_model_catalog_request", return_value=self._make_mock_response(body), ) as mock_urlopen: probe_api_models("provider-key", "https://api.example.com/v1") From 8582f35d9667e762816f4c6bf364334bdcb595a7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:22:48 -0700 Subject: [PATCH 037/149] fix(moa): flatten structured message content in the advisory view (#64319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cache-decorated turns (apply_anthropic_cache_control converts string content to [{type: text, ..., cache_control}] lists — applied BEFORE the MoA facade since the #57675 cache-cold fix) and multimodal turns (text + image_url parts) flattened to empty strings in _reference_messages, which only read str content. On turn 1 of a provider:moa session with a Claude aggregator the references received a single EMPTY user message: Anthropic-side providers 400'd ('messages: at least one message is required') while tolerant models answered 'no user request is present' (live incident Jul 14 2026, preset 'closed'). Fixes, in totality: - _reference_messages: extract visible text via agent/message_content.flatten_message_text for user/assistant/tool turns (skips image parts, so no base64 leaks into the advisory view); decorated and undecorated transcripts now produce a byte-identical advisory view (advisor cache prefix stays stable). - image-only user turns get a placeholder instead of an empty message (Anthropic rejects empty text blocks) or a silently dropped turn (would break user/assistant alternation). - degenerate-case fallback flattens structured content too. - _attach_reference_guidance: a decorated/multimodal trailing user turn now receives the guidance as a NEW text part appended AFTER the cache_control-marked part (cached prefix byte-stable) instead of falling through to a second consecutive user message (strict providers reject user/user). - conversation_loop MoA injection: multimodal user turns get the MoA context appended as a trailing text part instead of being dropped; user_prompt for the one-shot path flattens content lists instead of str()-ing them (which leaked base64 payloads into the prompt). Live-verified on the 'closed' preset (real OpenRouter wire, 2 user turns, tool loop): all 4 reference calls carry the full document + rendered tool state, end on user, zero tool-role/tool_calls; advisor cache_write 7968 then cache_read 5909+; aggregator cache_read 14880-15237 on iterations 2+. Co-authored-by: bo.fu --- agent/conversation_loop.py | 19 +++- agent/moa_loop.py | 59 +++++++++-- tests/run_agent/test_moa_loop_mode.py | 144 ++++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 9 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 12c80a8bfb71..dd05ce1e4795 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -857,10 +857,19 @@ def run_conversation( if moa_config: try: + from agent.message_content import flatten_message_text as _flatten_mt from agent.moa_loop import _preset_temperature, aggregate_moa_context _moa_context = aggregate_moa_context( - user_prompt=original_user_message if isinstance(original_user_message, str) else str(original_user_message), + user_prompt=( + original_user_message + if isinstance(original_user_message, str) + # Multimodal / decorated content list: extract the + # visible text instead of str()-ing a Python repr of + # the parts (which would leak base64 image payloads + # into the aggregator prompt). + else _flatten_mt(original_user_message) + ), api_messages=api_messages, reference_models=moa_config.get("reference_models") or [], aggregator=moa_config.get("aggregator") or {}, @@ -874,6 +883,14 @@ def run_conversation( _base = _msg.get("content", "") if isinstance(_base, str): _msg["content"] = _base + "\n\n" + _moa_context + elif isinstance(_base, list): + # Multimodal user turn (text + image parts): + # append the MoA context as a trailing text + # part instead of silently dropping it. + _msg["content"] = [ + *_base, + {"type": "text", "text": "\n\n" + _moa_context}, + ] break except Exception as _moa_exc: logger.warning("MoA context aggregation failed: %s", _moa_exc) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index cd7fa3ea4ea2..267f33dd7e5a 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -14,6 +14,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Any from agent.auxiliary_client import call_llm +from agent.message_content import flatten_message_text from agent.transports import get_transport logger = logging.getLogger(__name__) @@ -470,11 +471,36 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: for msg in messages: role = msg.get("role") content = msg.get("content") - text = content if isinstance(content, str) else "" + # Flatten structured content (lists of parts) to visible text. Content + # arrives as a list — not a string — in two common cases: + # 1. Anthropic prompt-cache decoration: conversation_loop runs + # apply_anthropic_cache_control BEFORE the MoA facade, converting + # string content to [{"type": "text", "text": ..., "cache_control": + # ...}]. A str-only read here flattened the user's ENTIRE prompt to + # "" — Claude references then 400'd ("messages: at least one + # message is required") while tolerant models answered "no user + # request is present". + # 2. Multimodal turns (pasted image → text + image_url parts) and + # multimodal tool results (screenshots). + # flatten_message_text extracts the text parts and skips image parts, + # and returns strings unchanged — so a decorated and an undecorated + # transcript produce a byte-identical advisory view (which keeps the + # advisory prefix stable across iterations for advisor prompt caching). + text = flatten_message_text(content) if role == "system": continue if role == "user": + if not text.strip() and content not in (None, "", []): + # Structured content with no extractable text (e.g. an + # image-only turn). Emitting an empty user message would be + # dropped/rejected by strict providers (Anthropic 400s on + # empty text blocks — the original "closed" preset failure + # mode), and silently skipping the turn would break + # user/assistant alternation in the advisory view. Substitute + # a placeholder so the reference knows a non-text turn + # happened. + text = "[user sent non-text content (e.g. an image attachment)]" if text.strip(): last_user_content = text rendered.append({"role": "user", "content": text}) @@ -517,8 +543,10 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: if last_user_content is not None: return [{"role": "user", "content": last_user_content}] for msg in reversed(messages): - if msg.get("role") == "user" and isinstance(msg.get("content"), str): - return [{"role": "user", "content": msg["content"]}] + if msg.get("role") == "user": + fallback_text = flatten_message_text(msg.get("content")) + if fallback_text.strip(): + return [{"role": "user", "content": fallback_text}] return rendered @@ -671,13 +699,28 @@ def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str Appending at the very end keeps the ``[system][task][tool-history]`` prefix stable and cache-reusable (only the new block re-prefills), and gives the aggregator the references with recency. Merge into the last message only when - it is already a trailing string ``user`` turn (plain chat — still at the end). + it is already a trailing ``user`` turn (plain chat — still at the end). + + A trailing user turn's content may be a STRING or a LIST of content parts — + Anthropic prompt-cache decoration (which runs before the MoA facade) + converts string content to ``[{"type": "text", ..., "cache_control": ...}]``, + and multimodal turns are lists natively. Both shapes are merged in place: + appending a new text part AFTER the cache_control-marked part keeps the + cached prefix byte-stable (the marker still terminates it) while the + turn-varying guidance rides outside the cached span. Appending a SEPARATE + user message here instead would produce two consecutive user turns — + strict providers reject that. """ last = agg_messages[-1] if agg_messages else None - if last is not None and last.get("role") == "user" and isinstance(last.get("content"), str): - last["content"] = last["content"] + "\n\n" + guidance - else: - agg_messages.append({"role": "user", "content": guidance}) + if last is not None and last.get("role") == "user": + last_content = last.get("content") + if isinstance(last_content, str): + last["content"] = last_content + "\n\n" + guidance + return + if isinstance(last_content, list): + last["content"] = [*last_content, {"type": "text", "text": "\n\n" + guidance}] + return + agg_messages.append({"role": "user", "content": guidance}) class MoAChatCompletions: diff --git a/tests/run_agent/test_moa_loop_mode.py b/tests/run_agent/test_moa_loop_mode.py index 04c7d51ceb10..094278541762 100644 --- a/tests/run_agent/test_moa_loop_mode.py +++ b/tests/run_agent/test_moa_loop_mode.py @@ -1059,3 +1059,147 @@ def test_reference_guidance_merges_into_trailing_user_in_plain_chat(): assert len(messages) == 2 assert messages[-1]["role"] == "user" assert messages[-1]["content"] == "hello\n\nREFERENCE BLOCK" + + +def test_reference_messages_flattens_cache_decorated_content(): + """Cache-decorated turns (content-part lists) must not blind the references. + + conversation_loop runs apply_anthropic_cache_control BEFORE the MoA facade + when the preset's aggregator is a cache-honoring Claude route (post-#57675). + That converts string content into [{"type": "text", "text": ..., + "cache_control": ...}] lists. The advisory view previously read only string + content, so the user's ENTIRE prompt flattened to "" — Claude references + then 400'd ("messages: at least one message is required") while tolerant + models answered "no user request is present" (live incident, Jul 14 2026, + preset "closed", session 20260714_001520_28157b). + """ + from agent.moa_loop import _reference_messages + from agent.prompt_caching import apply_anthropic_cache_control + + plain = [ + {"role": "system", "content": "hermes system prompt"}, + {"role": "user", "content": "Can we get codex usage resets into hermes?"}, + ] + decorated = apply_anthropic_cache_control(plain, native_anthropic=False) + # Premise: decoration really converts the user turn to a content-part list. + assert isinstance(decorated[1]["content"], list) + + view = _reference_messages(decorated) + + assert view == [ + {"role": "user", "content": "Can we get codex usage resets into hermes?"} + ] + # Invariant: decorated and undecorated transcripts produce the SAME + # advisory view — so decoration can never change what references see, + # and the advisory prefix stays byte-stable for advisor prompt caching. + assert view == _reference_messages(plain) + + +def test_reference_messages_flattens_multimodal_user_turn(): + """Multimodal user turns (text + image parts) keep their text in the view. + + Image parts carry no advisory text and are skipped; the text part must + survive. Previously the whole turn flattened to "". + """ + from agent.moa_loop import _reference_messages + + messages = [ + {"role": "user", "content": [ + {"type": "text", "text": "what is in this screenshot?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ]}, + ] + + view = _reference_messages(messages) + + assert view == [{"role": "user", "content": "what is in this screenshot?"}] + # No base64 payload leaks into the advisory view. + assert all("base64" not in m["content"] for m in view) + + +def test_reference_messages_image_only_user_turn_gets_placeholder(): + """An image-only user turn must not become an empty user message. + + Anthropic rejects empty text blocks (the original 400 class) and silently + skipping the turn would misalign user/assistant alternation in the view — + so a placeholder stands in for the non-text content. + """ + from agent.moa_loop import _reference_messages + + messages = [ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ]}, + {"role": "assistant", "content": "I see a diagram."}, + {"role": "user", "content": "now explain it"}, + ] + + view = _reference_messages(messages) + + assert view[0]["role"] == "user" + assert view[0]["content"].strip(), "image-only turn must not be empty" + assert "non-text" in view[0]["content"] + assert view[-1] == {"role": "user", "content": "now explain it"} + + +def test_reference_messages_flattens_structured_assistant_and_tool_content(): + """Assistant and tool turns with content-part lists are flattened too. + + Multimodal tool results (e.g. computer_use screenshots) and adapter-shaped + assistant turns arrive as lists; their text must reach the references and + their image parts must not leak. + """ + from agent.moa_loop import _reference_messages + + messages = [ + {"role": "user", "content": "check the screen"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "taking a screenshot"}], + "tool_calls": [{"id": "c1", "function": {"name": "capture", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "c1", "content": [ + {"type": "text", "text": "screenshot captured: login page visible"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBBB"}}, + ]}, + ] + + view = _reference_messages(messages) + + joined = "\n".join(m["content"] for m in view) + assert "taking a screenshot" in joined + assert "[called tool: capture(" in joined + assert "[tool result: screenshot captured: login page visible]" in joined + assert "BBBB" not in joined + assert view[-1]["role"] == "user" + + +def test_reference_guidance_appends_text_part_to_decorated_trailing_user(): + """A cache-decorated trailing user turn still receives the guidance block. + + Decoration converts the trailing user turn to a content-part list; the + guidance must be appended as a NEW text part AFTER the cache_control-marked + part (cached prefix stays byte-stable, no consecutive-user-turn 400s), not + silently dropped and not added as a second user message. + """ + from agent.moa_loop import _attach_reference_guidance + + marked_part = { + "type": "text", + "text": "hello", + "cache_control": {"type": "ephemeral"}, + } + messages = [ + {"role": "system", "content": "system prompt"}, + {"role": "user", "content": [dict(marked_part)]}, + ] + _attach_reference_guidance(messages, "REFERENCE BLOCK") + + # No extra message (would break user/user alternation). + assert len(messages) == 2 + content = messages[-1]["content"] + assert isinstance(content, list) and len(content) == 2 + # The cache-marked part is byte-identical (prefix stability). + assert content[0] == marked_part + # The guidance rides as a trailing text part outside the cached span. + assert content[1] == {"type": "text", "text": "\n\nREFERENCE BLOCK"} From 89bd0fba903bbfd78b0d99ce6f194863dd01b7e1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:23:19 -0700 Subject: [PATCH 038/149] feat(codex): redeem banked usage-limit resets via /usage reset (#64280) OpenAI lets ChatGPT-plan Codex users bank rate-limit reset credits, but until now they could only be redeemed from the Codex CLI/app or the website. This wires the same backend API into Hermes: - /usage on the openai-codex provider now shows "You have N resets banked - use /usage reset to activate" (parsed from the rate_limit_reset_credits field the /usage endpoint already returns). - New /usage reset subcommand (CLI + gateway) redeems one banked credit via POST .../rate-limit-reset-credits/consume with a UUID idempotency key, mirroring codex-rs backend-client semantics (PathStyle /wham vs /api/codex, ChatGPT-Account-Id header, reset/nothing_to_reset/no_credit/already_redeemed outcomes). - Guard: redemption is refused while no rate-limit window is fully exhausted, since a banked reset restores the FULL 5h + weekly allowance and spending it early wastes it. /usage reset --force overrides. Zero banked credits and non-codex providers are refused with clear messages; nothing_to_reset reports the credit was NOT spent. - i18n: new gateway.usage.unknown_subcommand / reset_wrong_provider keys across all 16 locales; docs updated (cli.md, messaging index). Tested with unit tests plus a real-socket E2E against a local fake Codex backend exercising redeem/guard/force and the /usage hint. --- agent/account_usage.py | 202 ++++++++++++++++++++- cli.py | 49 ++++- gateway/slash_commands.py | 24 +++ hermes_cli/commands.py | 3 +- locales/af.yaml | 2 + locales/de.yaml | 2 + locales/en.yaml | 2 + locales/es.yaml | 2 + locales/fr.yaml | 2 + locales/ga.yaml | 2 + locales/hu.yaml | 2 + locales/it.yaml | 2 + locales/ja.yaml | 2 + locales/ko.yaml | 2 + locales/pt.yaml | 2 + locales/ru.yaml | 2 + locales/tr.yaml | 2 + locales/uk.yaml | 2 + locales/zh-hant.yaml | 2 + locales/zh.yaml | 2 + tests/agent/test_account_usage.py | 201 ++++++++++++++++++++ tests/gateway/test_usage_command.py | 71 ++++++++ website/docs/user-guide/cli.md | 2 + website/docs/user-guide/messaging/index.md | 2 +- 24 files changed, 579 insertions(+), 7 deletions(-) diff --git a/agent/account_usage.py b/agent/account_usage.py index 712e57feda4a..9e48b0aac0cb 100644 --- a/agent/account_usage.py +++ b/agent/account_usage.py @@ -425,15 +425,28 @@ def build_credits_view(*, markdown: bool = False, timeout: float = 10.0) -> Cred ) -def _resolve_codex_usage_url(base_url: str) -> str: +def _codex_backend_urls(base_url: str) -> tuple[str, str, str]: + """Resolve the Codex backend endpoints (usage, reset-credits list, consume). + + Mirrors the Codex CLI's PathStyle split (codex-rs backend-client): base URLs + containing ``/backend-api`` use the ChatGPT ``/wham/...`` paths; everything + else uses ``/api/codex/...``. + """ normalized = (base_url or "").strip().rstrip("/") if not normalized: normalized = "https://chatgpt.com/backend-api/codex" if normalized.endswith("/codex"): normalized = normalized[: -len("/codex")] - if "/backend-api" in normalized: - return normalized + "/wham/usage" - return normalized + "/api/codex/usage" + prefix = normalized + ("/wham" if "/backend-api" in normalized else "/api/codex") + return ( + prefix + "/usage", + prefix + "/rate-limit-reset-credits", + prefix + "/rate-limit-reset-credits/consume", + ) + + +def _resolve_codex_usage_url(base_url: str) -> str: + return _codex_backend_urls(base_url)[0] def _resolve_codex_usage_credentials( @@ -525,6 +538,14 @@ def _fetch_codex_account_usage( ) ) details: list[str] = [] + reset_credits = payload.get("rate_limit_reset_credits") or {} + banked = reset_credits.get("available_count") + if isinstance(banked, (int, float)) and int(banked) > 0: + count = int(banked) + plural = "s" if count != 1 else "" + details.append( + f"You have {count} reset{plural} banked - use /usage reset to activate" + ) credits = payload.get("credits") or {} if credits.get("has_credits"): balance = credits.get("balance") @@ -542,6 +563,179 @@ def _fetch_codex_account_usage( ) +@dataclass(frozen=True) +class CodexResetRedeemResult: + """Outcome of a `/usage reset` attempt against the Codex backend.""" + + status: str # reset | nothing_to_reset | no_credit | already_redeemed | + # not_exhausted | no_credits_banked | unavailable + message: str + available_count: int = 0 + windows_reset: int = 0 + + @property + def redeemed(self) -> bool: + return self.status == "reset" + + +# Client-side guard threshold: a rate-limit window only counts as exhausted +# when it is fully used. Below this, redeeming a banked reset wastes most of +# its value, so we block and point at --force instead. +_CODEX_WINDOW_EXHAUSTED_PERCENT = 100.0 + + +def redeem_codex_reset_credit( + *, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + force: bool = False, +) -> CodexResetRedeemResult: + """Redeem one banked Codex rate-limit reset credit (`/usage reset`). + + Flow (mirrors the Codex CLI's reset-credits picker, codex-rs + ``backend-client``): + + 1. ``GET .../usage`` — read the current windows + banked credit count. + 2. Guard: zero banked credits → refuse. No window fully used and not + ``force`` → refuse with a warning (a banked reset restores the WHOLE + 5h + weekly allowance; burning it early wastes it). The backend has + the same protection (``nothing_to_reset`` doesn't consume the + credit), but failing fast client-side gives a clearer message. + 3. ``POST .../rate-limit-reset-credits/consume`` with a fresh UUID + idempotency key (``redeem_request_id``). No ``credit_id`` — the + backend picks the next available credit, exactly like the CLI's + default "Full reset" option. + + Never raises: every failure mode returns a ``CodexResetRedeemResult`` + with a user-renderable message. + """ + import uuid + + try: + token, resolved_base_url, account_id = _resolve_codex_usage_credentials(base_url, api_key) + except Exception: + return CodexResetRedeemResult( + status="unavailable", + message="No Codex credentials available. Run `hermes auth` to sign in with your ChatGPT account.", + ) + usage_url, _credits_url, consume_url = _codex_backend_urls(resolved_base_url) + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "User-Agent": "codex-cli", + } + if account_id: + headers["ChatGPT-Account-Id"] = account_id + + try: + with httpx.Client(timeout=15.0) as client: + usage_resp = client.get(usage_url, headers=headers) + usage_resp.raise_for_status() + payload = usage_resp.json() or {} + + reset_credits = payload.get("rate_limit_reset_credits") or {} + raw_count = reset_credits.get("available_count") + available = int(raw_count) if isinstance(raw_count, (int, float)) else 0 + if available <= 0: + return CodexResetRedeemResult( + status="no_credits_banked", + message="No banked reset credits on this account — nothing to redeem.", + ) + + rate_limit = payload.get("rate_limit") or {} + worst_used: Optional[float] = None + for key in ("primary_window", "secondary_window"): + used = (rate_limit.get(key) or {}).get("used_percent") + if isinstance(used, (int, float)): + worst_used = max(worst_used or 0.0, float(used)) + exhausted = worst_used is not None and worst_used >= _CODEX_WINDOW_EXHAUSTED_PERCENT + if not exhausted and not force: + usage_note = ( + f"your busiest window is only {worst_used:.0f}% used" + if worst_used is not None + else "your current usage could not be confirmed as exhausted" + ) + plural = "s" if available != 1 else "" + return CodexResetRedeemResult( + status="not_exhausted", + message=( + f"⚠️ Not redeeming: {usage_note}. A banked reset restores your FULL " + f"5h + weekly limits, so spending it now would waste most of it. " + f"You have {available} reset{plural} banked. " + f"Use `/usage reset --force` to redeem anyway." + ), + available_count=available, + ) + + consume_resp = client.post( + consume_url, + headers={**headers, "Content-Type": "application/json"}, + json={"redeem_request_id": str(uuid.uuid4())}, + ) + consume_resp.raise_for_status() + body = consume_resp.json() or {} + except httpx.HTTPStatusError as exc: + code = exc.response.status_code + if code in (401, 403): + return CodexResetRedeemResult( + status="unavailable", + message=( + "Codex backend rejected the request (HTTP " + f"{code}). Reset credits require ChatGPT-account (OAuth) auth — " + "run `hermes auth` and sign in with your ChatGPT account." + ), + ) + return CodexResetRedeemResult( + status="unavailable", + message=f"Codex backend error (HTTP {code}) — try again shortly.", + ) + except Exception as exc: + return CodexResetRedeemResult( + status="unavailable", + message=f"Could not reach the Codex backend: {exc}", + ) + + code = str(body.get("code", "") or "").strip().lower() + windows_reset = body.get("windows_reset") + windows_reset = int(windows_reset) if isinstance(windows_reset, (int, float)) else 0 + remaining = max(0, available - 1) + plural = "s" if remaining != 1 else "" + if code == "reset": + return CodexResetRedeemResult( + status="reset", + message=( + f"✅ Reset redeemed — your usage limits have been reset. " + f"{remaining} banked reset{plural} remaining." + ), + available_count=remaining, + windows_reset=windows_reset, + ) + if code == "nothing_to_reset": + return CodexResetRedeemResult( + status="nothing_to_reset", + message=( + "Backend reports nothing to reset — your limits aren't exhausted. " + "The credit was NOT spent." + ), + available_count=available, + ) + if code == "no_credit": + return CodexResetRedeemResult( + status="no_credit", + message="Backend reports no available reset credit on this account.", + ) + if code == "already_redeemed": + return CodexResetRedeemResult( + status="already_redeemed", + message="This redemption was already processed — no additional credit was spent.", + available_count=remaining, + ) + return CodexResetRedeemResult( + status="unavailable", + message=f"Unexpected response from the Codex backend: {body!r}", + ) + + def _fetch_anthropic_account_usage() -> Optional[AccountUsageSnapshot]: token = (resolve_anthropic_token() or "").strip() if not token: diff --git a/cli.py b/cli.py index 9887bb029780..796f38cb4c67 100644 --- a/cli.py +++ b/cli.py @@ -8722,7 +8722,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): elif canonical == "compress": self._manual_compress(cmd_original) elif canonical == "usage": - self._show_usage() + self._handle_usage_command(cmd_original) elif canonical == "credits": self._show_credits() elif canonical == "billing": @@ -9623,6 +9623,53 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): + def _handle_usage_command(self, cmd_original: str): + """Dispatch `/usage [reset [--force]]`. + + Bare `/usage` keeps the classic display. `/usage reset` redeems one + banked Codex rate-limit reset credit (guarded: refuses when limits + aren't exhausted unless --force). + """ + parts = cmd_original.split() + args = [p.lower() for p in parts[1:]] + if args and args[0] == "reset": + self._usage_reset(force="--force" in args[1:]) + return + if args: + print(f" Unknown /usage subcommand: {' '.join(parts[1:])}. Try /usage or /usage reset [--force].") + return + self._show_usage() + + def _usage_reset(self, force: bool = False): + """`/usage reset [--force]` — redeem one banked Codex reset credit.""" + provider = ( + (getattr(self.agent, "provider", None) if self.agent else None) + or getattr(self, "provider", None) + ) + normalized = str(provider or "").strip().lower() + if normalized != "openai-codex": + print(" Banked usage resets are only available on the openai-codex provider.") + print(" Switch with `/model` or `hermes auth` first.") + return + base_url = (getattr(self.agent, "base_url", None) if self.agent else None) or getattr(self, "base_url", None) + api_key = (getattr(self.agent, "api_key", None) if self.agent else None) or getattr(self, "api_key", None) + + from agent.account_usage import redeem_codex_reset_credit + + print(" ⏳ Checking banked reset credits...") + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as _pool: + try: + result = _pool.submit( + redeem_codex_reset_credit, + base_url=base_url, + api_key=api_key, + force=force, + ).result(timeout=45.0) + except concurrent.futures.TimeoutError: + print(" ❌ Timed out talking to the Codex backend — try again shortly.") + return + print(f" {result.message}") + def _show_usage(self): """Rate limits + session token usage (when a live agent exists) + Nous credits. diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 38ab051c8e47..765369f1c5e6 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -3951,6 +3951,15 @@ class GatewaySlashCommandsMixin: source = event.source session_key = self._session_key_for_source(source) + # `/usage reset [--force]` — redeem one banked Codex rate-limit reset + # credit. Parsed before the display path so it never mixes with the + # stats rendering below. + raw_args = event.get_command_args().strip() + args = [a.lower() for a in raw_args.split()] if raw_args else [] + wants_reset = bool(args) and args[0] == "reset" + if args and not wants_reset: + return t("gateway.usage.unknown_subcommand", args=raw_args) + # Try running agent first (mid-turn), then cached agent (between turns) agent = self._running_agents.get(session_key) if not agent or agent is _AGENT_PENDING_SENTINEL: @@ -3978,6 +3987,21 @@ class GatewaySlashCommandsMixin: provider = provider or persisted.get("billing_provider") base_url = base_url or persisted.get("billing_base_url") + if wants_reset: + normalized_provider = str(provider or "").strip().lower() + if normalized_provider != "openai-codex": + return t("gateway.usage.reset_wrong_provider") + force = "--force" in args[1:] + from agent.account_usage import redeem_codex_reset_credit + + result = await asyncio.to_thread( + redeem_codex_reset_credit, + base_url=base_url, + api_key=api_key, + force=force, + ) + return result.message + # Fetch account usage off the event loop so slow provider APIs don't # block the gateway. Failures are non-fatal -- account_lines stays []. account_lines: list[str] = [] diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index 4f32f7315cbd..10f8fbf046b8 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -228,7 +228,8 @@ COMMAND_REGISTRY: list[CommandDef] = [ CommandDef("help", "Show available commands", "Info"), CommandDef("restart", "Gracefully restart the gateway after draining active runs", "Session", gateway_only=True), - CommandDef("usage", "Show token usage and rate limits for the current session", "Info"), + CommandDef("usage", "Show token usage and rate limits; `reset` redeems a banked Codex limit reset", "Info", + args_hint="[reset [--force]]"), CommandDef("credits", "Show Nous credit balance and top up", "Info"), CommandDef("billing", "Manage Nous terminal billing — buy credits, auto-reload, limits", "Info", cli_only=True), diff --git a/locales/af.yaml b/locales/af.yaml index eecc3bdcf8c8..cc9a45060c93 100644 --- a/locales/af.yaml +++ b/locales/af.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Geskatte konteks: ~{count} tokens" detailed_after_first: "_(Gedetailleerde gebruik beskikbaar na die eerste agent-antwoord)_" no_data: "Geen gebruiksdata beskikbaar vir hierdie sessie nie." + unknown_subcommand: "Onbekende /usage subopdrag: `{args}`. Probeer `/usage` of `/usage reset [--force]`." + reset_wrong_provider: "Gebankte gebruikslimiet-terugstellings is slegs beskikbaar op die openai-codex verskaffer. Skakel eers oor met `/model`." credits: not_logged_in: "Nie by Nous Portal aangemeld nie. Meld aan om jou kredietsaldo te sien en op te laai." diff --git a/locales/de.yaml b/locales/de.yaml index 0d8539778554..093ee68be96f 100644 --- a/locales/de.yaml +++ b/locales/de.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Geschätzter Kontext: ~{count} Tokens" detailed_after_first: "_(Detaillierte Nutzung nach der ersten Agentenantwort verfügbar)_" no_data: "Keine Nutzungsdaten für diese Sitzung verfügbar." + unknown_subcommand: "Unbekannter /usage-Unterbefehl: `{args}`. Versuche `/usage` oder `/usage reset [--force]`." + reset_wrong_provider: "Gespeicherte Limit-Resets sind nur mit dem openai-codex-Anbieter verfügbar. Wechsle zuerst mit `/model`." credits: not_logged_in: "Nicht bei Nous Portal angemeldet. Melde dich an, um dein Guthaben zu sehen und aufzuladen." diff --git a/locales/en.yaml b/locales/en.yaml index 852ce88b4113..c47f875b0212 100644 --- a/locales/en.yaml +++ b/locales/en.yaml @@ -376,6 +376,8 @@ gateway: label_estimated_context: "Estimated context: ~{count} tokens" detailed_after_first: "_(Detailed usage available after the first agent response)_" no_data: "No usage data available for this session." + unknown_subcommand: "Unknown /usage subcommand: `{args}`. Try `/usage` or `/usage reset [--force]`." + reset_wrong_provider: "Banked usage resets are only available on the openai-codex provider. Switch with `/model` first." credits: not_logged_in: "Not logged into Nous Portal. Log in to see your credit balance and top up." diff --git a/locales/es.yaml b/locales/es.yaml index aaf0f1e14a83..c2ed50017c1a 100644 --- a/locales/es.yaml +++ b/locales/es.yaml @@ -361,6 +361,8 @@ gateway: label_estimated_context: "Contexto estimado: ~{count} tokens" detailed_after_first: "_(Uso detallado disponible tras la primera respuesta del agente)_" no_data: "No hay datos de uso disponibles para esta sesión." + unknown_subcommand: "Subcomando /usage desconocido: `{args}`. Prueba `/usage` o `/usage reset [--force]`." + reset_wrong_provider: "Los reinicios de límite acumulados solo están disponibles con el proveedor openai-codex. Cambia primero con `/model`." credits: not_logged_in: "No has iniciado sesión en Nous Portal. Inicia sesión para ver tu saldo de créditos y recargar." diff --git a/locales/fr.yaml b/locales/fr.yaml index 6173e684a1b5..da3bcaedd3ec 100644 --- a/locales/fr.yaml +++ b/locales/fr.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Contexte estimé : ~{count} jetons" detailed_after_first: "_(Utilisation détaillée disponible après la première réponse de l'agent)_" no_data: "Aucune donnée d'utilisation disponible pour cette session." + unknown_subcommand: "Sous-commande /usage inconnue : `{args}`. Essayez `/usage` ou `/usage reset [--force]`." + reset_wrong_provider: "Les réinitialisations de limite en réserve ne sont disponibles qu'avec le fournisseur openai-codex. Changez d'abord avec `/model`." credits: not_logged_in: "Non connecté à Nous Portal. Connecte-toi pour voir ton solde de crédits et recharger." diff --git a/locales/ga.yaml b/locales/ga.yaml index 7317c2eba171..8e56b693d590 100644 --- a/locales/ga.yaml +++ b/locales/ga.yaml @@ -368,6 +368,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Comhthéacs measta: ~{count} comhartha" detailed_after_first: "_(Úsáid mhionsonraithe ar fáil tar éis chéad fhreagra an ghníomhaire)_" no_data: "Níl aon sonraí úsáide ar fáil don seisiún seo." + unknown_subcommand: "Fo-ordú /usage anaithnid: `{args}`. Bain triail as `/usage` nó `/usage reset [--force]`." + reset_wrong_provider: "Níl athshocruithe teorann bainc ar fáil ach ar an soláthraí openai-codex. Athraigh le `/model` ar dtús." credits: not_logged_in: "Níl tú logáilte isteach i Nous Portal. Logáil isteach chun d'iarmhéid creidmheasa a fheiceáil agus breis a chur leis." diff --git a/locales/hu.yaml b/locales/hu.yaml index 547aa8767624..5bcd86f8c578 100644 --- a/locales/hu.yaml +++ b/locales/hu.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Becsült kontextus: ~{count} token" detailed_after_first: "_(A részletes használat az első ügynökválasz után érhető el)_" no_data: "Ehhez a munkamenethez nincsenek elérhető használati adatok." + unknown_subcommand: "Ismeretlen /usage alparancs: `{args}`. Próbáld: `/usage` vagy `/usage reset [--force]`." + reset_wrong_provider: "A félretett limit-visszaállítások csak az openai-codex szolgáltatónál érhetők el. Válts először a `/model` paranccsal." credits: not_logged_in: "Nincs bejelentkezve a Nous Portalra. Jelentkezz be a kreditegyenleg megtekintéséhez és feltöltéséhez." diff --git a/locales/it.yaml b/locales/it.yaml index 0cbdcf68fa4f..3cd822282702 100644 --- a/locales/it.yaml +++ b/locales/it.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Contesto stimato: ~{count} token" detailed_after_first: "_(L'uso dettagliato sarà disponibile dopo la prima risposta dell'agente)_" no_data: "Nessun dato di utilizzo disponibile per questa sessione." + unknown_subcommand: "Sottocomando /usage sconosciuto: `{args}`. Prova `/usage` o `/usage reset [--force]`." + reset_wrong_provider: "I reset dei limiti accumulati sono disponibili solo con il provider openai-codex. Cambia prima con `/model`." credits: not_logged_in: "Non hai effettuato l'accesso a Nous Portal. Accedi per vedere il saldo dei crediti e ricaricare." diff --git a/locales/ja.yaml b/locales/ja.yaml index 38f8822b71bb..75e92df462f2 100644 --- a/locales/ja.yaml +++ b/locales/ja.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "推定コンテキスト: ~{count} トークン" detailed_after_first: "_(詳細な使用状況は最初のエージェント応答後に利用可能)_" no_data: "このセッションの使用データはありません。" + unknown_subcommand: "不明な /usage サブコマンド: `{args}`。`/usage` または `/usage reset [--force]` をお試しください。" + reset_wrong_provider: "バンクされた使用制限リセットは openai-codex プロバイダーでのみ利用できます。まず `/model` で切り替えてください。" credits: not_logged_in: "Nous Portal にログインしていません。ログインすると残高の確認とチャージができます。" diff --git a/locales/ko.yaml b/locales/ko.yaml index 752012b0fd4f..d5a368f74c98 100644 --- a/locales/ko.yaml +++ b/locales/ko.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "예상 컨텍스트: 약 {count} 토큰" detailed_after_first: "_(자세한 사용량은 첫 에이전트 응답 이후 확인할 수 있습니다)_" no_data: "이 세션에 사용 가능한 사용량 데이터가 없습니다." + unknown_subcommand: "알 수 없는 /usage 하위 명령: `{args}`. `/usage` 또는 `/usage reset [--force]`를 사용해 보세요." + reset_wrong_provider: "적립된 사용량 한도 초기화는 openai-codex 공급자에서만 사용할 수 있습니다. 먼저 `/model`로 전환하세요." credits: not_logged_in: "Nous Portal에 로그인되어 있지 않습니다. 로그인하면 크레딧 잔액 확인 및 충전을 할 수 있습니다." diff --git a/locales/pt.yaml b/locales/pt.yaml index 12d53e995dc1..2a2fd3068d18 100644 --- a/locales/pt.yaml +++ b/locales/pt.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Contexto estimado: ~{count} tokens" detailed_after_first: "_(Utilização detalhada disponível após a primeira resposta do agente)_" no_data: "Não há dados de utilização disponíveis para esta sessão." + unknown_subcommand: "Subcomando /usage desconhecido: `{args}`. Tente `/usage` ou `/usage reset [--force]`." + reset_wrong_provider: "Os resets de limite acumulados só estão disponíveis no provedor openai-codex. Troque primeiro com `/model`." credits: not_logged_in: "Você não está conectado ao Nous Portal. Faça login para ver seu saldo de créditos e recarregar." diff --git a/locales/ru.yaml b/locales/ru.yaml index c19536afba8a..3ebf4c94b94f 100644 --- a/locales/ru.yaml +++ b/locales/ru.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Ориентировочный контекст: ~{count} токенов" detailed_after_first: "_(Подробное использование доступно после первого ответа агента)_" no_data: "Данные об использовании для этого сеанса отсутствуют." + unknown_subcommand: "Неизвестная подкоманда /usage: `{args}`. Попробуйте `/usage` или `/usage reset [--force]`." + reset_wrong_provider: "Накопленные сбросы лимитов доступны только с провайдером openai-codex. Сначала переключитесь через `/model`." credits: not_logged_in: "Вы не вошли в Nous Portal. Войдите, чтобы увидеть баланс кредитов и пополнить его." diff --git a/locales/tr.yaml b/locales/tr.yaml index 9a576624bc77..c62d98b57f81 100644 --- a/locales/tr.yaml +++ b/locales/tr.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Tahmini bağlam: ~{count} token" detailed_after_first: "_(Ayrıntılı kullanım, ilk ajan yanıtından sonra kullanılabilir)_" no_data: "Bu oturum için kullanım verisi yok." + unknown_subcommand: "Bilinmeyen /usage alt komutu: `{args}`. `/usage` veya `/usage reset [--force]` deneyin." + reset_wrong_provider: "Biriktirilen limit sıfırlamaları yalnızca openai-codex sağlayıcısında kullanılabilir. Önce `/model` ile geçiş yapın." credits: not_logged_in: "Nous Portal'a giriş yapılmadı. Bakiyenizi görmek ve yükleme yapmak için giriş yapın." diff --git a/locales/uk.yaml b/locales/uk.yaml index e78d47653d44..51f8d6c4708f 100644 --- a/locales/uk.yaml +++ b/locales/uk.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "Орієнтовний контекст: ~{count} токенів" detailed_after_first: "_(Детальне використання доступне після першої відповіді агента)_" no_data: "Дані про використання для цього сеансу відсутні." + unknown_subcommand: "Невідома підкоманда /usage: `{args}`. Спробуйте `/usage` або `/usage reset [--force]`." + reset_wrong_provider: "Накопичені скидання лімітів доступні лише з провайдером openai-codex. Спочатку перемкніться через `/model`." credits: not_logged_in: "Ви не ввійшли в Nous Portal. Увійдіть, щоб переглянути баланс кредитів і поповнити його." diff --git a/locales/zh-hant.yaml b/locales/zh-hant.yaml index 6602231812f1..537492311724 100644 --- a/locales/zh-hant.yaml +++ b/locales/zh-hant.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "預估上下文：~{count} 個 token" detailed_after_first: "_（首次代理回應後可檢視詳細使用情況）_" no_data: "此工作階段沒有可用的使用資料。" + unknown_subcommand: "未知的 /usage 子命令:`{args}`。請嘗試 `/usage` 或 `/usage reset [--force]`。" + reset_wrong_provider: "儲存的用量重設僅在 openai-codex 提供者上可用。請先使用 `/model` 切換。" credits: not_logged_in: "未登入 Nous Portal。登入後即可查看額度餘額並儲值。" diff --git a/locales/zh.yaml b/locales/zh.yaml index 2e307e4d60ab..d43a804ad4fc 100644 --- a/locales/zh.yaml +++ b/locales/zh.yaml @@ -364,6 +364,8 @@ Future messages in this room will use that transcript until `/reset` or another label_estimated_context: "估计上下文：~{count} 个令牌" detailed_after_first: "_（首次代理响应后可查看详细使用情况）_" no_data: "此会话暂无使用数据。" + unknown_subcommand: "未知的 /usage 子命令：`{args}`。请尝试 `/usage` 或 `/usage reset [--force]`。" + reset_wrong_provider: "存储的用量重置仅在 openai-codex 提供商上可用。请先使用 `/model` 切换。" credits: not_logged_in: "未登录 Nous Portal。登录后即可查看额度余额并充值。" diff --git a/tests/agent/test_account_usage.py b/tests/agent/test_account_usage.py index 41950c70a12c..86a88d4d8230 100644 --- a/tests/agent/test_account_usage.py +++ b/tests/agent/test_account_usage.py @@ -235,3 +235,204 @@ def test_codex_usage_treats_wham_used_percent_as_used_not_remaining(monkeypatch) assert "14% used" in rendered assert "15% used" not in rendered assert "86% used" not in rendered + + +# ── Banked rate-limit reset credits (`/usage reset`) ───────────────────────── + + +class _FakeResetClient: + """GET returns the usage payload; POST returns the consume payload.""" + + def __init__(self, calls, usage_payload, consume_payload=None): + self.calls = calls + self.usage_payload = usage_payload + self.consume_payload = consume_payload or {} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get(self, url, headers): + self.calls.append({"method": "GET", "url": url, "headers": headers}) + return _FakeResponse(self.usage_payload) + + def post(self, url, headers=None, json=None): + self.calls.append({"method": "POST", "url": url, "headers": headers, "json": json}) + return _FakeResponse(self.consume_payload) + + +def _usage_payload_with_resets(primary_used, secondary_used, banked): + return { + "plan_type": "plus", + "rate_limit": { + "primary_window": {"used_percent": primary_used, "reset_at": 1779846359}, + "secondary_window": {"used_percent": secondary_used, "reset_at": 1780230796}, + }, + "rate_limit_reset_credits": {"available_count": banked}, + "credits": {"has_credits": False}, + } + + +def test_usage_snapshot_shows_banked_resets_hint(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient(calls, _usage_payload_with_resets(21, 4, 2)), + ) + + snapshot = account_usage.fetch_account_usage( + "openai-codex", + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert snapshot is not None + rendered = "\n".join(account_usage.render_account_usage_lines(snapshot)) + assert "You have 2 resets banked - use /usage reset to activate" in rendered + + +def test_usage_snapshot_hides_reset_hint_when_none_banked(monkeypatch, codex_usage_payload): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient(calls, codex_usage_payload), + ) + + snapshot = account_usage.fetch_account_usage( + "openai-codex", + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert snapshot is not None + rendered = "\n".join(account_usage.render_account_usage_lines(snapshot)) + assert "banked" not in rendered + + +def test_redeem_blocked_when_limits_not_exhausted(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient(calls, _usage_payload_with_resets(60, 30, 2)), + ) + + result = account_usage.redeem_codex_reset_credit( + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert result.status == "not_exhausted" + assert not result.redeemed + assert "--force" in result.message + assert "60% used" in result.message + assert result.available_count == 2 + # The consume endpoint must never be hit — the credit is protected. + assert [c["method"] for c in calls] == ["GET"] + + +def test_redeem_force_bypasses_exhaustion_guard(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient( + calls, + _usage_payload_with_resets(60, 30, 2), + consume_payload={"code": "reset", "windows_reset": 2}, + ), + ) + + result = account_usage.redeem_codex_reset_credit( + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + force=True, + ) + + assert result.redeemed + assert result.windows_reset == 2 + assert result.available_count == 1 # 2 banked - 1 spent + assert "1 banked reset remaining" in result.message + post = [c for c in calls if c["method"] == "POST"][0] + assert post["url"] == "https://chatgpt.com/backend-api/wham/rate-limit-reset-credits/consume" + assert post["json"]["redeem_request_id"] # idempotency key present + assert "credit_id" not in post["json"] + + +def test_redeem_allowed_without_force_when_window_exhausted(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient( + calls, + _usage_payload_with_resets(100, 42, 1), + consume_payload={"code": "reset", "windows_reset": 2}, + ), + ) + + result = account_usage.redeem_codex_reset_credit( + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert result.redeemed + assert result.available_count == 0 + assert "0 banked resets remaining" in result.message + + +def test_redeem_refuses_when_no_credits_banked(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient(calls, _usage_payload_with_resets(100, 100, 0)), + ) + + result = account_usage.redeem_codex_reset_credit( + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert result.status == "no_credits_banked" + assert [c["method"] for c in calls] == ["GET"] + + +def test_redeem_nothing_to_reset_reports_credit_not_spent(monkeypatch): + calls = [] + monkeypatch.setattr( + account_usage.httpx, + "Client", + lambda timeout: _FakeResetClient( + calls, + _usage_payload_with_resets(100, 100, 3), + consume_payload={"code": "nothing_to_reset"}, + ), + ) + + result = account_usage.redeem_codex_reset_credit( + base_url="https://chatgpt.com/backend-api/codex", + api_key="live-agent-token", + ) + + assert result.status == "nothing_to_reset" + assert not result.redeemed + assert "NOT spent" in result.message + assert result.available_count == 3 + + +def test_redeem_missing_credentials_reports_unavailable(monkeypatch): + monkeypatch.setattr( + account_usage, + "_resolve_codex_usage_credentials", + lambda base_url, api_key: (_ for _ in ()).throw(RuntimeError("no creds")), + ) + + result = account_usage.redeem_codex_reset_credit() + + assert result.status == "unavailable" + assert "hermes auth" in result.message diff --git a/tests/gateway/test_usage_command.py b/tests/gateway/test_usage_command.py index e2a207cfbc70..a93ebe5b759b 100644 --- a/tests/gateway/test_usage_command.py +++ b/tests/gateway/test_usage_command.py @@ -245,6 +245,77 @@ class TestUsageAccountSection: assert "📈 **Account limits**" in result +class TestUsageReset: + """`/usage reset [--force]` — banked Codex reset redemption via the gateway.""" + + def _event(self, args): + event = MagicMock() + event.get_command_args.return_value = args + return event + + @pytest.mark.asyncio + async def test_reset_dispatches_redeem_for_codex_agent(self, monkeypatch): + agent = _make_mock_agent(provider="openai-codex", + base_url="https://chatgpt.com/backend-api/codex", + api_key="tok") + runner = _make_runner(SK, cached_agent=agent) + + seen = {} + + def fake_redeem(*, base_url=None, api_key=None, force=False): + seen.update(base_url=base_url, api_key=api_key, force=force) + from agent.account_usage import CodexResetRedeemResult + return CodexResetRedeemResult(status="reset", message="✅ redeemed", available_count=1) + + monkeypatch.setattr("agent.account_usage.redeem_codex_reset_credit", fake_redeem) + + result = await runner._handle_usage_command(self._event("reset")) + + assert result == "✅ redeemed" + assert seen["force"] is False + assert seen["api_key"] == "tok" + + @pytest.mark.asyncio + async def test_reset_force_flag_propagates(self, monkeypatch): + agent = _make_mock_agent(provider="openai-codex", api_key="tok") + runner = _make_runner(SK, cached_agent=agent) + + seen = {} + + def fake_redeem(*, base_url=None, api_key=None, force=False): + seen["force"] = force + from agent.account_usage import CodexResetRedeemResult + return CodexResetRedeemResult(status="reset", message="ok") + + monkeypatch.setattr("agent.account_usage.redeem_codex_reset_credit", fake_redeem) + + await runner._handle_usage_command(self._event("reset --force")) + + assert seen["force"] is True + + @pytest.mark.asyncio + async def test_reset_rejected_on_non_codex_provider(self, monkeypatch): + agent = _make_mock_agent(provider="openrouter") + runner = _make_runner(SK, cached_agent=agent) + monkeypatch.setattr( + "agent.account_usage.redeem_codex_reset_credit", + lambda **kw: (_ for _ in ()).throw(AssertionError("must not redeem")), + ) + + result = await runner._handle_usage_command(self._event("reset")) + + assert "openai-codex" in result + + @pytest.mark.asyncio + async def test_unknown_subcommand_rejected(self): + agent = _make_mock_agent(provider="openai-codex") + runner = _make_runner(SK, cached_agent=agent) + + result = await runner._handle_usage_command(self._event("bogus")) + + assert "Unknown /usage subcommand" in result + + class TestUsageContextBreakdown: """The /usage output includes the per-category context breakdown.""" diff --git a/website/docs/user-guide/cli.md b/website/docs/user-guide/cli.md index 7bbc1fe6ec56..f786e5a42f91 100644 --- a/website/docs/user-guide/cli.md +++ b/website/docs/user-guide/cli.md @@ -90,6 +90,8 @@ The bar adapts to terminal width — full layout at ≥ 76 columns, compact at 5 Use `/usage` for a detailed breakdown including per-category costs (input vs output tokens). +On the `openai-codex` provider, `/usage` also shows any banked usage-limit resets on your ChatGPT account ("You have N resets banked - use /usage reset to activate"). `/usage reset` redeems one banked reset, fully restoring your 5-hour and weekly limits. Hermes refuses to redeem while your limits aren't exhausted (a banked reset restores the full allowance, so spending it early wastes it) — pass `/usage reset --force` to redeem anyway. + ### Session Resume Display When resuming a previous session (`hermes -c` or `hermes --resume `), a "Previous Conversation" panel appears between the banner and the input prompt, showing a compact recap of the conversation history. See [Sessions — Conversation Recap on Resume](sessions.md#conversation-recap-on-resume) for details and configuration. diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index 8011191c89ff..ac69b9ffd048 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -172,7 +172,7 @@ hermes gateway status --system # Linux only: inspect the system service | `/compress` | Manually compress conversation context | | `/title [name]` | Set or show the session title | | `/resume [name]` | Resume a previously named session | -| `/usage` | Show token usage for this session | +| `/usage` | Show token usage for this session (`/usage reset [--force]` redeems a banked Codex limit reset) | | `/insights [days]` | Show usage insights and analytics | | `/reasoning [level\|show\|hide]` | Change reasoning effort or toggle reasoning display | | `/voice [on\|off\|tts\|join\|leave\|status]` | Control messaging voice replies and Discord voice-channel behavior | From 8a7d32d4e40f9e562e3a825466f55e1d7f15b378 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Tue, 14 Jul 2026 01:32:29 +0800 Subject: [PATCH 039/149] fix(conversation): clear stale housekeeping fallback on substantive tool-only turns A cached _last_content_with_tools response from a housekeeping-only turn could survive a later substantive tool-only turn. When the model returned an empty response, Hermes incorrectly finalized the older housekeeping narration instead of invoking the post-tool empty-response nudge. Production impact: scheduled cron jobs could return early without completing their actual work (e.g., daily report job returning a housekeeping message instead of producing the report artifact). Root cause: The fallback state was only updated when a turn had both content AND tool_calls. A turn with tool_calls but empty visible content would skip state updates entirely, leaving stale fallback state intact. Fix: Classify tools in every tool-call turn (regardless of visible content). When any tool is substantive (non-housekeeping), clear the older fallback state before processing later empty responses. This prevents two-turn-old housekeeping narration from being treated as if it belonged to the immediately preceding substantive tool turn. Regression test added: tests/run_agent/test_conversation_fallback_state.py Fixes #63860 --- agent/conversation_loop.py | 28 ++-- .../test_conversation_fallback_state.py | 126 ++++++++++++++++++ 2 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 tests/run_agent/test_conversation_fallback_state.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index dd05ce1e4795..5906cf60eada 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -4676,11 +4676,30 @@ def run_conversation( assistant_msg = agent._build_assistant_message(assistant_message, finish_reason) + turn_content = assistant_message.content or "" + + # Classify tools in this turn to determine if they are all housekeeping. + # This classification is needed regardless of whether the turn has visible content, + # because a substantive tool-only turn must invalidate any older housekeeping fallback. + _HOUSEKEEPING_TOOLS = frozenset({ + "memory", "todo", "skill_manage", "session_search", + }) + _all_housekeeping = all( + tc.function.name in _HOUSEKEEPING_TOOLS + for tc in assistant_message.tool_calls + ) + + # If this turn has substantive tools (non-housekeeping), clear any older fallback. + # Prevents a two-turn-old housekeeping narration from being treated as if it belonged + # to the immediately preceding substantive tool turn. + if assistant_message.tool_calls and not _all_housekeeping: + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + # If this turn has both content AND tool_calls, capture the content # as a fallback final response. Common pattern: model delivers its # answer and calls memory/skill tools as a side-effect in the same # turn. If the follow-up turn after tools is empty, we use this. - turn_content = assistant_message.content or "" if turn_content and agent._has_content_after_think_block(turn_content): agent._last_content_with_tools = turn_content # Only mute subsequent output when EVERY tool call in @@ -4688,13 +4707,6 @@ def run_conversation( # skill_manage, etc.). If any substantive tool is present # (search_files, read_file, write_file, terminal, ...), # keep output visible so the user sees progress. - _HOUSEKEEPING_TOOLS = frozenset({ - "memory", "todo", "skill_manage", "session_search", - }) - _all_housekeeping = all( - tc.function.name in _HOUSEKEEPING_TOOLS - for tc in assistant_message.tool_calls - ) agent._last_content_tools_all_housekeeping = _all_housekeeping if _all_housekeeping and agent._has_stream_consumers(): agent._mute_post_response = True diff --git a/tests/run_agent/test_conversation_fallback_state.py b/tests/run_agent/test_conversation_fallback_state.py new file mode 100644 index 000000000000..126e0a861ae4 --- /dev/null +++ b/tests/run_agent/test_conversation_fallback_state.py @@ -0,0 +1,126 @@ +"""Regression tests for conversation loop fallback state management.""" +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from run_agent import AIAgent + + +def _tool_defs(*names): + """Helper: create minimal tool definitions for given names.""" + return [ + { + "type": "function", "function": { + "name": name, + "description": "test tool", + "parameters": {"type": "object", "properties": {}}, + } + } + for name in names + ] + + +def _tool_call(name, call_id): + """Helper: create a minimal tool call object.""" + return SimpleNamespace( + id=call_id, type="function", + function=SimpleNamespace(name=name, arguments="{}"), + ) + + +def _response(*, content, finish_reason, tool_calls=None): + """Helper: create a minimal API response object.""" + message = SimpleNamespace(content=content, tool_calls=tool_calls) + choice = SimpleNamespace(message=message, finish_reason=finish_reason) + return SimpleNamespace(choices=[choice], model="test/model", usage=None) + + +def test_substantive_tool_only_turn_invalidates_older_housekeeping_fallback(): + """ + Regression test for #63860. + + A cached `_last_content_with_tools` response from a housekeeping-only turn + must not survive a later substantive tool-only turn. When the model returns + an empty response after the substantive tool turn, the system should enter + the post-tool nudge path, not use the stale housekeeping fallback. + + Production impact: scheduled cron jobs could return early without + completing their actual work (e.g., daily report job returning a + housekeeping message instead of producing the report artifact). + + Test sequence: + 1. Content + todo (housekeeping) → sets fallback, marks as all-housekeeping + 2. Empty content + web_search (substantive) → should CLEAR old fallback + 3. Empty content, no tool calls → should enter post-tool nudge, not use old fallback + 4. Content "Recovered after nudge." → should be returned as final response + + Before the fix: + - Step 2 would not clear the fallback state (no visible content) + - Step 3 would incorrectly use the housekeeping fallback from step 1 + - API calls would stop at 3, never reaching the nudge response + + After the fix: + - Step 2 classifies tools and clears the fallback because web_search is substantive + - Step 3 enters the post-tool nudge path (no stale housekeeping fallback available) + - Step 4 returns the nudge response as the final answer + """ + with ( + patch("run_agent.get_tool_definitions", return_value=_tool_defs("todo", "web_search")), + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + ): + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1/", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + + agent._cached_system_prompt = "You are helpful." + agent._use_prompt_caching = False + agent.tool_delay = 0 + agent.compression_enabled = False + agent.save_trajectories = False + agent.valid_tool_names = {"todo", "web_search"} + agent.client = MagicMock() + agent.client.chat.completions.create.side_effect = [ + # Turn 1: Content + housekeeping tool + _response( + content="I'll begin the work.", + finish_reason="tool_calls", + tool_calls=[_tool_call("todo", "todo1")], + ), + # Turn 2: Empty content + substantive tool (should clear stale fallback) + _response( + content="", + finish_reason="tool_calls", + tool_calls=[_tool_call("web_search", "search1")], + ), + # Turn 3: Empty response (should enter nudge path, not use stale fallback) + _response(content="", finish_reason="stop"), + # Turn 4: Nudge response + _response(content="Recovered after nudge.", finish_reason="stop"), + ] + + with ( + patch("run_agent.handle_function_call", return_value="ok"), + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation("do the full task") + + assert result["final_response"] == "Recovered after nudge.", ( + f"Expected nudge recovery response, got: {result['final_response']}. " + f"This indicates the stale housekeeping fallback was incorrectly used." + ) + assert result["api_calls"] == 4, ( + f"Expected 4 API calls (including nudge), got: {result['api_calls']}. " + f"This indicates the conversation exited early without retrying." + ) + assert result["turn_exit_reason"].startswith("text_response"), ( + f"Expected text_response exit, got: {result['turn_exit_reason']}. " + f"This indicates the wrong fallback path was taken." + ) \ No newline at end of file From 3c2886f599692a341570546ba5af34fe3e451345 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:27:29 +0530 Subject: [PATCH 040/149] fix(conversation): clear _mute_post_response on substantive tool-only turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvage of #63888. The original fix clears stale _last_content_with_tools on substantive tool-only turns but doesn't clear _mute_post_response, which a prior housekeeping turn may have set. This suppresses tool progress output via _vprint until the no-tool-call branch resets it at line ~4834 — after all tools have finished executing. Fix: also reset _mute_post_response = False when clearing stale fallback. Added test: verify pure housekeeping turns (content + only housekeeping tools) still set the fallback correctly — the original use case the fallback was designed for. Co-authored-by: liuhao1024 --- agent/conversation_loop.py | 7 +++ .../test_conversation_fallback_state.py | 54 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 5906cf60eada..2468fcfa090b 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -4695,6 +4695,13 @@ def run_conversation( if assistant_message.tool_calls and not _all_housekeeping: agent._last_content_with_tools = None agent._last_content_tools_all_housekeeping = False + # Also clear the mute flag: a prior housekeeping turn may + # have set _mute_post_response (line ~4667), and the + # substantive tools in THIS turn should produce visible + # progress output. Without this reset, _vprint suppresses + # tool progress until the no-tool-call branch clears it at + # line ~4834 — after all tools have finished. + agent._mute_post_response = False # If this turn has both content AND tool_calls, capture the content # as a fallback final response. Common pattern: model delivers its diff --git a/tests/run_agent/test_conversation_fallback_state.py b/tests/run_agent/test_conversation_fallback_state.py index 126e0a861ae4..c92cee2937c6 100644 --- a/tests/run_agent/test_conversation_fallback_state.py +++ b/tests/run_agent/test_conversation_fallback_state.py @@ -123,4 +123,58 @@ def test_substantive_tool_only_turn_invalidates_older_housekeeping_fallback(): assert result["turn_exit_reason"].startswith("text_response"), ( f"Expected text_response exit, got: {result['turn_exit_reason']}. " f"This indicates the wrong fallback path was taken." + ) + + +def test_housekeeping_only_turn_still_sets_fallback(): + """Regression: pure housekeeping turns (content + only housekeeping tools) + must still set the fallback so the post-response mute path works. This + verifies the fix doesn't break the original use case the fallback was + designed for. + """ + with ( + patch("run_agent.get_tool_definitions", return_value=_tool_defs("memory")), + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + ): + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1/", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + + agent._cached_system_prompt = "You are helpful." + agent._use_prompt_caching = False + agent.tool_delay = 0 + agent.compression_enabled = False + agent.save_trajectories = False + agent.valid_tool_names = {"memory"} + agent.client = MagicMock() + agent.client.chat.completions.create.side_effect = [ + # Turn 1: Content + housekeeping tool (should set fallback) + _response( + content="You're welcome!", + finish_reason="tool_calls", + tool_calls=[_tool_call("memory", "mem1")], + ), + # Turn 2: Empty response (should use the housekeeping fallback) + _response(content="", finish_reason="stop"), + ] + + with ( + patch("run_agent.handle_function_call", return_value="ok"), + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation("save this") + + assert result["final_response"] == "You're welcome!", ( + f"Expected housekeeping fallback content, got: {result['final_response']}. " + f"Pure housekeeping turns should still set the fallback." + ) + assert "fallback_prior_turn_content" in result.get("turn_exit_reason", ""), ( + f"Expected fallback_prior_turn_content exit, got: {result['turn_exit_reason']}." ) \ No newline at end of file From 03fbf6edbb92a5306c15dbb1bf437d68ebeea655 Mon Sep 17 00:00:00 2001 From: mdc2122 <259970464+mdc2122@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:25:53 +0530 Subject: [PATCH 041/149] fix(kanban): nudge workers that exit without complete/block Add a bounded turn-end stop guard for kanban workers. When a worker tries to exit with finish_reason=stop without having called kanban_complete or kanban_block, inject up to two synthetic nudges so the conversation loop continues instead of exiting cleanly (which the dispatcher records as protocol_violation). Mirrors the existing verify-on-stop pattern: same ephemeral scaffolding flag (_kanban_stop_synthetic), same role-alternation contract, same _pending_verification_response fallback for budget exhaustion. Disabled by default (gated on HERMES_KANBAN_TASK env var set by the dispatcher); kill switch via HERMES_KANBAN_STOP_NUDGE=0. Salvaged from #62262 by @mdc2122. The original branch was 272 commits behind main with ~538 files of stale-base reversions; this salvage applies only the 4 substantive files (agent/kanban_stop.py, conversation_loop.py insertion, run_agent.py _EPHEMERAL_SCAFFOLDING_FLAGS, tests/agent/test_kanban_stop.py). --- agent/conversation_loop.py | 47 ++++++++++++++ agent/kanban_stop.py | 107 ++++++++++++++++++++++++++++++++ run_agent.py | 2 + tests/agent/test_kanban_stop.py | 95 ++++++++++++++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 agent/kanban_stop.py create mode 100644 tests/agent/test_kanban_stop.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 2468fcfa090b..23b977973642 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -5329,6 +5329,53 @@ def run_conversation( final_response = None continue + # ── Kanban worker terminal-tool stop guard ───────────── + # Workers must end with kanban_complete / kanban_block. + # Models sometimes narrate the next step ("Let me write the + # report") and stop with finish_reason=stop — a clean exit + # that the dispatcher records as protocol_violation. Nudge + # once or twice before allowing that exit. + try: + from agent.kanban_stop import build_kanban_stop_nudge + + _kanban_nudge = build_kanban_stop_nudge( + messages=messages, + attempts=getattr(agent, "_kanban_stop_nudges", 0), + ) + except Exception: + logger.debug("kanban stop-loop check failed", exc_info=True) + _kanban_nudge = None + + if _kanban_nudge: + agent._kanban_stop_nudges = ( + getattr(agent, "_kanban_stop_nudges", 0) + 1 + ) + final_msg["finish_reason"] = "kanban_terminal_required" + final_msg["_kanban_stop_synthetic"] = True + messages.append(final_msg) + messages.append({ + "role": "user", + "content": _kanban_nudge, + "_kanban_stop_synthetic": True, + }) + agent._session_messages = messages + logger.info( + "kanban stop-loop nudge issued (attempt %d) task=%s", + agent._kanban_stop_nudges, + os.environ.get("HERMES_KANBAN_TASK", ""), + ) + agent._emit_status( + "⚠️ Kanban worker tried to exit without " + "kanban_complete/kanban_block — nudging to finish" + ) + # Same finalizer contract as verify-on-stop: clear + # final_response while continuing so a later budget + # exhaustion path does not treat the narrated stop as + # a completed answer. + _pending_verification_response = final_response + final_response = None + continue + messages.append(final_msg) _turn_exit_reason = f"text_response(finish_reason={finish_reason})" diff --git a/agent/kanban_stop.py b/agent/kanban_stop.py new file mode 100644 index 000000000000..bb97ddd425f0 --- /dev/null +++ b/agent/kanban_stop.py @@ -0,0 +1,107 @@ +"""Turn-end guard for kanban workers. + +Kanban workers must end with ``kanban_complete`` or ``kanban_block``. Models +(especially GLM / Qwen families) sometimes narrate the next step +("Let me write the report now") and stop with ``finish_reason=stop`` and no +tool calls. Hermes treats that as a clean exit → ``rc=0`` → dispatcher +``protocol_violation``. + +This module is policy-only: when a kanban worker tries to finish without a +terminal board tool, return a bounded synthetic nudge so the conversation +loop continues instead of exiting. +""" + +from __future__ import annotations + +import os +from typing import Any, Iterable, Optional + + +_TERMINAL_KANBAN_TOOLS = frozenset({"kanban_complete", "kanban_block"}) + +_DEFAULT_MAX_ATTEMPTS = 2 + + +def kanban_stop_nudge_enabled() -> bool: + """Return whether the kanban stop-guard is active for this process. + + On when ``HERMES_KANBAN_TASK`` is set (dispatcher-spawned worker), unless + ``HERMES_KANBAN_STOP_NUDGE`` explicitly disables it. + """ + env = os.environ.get("HERMES_KANBAN_STOP_NUDGE") + if env is not None and env.strip().lower() in {"0", "false", "no", "off"}: + return False + task = (os.environ.get("HERMES_KANBAN_TASK") or "").strip() + return bool(task) + + +def _tool_call_name(tc: Any) -> str: + if isinstance(tc, dict): + fn = tc.get("function") + if isinstance(fn, dict): + return str(fn.get("name") or "") + return str(tc.get("name") or "") + fn = getattr(tc, "function", None) + if fn is not None: + return str(getattr(fn, "name", "") or "") + return str(getattr(tc, "name", "") or "") + + +def session_called_kanban_terminal(messages: Iterable[dict] | None) -> bool: + """True if this conversation already invoked a terminal kanban tool.""" + if not messages: + return False + for msg in messages: + if not isinstance(msg, dict): + continue + role = msg.get("role") + if role == "assistant": + for tc in msg.get("tool_calls") or []: + if _tool_call_name(tc) in _TERMINAL_KANBAN_TOOLS: + return True + elif role == "tool": + name = str(msg.get("name") or "") + if name in _TERMINAL_KANBAN_TOOLS: + return True + return False + + +def build_kanban_stop_nudge( + *, + messages: Iterable[dict] | None = None, + attempts: int = 0, + max_attempts: int = _DEFAULT_MAX_ATTEMPTS, + task_id: Optional[str] = None, +) -> Optional[str]: + """Return a synthetic follow-up when a kanban worker exits without a terminal tool. + + Returns ``None`` when the guard should not fire (not a kanban worker, + already completed/blocked, or nudge budget exhausted). + """ + if not kanban_stop_nudge_enabled(): + return None + if attempts >= max_attempts: + return None + if session_called_kanban_terminal(messages): + return None + + tid = (task_id or os.environ.get("HERMES_KANBAN_TASK") or "").strip() or "this task" + return ( + "[System: You are a Hermes kanban worker. A plain-text reply is NOT a " + "terminal state for the board.\n\n" + f"Task `{tid}` is still `running`. Ending now without a board tool " + "causes a protocol violation (clean exit with no " + "`kanban_complete` / `kanban_block`).\n\n" + "Do this immediately in your next response — do not narrate intent:\n" + "1. Finish any remaining deliverable (write the required file(s) now).\n" + "2. Call `kanban_complete(summary=..., artifacts=[...])` if the work " + "is done, OR `kanban_block(reason=...)` if you are blocked.\n\n" + "Never end a turn with only a promise of future action.]" + ) + + +__all__ = [ + "build_kanban_stop_nudge", + "kanban_stop_nudge_enabled", + "session_called_kanban_terminal", +] diff --git a/run_agent.py b/run_agent.py index fe378f396ae9..805f5ced6fb0 100644 --- a/run_agent.py +++ b/run_agent.py @@ -232,6 +232,8 @@ _EPHEMERAL_SCAFFOLDING_FLAGS = ( # transcript and breaks prompt-prefix cache reuse on later turns. (#55733) "_verification_stop_synthetic", "_pre_verify_synthetic", + # kanban worker stop-guard: narrated exit without kanban_complete/block + "_kanban_stop_synthetic", ) diff --git a/tests/agent/test_kanban_stop.py b/tests/agent/test_kanban_stop.py new file mode 100644 index 000000000000..62ba78a42a79 --- /dev/null +++ b/tests/agent/test_kanban_stop.py @@ -0,0 +1,95 @@ +"""Tests for the kanban worker turn-end stop guard.""" + +from __future__ import annotations + +import pytest + +from agent.kanban_stop import ( + build_kanban_stop_nudge, + kanban_stop_nudge_enabled, + session_called_kanban_terminal, +) + + +@pytest.fixture +def clear_kanban_env(monkeypatch): + for var in ("HERMES_KANBAN_TASK", "HERMES_KANBAN_STOP_NUDGE"): + monkeypatch.delenv(var, raising=False) + return monkeypatch + + +def test_disabled_without_kanban_task(clear_kanban_env): + assert kanban_stop_nudge_enabled() is False + assert build_kanban_stop_nudge(messages=[]) is None + + +def test_enabled_with_kanban_task(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + assert kanban_stop_nudge_enabled() is True + + +def test_env_can_disable(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + clear_kanban_env.setenv("HERMES_KANBAN_STOP_NUDGE", "0") + assert kanban_stop_nudge_enabled() is False + assert build_kanban_stop_nudge(messages=[]) is None + + +def test_nudge_when_no_terminal_tool(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_46be8aa5") + messages = [ + {"role": "user", "content": "work kanban task"}, + { + "role": "assistant", + "content": "Let me write the comprehensive recipe.", + "tool_calls": [ + { + "id": "1", + "type": "function", + "function": {"name": "kanban_heartbeat", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "name": "kanban_heartbeat", "tool_call_id": "1", "content": "ok"}, + ] + nudge = build_kanban_stop_nudge(messages=messages, attempts=0) + assert nudge is not None + assert "kanban_complete" in nudge + assert "kanban_block" in nudge + assert "t_46be8aa5" in nudge + assert "protocol violation" in nudge.lower() or "protocol" in nudge.lower() + + +def test_no_nudge_after_kanban_complete(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "1", + "type": "function", + "function": {"name": "kanban_complete", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "name": "kanban_complete", "tool_call_id": "1", "content": "done"}, + ] + assert session_called_kanban_terminal(messages) is True + assert build_kanban_stop_nudge(messages=messages) is None + + +def test_no_nudge_after_kanban_block(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + messages = [ + {"role": "tool", "name": "kanban_block", "tool_call_id": "1", "content": "blocked"}, + ] + assert build_kanban_stop_nudge(messages=messages) is None + + +def test_nudge_budget_exhausted(clear_kanban_env): + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + assert build_kanban_stop_nudge(messages=[], attempts=2) is None + assert build_kanban_stop_nudge(messages=[], attempts=1, max_attempts=1) is None + assert build_kanban_stop_nudge(messages=[], attempts=0, max_attempts=1) is not None From 9aba95b053170e3bdd326d18ac9c57d8acb25574 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Mon, 13 Jul 2026 11:44:32 +0800 Subject: [PATCH 042/149] fix(state): enforce synchronous=FULL on macOS to prevent btree corruption On Darwin, the default synchronous=NORMAL only calls fsync(), which Apple explicitly states does not guarantee data-on-platter or write-ordering. During a WAL checkpoint race with process termination (e.g., launchd shutdown), this can leave the main DB with half-written btree pages, resulting in btreeInitPage error 11 corruption. WAL mode's durability guarantee assumes the OS honors fsync barriers; macOS does not unless we explicitly set synchronous=FULL (which issues fsync() and F_FULLFSYNC via checkpoint_fullfsync=1). Previously, apply_wal_with_fallback() skipped setting synchronous=FULL when the DB was already in WAL mode, leaving connections at the unsafe synchronous=NORMAL default. This commit adds _enforce_macos_synchronous_full() to always enforce synchronous=FULL on macOS after any WAL activation. Fixes #63531 --- hermes_state.py | 30 +++++++++++++++++ tests/test_hermes_state.py | 69 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/hermes_state.py b/hermes_state.py index 67dc9d5d8cb9..a8c529acecfd 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -356,6 +356,34 @@ def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None: pass +def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: + """Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption. + + On Darwin, the default ``synchronous=NORMAL`` only calls ``fsync()``, + which Apple's fsync(2) man page explicitly states does *not* guarantee + data-on-platter or write-ordering. During a WAL checkpoint race with + process termination (e.g., launchd shutdown), this can leave the main + DB with half-written btree pages → ``btreeInitPage error 11``. + + WAL mode's durability guarantee assumes the OS honors fsync barriers; + macOS does not unless we explicitly set ``synchronous=FULL`` (which + issues ``fsync()`` *and* ``F_FULLFSYNC`` via checkpoint_fullfsync=1). + + This function is called after any successful WAL activation (either + from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing + an existing WAL mode). It ensures macOS connections always use FULL + synchronous mode, even if a prior connection set ``synchronous=NORMAL``. + + Best-effort: never raises. + """ + if sys.platform != "darwin": + return + try: + conn.execute("PRAGMA synchronous=FULL") + except sqlite3.OperationalError: + pass + + def apply_wal_with_fallback( conn: sqlite3.Connection, *, @@ -387,6 +415,7 @@ def apply_wal_with_fallback( current_mode = conn.execute("PRAGMA journal_mode").fetchone() if current_mode and current_mode[0] == "wal": _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) return "wal" except sqlite3.OperationalError: pass @@ -394,6 +423,7 @@ def apply_wal_with_fallback( try: conn.execute("PRAGMA journal_mode=WAL") _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) return "wal" except sqlite3.OperationalError as exc: msg = str(exc).lower() diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index c45ad6344ccb..2afcd6e4c6b0 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -5180,6 +5180,75 @@ class TestApplyWalProbe: assert not any("checkpoint_fullfsync" in sql for sql in conn.executed), ( "checkpoint_fullfsync must not be issued off macOS" ) + assert not any("synchronous=FULL" in sql for sql in conn.executed), ( + "synchronous=FULL must not be issued off macOS" + ) + + def test_macos_synchronous_full_enforced_fresh(self, tmp_path, monkeypatch): + """On Darwin, apply_wal_with_fallback enforces synchronous=FULL (issue #63531).""" + import sqlite3 + import hermes_state + from hermes_state import apply_wal_with_fallback + + class _TracingConn(sqlite3.Connection): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.executed = [] + + def execute(self, sql, params=()): + self.executed.append(sql) + return super().execute(sql, params) + + monkeypatch.setattr(hermes_state.sys, "platform", "darwin") + + db_path = tmp_path / "macos_fresh_sync.db" + conn = _TracingConn(str(db_path)) + try: + result = apply_wal_with_fallback(conn) + finally: + conn.close() + + assert result == "wal" + assert any("synchronous=FULL" in sql for sql in conn.executed), ( + "synchronous=FULL must be enforced on macOS" + ) + + def test_macos_synchronous_full_enforced_already_wal(self, tmp_path, monkeypatch): + """synchronous=FULL is enforced even when DB is already in WAL mode (issue #63531).""" + import sqlite3 + import hermes_state + from hermes_state import apply_wal_with_fallback + + class _TracingConn(sqlite3.Connection): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.executed = [] + + def execute(self, sql, params=()): + self.executed.append(sql) + return super().execute(sql, params) + + # Prime the file into WAL mode first (simulating an existing WAL DB). + db_path = tmp_path / "macos_wal_sync.db" + with sqlite3.connect(str(db_path)) as seed: + seed.execute("PRAGMA journal_mode=WAL") + + monkeypatch.setattr(hermes_state.sys, "platform", "darwin") + + conn = _TracingConn(str(db_path)) + try: + result = apply_wal_with_fallback(conn) + finally: + conn.close() + + assert result == "wal" + # The early-return path for existing WAL must also enforce synchronous=FULL. + assert any("synchronous=FULL" in sql for sql in conn.executed), ( + "synchronous=FULL must be enforced even on existing WAL DBs" + ) + assert not any("journal_mode=WAL" in sql for sql in conn.executed), ( + "set-pragma must not run when already in WAL mode" + ) def test_apply_wal_concurrent_connects_no_eio(self, tmp_path): """20 threads calling connect() on the same DB must not see disk I/O error.""" From f9c6f92c4b21bd8a03696e1266278c8685305595 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:31:51 +0530 Subject: [PATCH 043/149] docs(state): fix _enforce_macos_synchronous_full docstring synchronous=FULL issues plain fsync(), not F_FULLFSYNC. The F_FULLFSYNC barrier comes from checkpoint_fullfsync=1, set by the separate _apply_macos_checkpoint_barrier(). The original docstring conflated the two PRAGMAs. --- hermes_state.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index a8c529acecfd..79e95ffa66e4 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -366,8 +366,10 @@ def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: DB with half-written btree pages → ``btreeInitPage error 11``. WAL mode's durability guarantee assumes the OS honors fsync barriers; - macOS does not unless we explicitly set ``synchronous=FULL`` (which - issues ``fsync()`` *and* ``F_FULLFSYNC`` via checkpoint_fullfsync=1). + macOS does not unless we explicitly set ``synchronous=FULL``, which issues + a real ``fsync()`` on every transaction commit. The ``F_FULLFSYNC`` + barrier at checkpoint boundaries is handled separately by + :func:`_apply_macos_checkpoint_barrier`. This function is called after any successful WAL activation (either from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing From 370ebf2d3509b1fb7547e2ccdc55fe2a709e7400 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:59:59 +0530 Subject: [PATCH 044/149] fix(skills): guard skill slash commands against core-command and slug collisions scan_skill_commands() had two collision bugs in the same loop body: 1. Core-command collision: a skill whose normalized slug matches a core Hermes command name or alias (e.g. "skills", "learn", "bg") would get an auto-generated /command that shadows the core command in the gateway dispatch path (skill map is consulted before built-in handlers). The skill command silently overrode the core command. 2. Inter-skill slug collision: the seen_names set deduped on the raw frontmatter name, but the command map was keyed by the normalized slug. Two distinct names collapsing to the same slug (e.g. "git_helper" vs "git-helper") both passed the dedup, and the second silently clobbered the first. Fix: add two guards in scan_skill_commands() after slug normalization: - resolve_command(cmd_name) check skips skills colliding with any core CommandDef (name or alias), logging a warning. Uses the existing resolve_command() API so aliases and case variants are covered without a separate cache. The skill remains loadable via /skill. - cmd_key in _skill_commands check dedups on the resolved slug, first-wins (preserving local-before-external precedence), logging a warning naming the shadowed skill. Combines and supersedes #31204 (@cyrkstudios), #53450 (@Gridzilla), #50304 (@petrichor-op), and #63305 (@Vissirexa). Co-authored-by: cyrkstudios Co-authored-by: Gridzilla Co-authored-by: petrichor-op Co-authored-by: Vissirexa --- agent/skill_commands.py | 28 ++++++++++- tests/agent/test_skill_commands.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/agent/skill_commands.py b/agent/skill_commands.py index f4f470c4c014..11d425b91a52 100644 --- a/agent/skill_commands.py +++ b/agent/skill_commands.py @@ -329,6 +329,7 @@ def scan_skill_commands() -> Dict[str, Dict[str, Any]]: try: from tools.skills_tool import SKILLS_DIR, _parse_frontmatter, skill_matches_platform, skill_matches_environment, _get_disabled_skill_names from agent.skill_utils import get_external_skills_dirs, iter_skill_index_files + from hermes_cli.commands import resolve_command disabled = _get_disabled_skill_names() seen_names: set = set() @@ -374,7 +375,32 @@ def scan_skill_commands() -> Dict[str, Dict[str, Any]]: cmd_name = _SKILL_MULTI_HYPHEN.sub('-', cmd_name).strip('-') if not cmd_name: continue - _skill_commands[f"/{cmd_name}"] = { + # Skip if this skill's auto-generated /command collides + # with a core Hermes slash command (name or alias). The + # skill remains fully loadable via /skill . + # Uses resolve_command() so aliases and case variants are + # covered without maintaining a separate cache. + if resolve_command(cmd_name) is not None: + logger.warning( + "Skill %r generates slash command '/%s' which " + "collides with a core Hermes command; skipping " + "auto-registration. Use '/skill %s' instead.", + name, cmd_name, name, + ) + continue + # Dedup on the resolved slug, not just the raw name: two + # distinct frontmatter names can normalize to the same + # slug (e.g. "git_helper" vs "git-helper"). First-wins + # preserves local-before-external precedence. + cmd_key = f"/{cmd_name}" + if cmd_key in _skill_commands: + logger.warning( + "Skill %r maps to slash command %s already claimed " + "by %r; keeping the first and skipping this one.", + name, cmd_key, _skill_commands[cmd_key]["name"], + ) + continue + _skill_commands[cmd_key] = { "name": name, "description": description or f"Invoke the {name} skill", "skill_md_path": str(skill_md), diff --git a/tests/agent/test_skill_commands.py b/tests/agent/test_skill_commands.py index 08fe9f41b4fe..5d9d58c1e927 100644 --- a/tests/agent/test_skill_commands.py +++ b/tests/agent/test_skill_commands.py @@ -377,6 +377,82 @@ class TestScanSkillCommands: assert "/sonarr-v3v4-api" in result assert any("/" in k[1:] for k in result) is False # no unescaped / + # -- core-command collision guard (#31204 / #53450) --------------------- + + def test_skill_collides_with_core_command_is_skipped(self, tmp_path): + """A skill whose auto-generated /command collides with a core Hermes + command (e.g. 'skills') should be excluded from the slash-command map. + The skill remains loadable via /skill .""" + with patch("tools.skills_tool.SKILLS_DIR", tmp_path): + _make_skill(tmp_path, "skills") + result = scan_skill_commands() + assert "/skills" not in result + + def test_skill_collides_with_core_alias_is_skipped(self, tmp_path): + """A skill whose slug matches a core command *alias* is also skipped.""" + with patch("tools.skills_tool.SKILLS_DIR", tmp_path): + _make_skill(tmp_path, "bg") + result = scan_skill_commands() + # "bg" is an alias of the "background" command + assert "/bg" not in result + + def test_core_command_collision_does_not_block_others(self, tmp_path): + """A colliding skill is skipped, but non-colliding skills in the same + scan pass are still registered.""" + with patch("tools.skills_tool.SKILLS_DIR", tmp_path): + _make_skill(tmp_path, "skills") + _make_skill(tmp_path, "my-other-skill") + result = scan_skill_commands() + assert "/skills" not in result + assert "/my-other-skill" in result + + # -- inter-skill slug collision dedup (#50304 / #63305) ------------------ + + def test_slug_collision_keeps_first_skill(self, tmp_path): + """Two skills whose names normalize to the same slug do not clobber. + + ``git_helper`` and ``git-helper`` are distinct frontmatter names but + both reduce to the ``/git-helper`` command. The first one scanned must + keep the command rather than being silently overwritten by the second. + """ + with patch("tools.skills_tool.SKILLS_DIR", tmp_path): + # ``a-first`` sorts before ``z-second`` so the index walk visits the + # underscore-named skill first; that one must win the slash command. + first = tmp_path / "a-first" + first.mkdir() + (first / "SKILL.md").write_text( + "---\nname: git_helper\ndescription: First skill.\n---\n\nBody.\n" + ) + second = tmp_path / "z-second" + second.mkdir() + (second / "SKILL.md").write_text( + "---\nname: git-helper\ndescription: Second skill.\n---\n\nBody.\n" + ) + result = scan_skill_commands() + assert "/git-helper" in result + # First-wins: the entry resolves to the first skill, not the shadowing one. + assert result["/git-helper"]["name"] == "git_helper" + assert result["/git-helper"]["skill_dir"] == str(first) + + def test_slug_collision_warns(self, tmp_path, caplog): + """A slug collision emits a warning so the user can diagnose the + shadowed skill.""" + import logging as _logging + with patch("tools.skills_tool.SKILLS_DIR", tmp_path): + first = tmp_path / "a-first" + first.mkdir() + (first / "SKILL.md").write_text( + "---\nname: my-skill\ndescription: First.\n---\n\nBody.\n" + ) + second = tmp_path / "z-second" + second.mkdir() + (second / "SKILL.md").write_text( + "---\nname: my_skill\ndescription: Second.\n---\n\nBody.\n" + ) + with caplog.at_level(_logging.WARNING, logger="agent.skill_commands"): + scan_skill_commands() + assert any("already claimed" in r.message for r in caplog.records) + class TestResolveSkillCommandKey: """Telegram bot-command names disallow hyphens, so the menu registers From c3656e9f0cdbd690ed84971a1a31a3991c592534 Mon Sep 17 00:00:00 2001 From: slow4cyl <22582211+slow4cyl@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:29:35 -0500 Subject: [PATCH 045/149] fix(kanban_db): bounded retry for clean-exit protocol violations A worker that exits 0 without calling kanban_complete/kanban_block (model stops early, transient tool wedge) tripped the failure breaker on FIRST occurrence and the task was blocked. These are overwhelmingly transient: with a bounded retry (limit 3, tracked via a violation fingerprint) ~96%% of them complete on respawn. Genuine repeat offenders still trip the breaker at the limit. Co-Authored-By: Claude Fable 5 --- hermes_cli/kanban_db.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index ed617c8634ab..412cac5b6902 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -6567,6 +6567,13 @@ def _error_fingerprint(error_text: str) -> str: return fp.lower().strip() +# Empirically ~96% of "clean exit without a terminal tool call" tasks complete +# on a later run (a goal-mode finalize nudge, or the model simply emitting the +# tool call next time), so a protocol violation is NOT deterministic — give it a +# bounded retry before the breaker trips instead of blocking on the first hit. +_PROTOCOL_VIOLATION_FAILURE_LIMIT = 3 + + def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: """Reclaim ``running`` tasks whose worker PID is no longer alive. @@ -6724,11 +6731,12 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: # ready → blocked with a ``gave_up`` event on top of the ``crashed`` # event we already emitted. # - # Protocol-violation crashes force an immediate trip (failure_limit=1) - # because clean-exit-without-transition is deterministic: the next - # respawn will do exactly the same thing. Better to surface to a - # human with a clear reason than to loop ``DEFAULT_FAILURE_LIMIT`` - # times first. + # Protocol-violation crashes (clean exit, no terminal tool call) get a + # BOUNDED retry, not an immediate trip: empirically ~96% of these tasks + # complete on a later run (a goal-mode finalize nudge, or the model simply + # emitting kanban_complete/kanban_block next time), so blocking on the first + # occurrence just churned them through the respawn cycle. Systemic + # same-error crashes still trip immediately. auto_blocked: list[str] = [] if crash_details: # Fingerprint errors to detect systemic failures. @@ -6742,11 +6750,20 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: not protocol_violation and _fp_counts.get(fp, 0) >= 3 ) + # Systemic same-error crashes trip on the first hit; a protocol + # violation gets a bounded retry (it usually recovers); everything + # else uses the task's normal failure budget. + if is_systemic: + _flim = 1 + elif protocol_violation: + _flim = _PROTOCOL_VIOLATION_FAILURE_LIMIT + else: + _flim = None tripped = _record_task_failure( conn, tid, error=error_text, outcome="crashed", - failure_limit=1 if (protocol_violation or is_systemic) else None, + failure_limit=_flim, release_claim=False, end_run=False, event_payload_extra={"pid": pid, "claimer": claimer}, From 452861fdc1825702198f743c116513107f3b4831 Mon Sep 17 00:00:00 2001 From: slow4cyl <22582211+slow4cyl@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:34:13 -0500 Subject: [PATCH 046/149] review follow-up: violation-only retry streak with defined max_retries precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the hermes-sweeper review of #61233: the bounded retry budget is now a clean-exit-specific streak, not a share of the unified consecutive_failures counter. - detect_crashed_workers stamps a protocol_violation marker into the violation run's metadata (via the event payload _end_run copies); _protocol_violation_streak derives the streak from run history: consecutive most-recent violation runs, rate_limited runs neutral (mirroring their unified-counter treatment), any other closed run resets it. Mixed failure kinds can neither consume nor extend the budget. - Below-budget violations no longer call _record_task_failure at all: the task returns to ready with last_failure_error stamped directly (including the corrective retry guidance wording adopted from #61817, which build_worker_context surfaces to the retry worker) and the unified counter is untouched, keeping the two budgets independent. - At the bound the trip funnels through _record_task_failure with a new keyword-only force_trip=True: the reaper has already resolved the per-task max_retries override against the violation streak itself, so the threshold comparison is skipped rather than double-applied. max_retries keeps its documented top precedence in both directions: max_retries=1 blocks on the first violation, max_retries=5 blocks on the fifth consecutive one, unset uses the default bound of 3. - Replace the first-violation-blocks regression test with five tests: first occurrence retries (ready + guidance stamped + no gave_up + unified counter untouched); streak trips exactly at the bound with protocol_violations/protocol_violation_limit in the gave_up payload and the auto-blocked side channel set; a prior nonzero crash does not consume the violation budget; a non-violation failure between violations resets the streak; max_retries precedence both directions. All five fail against the previously reviewed diff and pass with this follow-up. The test harness resolves hermes_cli.kanban_db fresh and uses that single module object for the exit registry, liveness patch, and reaper — earlier suite tests reload the module, and the old mixed-object harness made _classify_worker_exit return unknown (the reason the old test failed in full-suite runs on main). Kanban suite: zero introduced failures vs upstream/main tip (62 vs 63 pre-existing environmental failures — the one no longer failing is the old violation test this replaces; 662 passed vs 657). Co-Authored-By: Claude Fable 5 --- hermes_cli/kanban_db.py | 189 +++++++++++-- .../test_kanban_core_functionality.py | 252 +++++++++++++++--- 2 files changed, 384 insertions(+), 57 deletions(-) diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 412cac5b6902..39c434d4021d 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -6571,8 +6571,72 @@ def _error_fingerprint(error_text: str) -> str: # on a later run (a goal-mode finalize nudge, or the model simply emitting the # tool call next time), so a protocol violation is NOT deterministic — give it a # bounded retry before the breaker trips instead of blocking on the first hit. +# +# The budget is a violation-only STREAK, not a share of the unified +# ``consecutive_failures`` counter: it counts consecutive clean-exit protocol +# violations (derived from run history by ``_protocol_violation_streak``), so +# earlier timeouts / nonzero exits neither consume nor extend it, and a +# below-budget violation does not tick the unified counter either. A per-task +# ``max_retries`` overrides this bound — the same "task override wins" +# precedence ``_record_task_failure`` documents for every other failure kind. _PROTOCOL_VIOLATION_FAILURE_LIMIT = 3 +# How far back to walk a task's closed runs when counting the violation +# streak. The streak trips at a handful of violations, so anything beyond a +# few dozen rows (violations interleaved with neutral rate-limited requeues) +# can only mean "way past the bound" anyway. +_PROTOCOL_VIOLATION_SCAN_LIMIT = 50 + + +def _protocol_violation_streak(conn: sqlite3.Connection, task_id: str) -> int: + """Count the task's trailing run of clean-exit protocol violations. + + Walks the task's closed runs newest-first — including the violation run + ``detect_crashed_workers`` just closed — and counts how many in a row were + clean-exit protocol violations: + + * ``rate_limited`` runs are neutral and skipped: a quota wall says nothing + about the task, exactly as it is neutral for the unified + ``consecutive_failures`` counter. + * Any other closed run (completed, plain crash, timeout, spawn failure, + reclaim, …) breaks the streak, so the bounded retry budget counts ONLY + protocol violations — mixed failure kinds can neither consume nor + extend it. + + Violation runs are recognized by the ``protocol_violation`` marker that + ``detect_crashed_workers`` stamps into the run metadata; the violation + error text is matched as a fallback for runs recorded before the marker + existed. + """ + streak = 0 + rows = conn.execute( + "SELECT outcome, error, metadata FROM task_runs " + "WHERE task_id = ? AND ended_at IS NOT NULL " + "ORDER BY id DESC LIMIT ?", + (task_id, _PROTOCOL_VIOLATION_SCAN_LIMIT), + ).fetchall() + for row in rows: + outcome = row["outcome"] or "" + if outcome == "rate_limited": + continue + if outcome == "crashed": + is_violation = False + raw_meta = row["metadata"] + if raw_meta: + try: + is_violation = bool( + json.loads(raw_meta).get("protocol_violation") + ) + except (ValueError, TypeError): + is_violation = False + if not is_violation: + is_violation = "protocol violation" in (row["error"] or "") + if is_violation: + streak += 1 + continue + break + return streak + def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: """Reclaim ``running`` tasks whose worker PID is no longer alive. @@ -6607,8 +6671,9 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: # Per-crash details collected inside the main txn, used after it # closes to run ``_record_task_failure`` (which needs its own # write_txn so can't nest). ``protocol_violation`` flags the - # clean-exit-but-still-running case so we can trip the breaker - # immediately instead of incrementing by 1. + # clean-exit-but-still-running case, which is accounted against its + # own bounded violation streak instead of the unified failure + # counter (see the post-txn loop below). crash_details: list[tuple[str, int, str, bool, str]] = [] # (task_id, pid, claimer, protocol_violation, error_text) with write_txn(conn): @@ -6639,18 +6704,29 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: if kind == "clean_exit": # Worker subprocess returned 0 but its task is still # ``running`` in the DB — it exited without calling - # ``kanban_complete`` / ``kanban_block``. Retrying won't - # help. + # ``kanban_complete`` / ``kanban_block``. Overwhelmingly the + # work itself succeeded and only the paperwork was skipped, so + # a retry usually completes; the corrective sentence below is + # surfaced to the retry worker via the prior-attempt error in + # ``build_worker_context`` (guidance approach from #61817). protocol_violation = True error_text = ( "worker exited cleanly (rc=0) without calling " - "kanban_complete or kanban_block — protocol violation" + "kanban_complete or kanban_block — protocol violation. " + "If the prior run already did the work, verify it and " + "report the result via kanban_complete; a run that ends " + "without a terminal kanban call counts as failed no " + "matter what it did." ) event_kind = "protocol_violation" event_payload = { "pid": pid, "claimer": row["claim_lock"], "exit_code": code, + # Durable marker for _protocol_violation_streak: _end_run + # copies this payload into the run metadata, which is how + # the violation-only retry budget is derived later. + "protocol_violation": True, } elif kind == "rate_limited": # Worker bailed because the provider rate-limited / exhausted @@ -6721,22 +6797,39 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: ) rate_limited.append(row["id"]) else: + if protocol_violation: + # Stamp the failure error now: a below-budget + # violation never reaches ``_record_task_failure`` + # (which stamps this column for every other failure + # kind), yet the board UI and the retry worker's + # context still need the violation message + the + # corrective guidance it carries. + conn.execute( + "UPDATE tasks SET last_failure_error = ? " + "WHERE id = ?", + (error_text[:500], row["id"]), + ) crashed.append(row["id"]) crash_details.append( (row["id"], pid, row["claim_lock"], protocol_violation, error_text) ) - # Outside the main txn: increment the unified failure counter for - # each crashed task. If the breaker trips, the task transitions - # ready → blocked with a ``gave_up`` event on top of the ``crashed`` - # event we already emitted. + # Outside the main txn: account each crashed task and maybe trip the + # breaker (the task transitions ready → blocked with a ``gave_up`` event + # on top of the event we already emitted). # # Protocol-violation crashes (clean exit, no terminal tool call) get a # BOUNDED retry, not an immediate trip: empirically ~96% of these tasks # complete on a later run (a goal-mode finalize nudge, or the model simply # emitting kanban_complete/kanban_block next time), so blocking on the first - # occurrence just churned them through the respawn cycle. Systemic - # same-error crashes still trip immediately. + # occurrence just churned them through the respawn cycle. The retry budget + # is a violation-only streak (``_protocol_violation_streak``): earlier + # timeouts / nonzero exits neither consume nor extend it, and a + # below-budget violation does not tick the unified + # ``consecutive_failures`` counter, so the two budgets stay independent. + # A per-task ``max_retries`` overrides the violation bound with the same + # top precedence it has for every other failure kind. Systemic same-error + # crashes still trip immediately. auto_blocked: list[str] = [] if crash_details: # Fingerprint errors to detect systemic failures. @@ -6745,25 +6838,59 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: fp = _error_fingerprint(err_text) _fp_counts[fp] = _fp_counts.get(fp, 0) + 1 for tid, pid, claimer, protocol_violation, error_text in crash_details: + if protocol_violation: + streak = _protocol_violation_streak(conn, tid) + trow = conn.execute( + "SELECT max_retries FROM tasks WHERE id = ?", (tid,), + ).fetchone() + if trow is None: + continue # task deleted mid-loop + task_override = ( + trow["max_retries"] if "max_retries" in trow.keys() else None + ) + violation_limit = ( + int(task_override) + if task_override is not None + else _PROTOCOL_VIOLATION_FAILURE_LIMIT + ) + if streak < violation_limit: + # Below budget: the task is already back at ``ready`` + # (respawn allowed) with ``last_failure_error`` stamped. + # Deliberately no ``_record_task_failure`` call — a + # below-budget violation must not consume the unified + # failure budget, just as other failure kinds don't + # consume this one. + continue + # Streak reached the bound: trip the breaker. ``force_trip`` + # skips the threshold resolution inside + # ``_record_task_failure`` because the decision — including + # the per-task ``max_retries`` override — was already made + # against the violation streak above. + tripped = _record_task_failure( + conn, tid, + error=error_text, + outcome="crashed", + failure_limit=violation_limit, + force_trip=True, + release_claim=False, + end_run=False, + event_payload_extra={ + "pid": pid, + "claimer": claimer, + "protocol_violations": streak, + "protocol_violation_limit": violation_limit, + }, + ) + if tripped: + auto_blocked.append(tid) + continue fp = _error_fingerprint(error_text) - is_systemic = ( - not protocol_violation - and _fp_counts.get(fp, 0) >= 3 - ) - # Systemic same-error crashes trip on the first hit; a protocol - # violation gets a bounded retry (it usually recovers); everything - # else uses the task's normal failure budget. - if is_systemic: - _flim = 1 - elif protocol_violation: - _flim = _PROTOCOL_VIOLATION_FAILURE_LIMIT - else: - _flim = None + is_systemic = _fp_counts.get(fp, 0) >= 3 tripped = _record_task_failure( conn, tid, error=error_text, outcome="crashed", - failure_limit=_flim, + failure_limit=1 if is_systemic else None, release_claim=False, end_run=False, event_payload_extra={"pid": pid, "claimer": claimer}, @@ -6788,6 +6915,7 @@ def _record_task_failure( *, outcome: str, failure_limit: int = None, + force_trip: bool = False, release_claim: bool = False, end_run: bool = False, event_payload_extra: Optional[dict] = None, @@ -6825,6 +6953,15 @@ def _record_task_failure( 2. caller-supplied ``failure_limit`` (gateway passes the config value from ``kanban.failure_limit``; tests pass fixed values) 3. ``DEFAULT_FAILURE_LIMIT`` + + ``force_trip=True`` trips the breaker unconditionally, skipping the + counter-vs-threshold comparison (the resolution order above is then + only reported in the ``gave_up`` payload, not re-evaluated). Callers + use it when they have already applied their own bounded-retry policy + — e.g. the clean-exit protocol-violation streak in + ``detect_crashed_workers``, which resolves the per-task + ``max_retries`` override against the violation streak itself. The + failure is still counted into ``consecutive_failures``. """ if failure_limit is None: failure_limit = DEFAULT_FAILURE_LIMIT @@ -6851,7 +6988,7 @@ def _record_task_failure( effective_limit = int(failure_limit) limit_source = "dispatcher" - if failures >= effective_limit: + if force_trip or failures >= effective_limit: # Trip the breaker. if release_claim: # Spawn path: still running, also clear claim state. diff --git a/tests/hermes_cli/test_kanban_core_functionality.py b/tests/hermes_cli/test_kanban_core_functionality.py index 353722d198bd..4884862e1f79 100644 --- a/tests/hermes_cli/test_kanban_core_functionality.py +++ b/tests/hermes_cli/test_kanban_core_functionality.py @@ -4398,43 +4398,71 @@ def test_detect_crashed_workers_increments_counter(kanban_home): conn.close() -def test_detect_crashed_workers_protocol_violation_auto_blocks(kanban_home): - """A worker that exited rc=0 while its task was still ``running`` - is a protocol violation (agent answered conversationally without - calling kanban_complete / kanban_block). Retrying will just loop, - so auto-block immediately instead of waiting for the breaker to - trip at ``DEFAULT_FAILURE_LIMIT``. +def _drive_worker_exit(conn, tid, fake_pid, raw_status): + """Claim ``tid``, record ``raw_status`` for its dead worker pid, and run + one reaper pass. - Regression test for the respawn-loop-after-completion bug reported - against small local models (gemma4-e2b q4) where the model writes - the answer as plain text and the CLI exits rc=0 cleanly. + Deliberately resolves ``hermes_cli.kanban_db`` fresh and uses that single + module object for the exit registry, the liveness patch, AND the reaper: + earlier tests in a full-suite run can reload the module, and recording + the exit into one module object while reaping through another (stale) + one makes ``_classify_worker_exit`` return ``unknown`` — silently turning + a clean-exit protocol violation into a plain crash. """ import hermes_cli.kanban_db as _kb + host_prefix = _kb._claimer_id().split(":", 1)[0] + claimed = _kb.claim_task(conn, tid, claimer=f"{host_prefix}:mock") + assert claimed is not None, "task was not claimable for the next attempt" + _kb._set_worker_pid(conn, tid, fake_pid) + _kb._record_worker_exit(fake_pid, raw_status) + original_alive = _kb._pid_alive + _kb._pid_alive = lambda p: False + try: + return _kb.detect_crashed_workers(conn) + finally: + _kb._pid_alive = original_alive + + +def _drive_protocol_violation(conn, tid, fake_pid): + """One clean-exit protocol violation reaper pass for ``tid``. + + os.W_EXITCODE(status=0, signal=0) == 0 on POSIX. + """ + return _drive_worker_exit(conn, tid, fake_pid, 0) + + +def _drive_nonzero_crash(conn, tid, fake_pid): + """One plain non-zero-exit crash reaper pass for ``tid``. + + W_EXITCODE(1, 0) == 256 — WIFEXITED True, WEXITSTATUS == 1. + """ + return _drive_worker_exit(conn, tid, fake_pid, 256) + + +def test_detect_crashed_workers_protocol_violation_first_occurrence_retries(kanban_home): + """A first clean-exit protocol violation gets a retry, not a block. + + A worker that exited rc=0 while its task was still ``running`` skipped + the terminal kanban call (model answered conversationally, transient tool + wedge). Empirically these overwhelmingly complete on respawn, so the + first violation must leave the task ``ready`` with corrective guidance + stamped in ``last_failure_error`` — not trip the breaker like the pre-fix + behavior did. The violation is accounted against its own violation-only + streak, so it must NOT tick the unified ``consecutive_failures`` counter. + """ conn = kb.connect() try: tid = kb.create_task(conn, title="quiet", assignee="worker") - host_prefix = _kb._claimer_id().split(":", 1)[0] - lock = f"{host_prefix}:mock" - kb.claim_task(conn, tid, claimer=lock) - fake_pid = 999998 - kb._set_worker_pid(conn, tid, fake_pid) - - # Simulate the reap loop having recorded a clean exit for this pid. - # os.W_EXITCODE(status=0, signal=0) == 0 on POSIX. - _kb._record_worker_exit(fake_pid, 0) - # Force liveness check to say "dead" for the fake pid. - original_alive = _kb._pid_alive - _kb._pid_alive = lambda p: False - try: - result_crashed = kb.detect_crashed_workers(conn) - finally: - _kb._pid_alive = original_alive - + result_crashed = _drive_protocol_violation(conn, tid, 999998) assert tid in result_crashed, "should be detected as crashed" + task = kb.get_task(conn, tid) - assert task.status == "blocked", ( - f"protocol violation should auto-block on first occurrence, " - f"got status={task.status}" + assert task.status == "ready", ( + f"first protocol violation should retry, got status={task.status}" + ) + assert task.consecutive_failures == 0, ( + "a below-budget violation must not consume the unified failure " + f"budget, got consecutive_failures={task.consecutive_failures}" ) assert "kanban_complete" in (task.last_failure_error or ""), ( f"expected protocol-violation message, got {task.last_failure_error!r}" @@ -4450,13 +4478,175 @@ def test_detect_crashed_workers_protocol_violation_auto_blocks(kanban_home): assert "crashed" not in kinds, ( f"should NOT emit 'crashed' event on clean exit, got {kinds}" ) - assert "gave_up" in kinds, ( - f"breaker should trip, expected 'gave_up' event, got {kinds}" + assert "gave_up" not in kinds, ( + f"breaker must not trip on the first violation, got {kinds}" ) finally: conn.close() +def test_detect_crashed_workers_protocol_violation_streak_trips_at_limit(kanban_home): + """The violation streak trips the terminal path exactly at the bound. + + Genuine repeat offenders (a worker whose CLI keeps returning 0 without a + terminal transition) must still surface to a human: the + ``_PROTOCOL_VIOLATION_FAILURE_LIMIT``-th consecutive violation blocks the + task with a ``gave_up`` event carrying the streak accounting. + """ + import hermes_cli.kanban_db as _kb + conn = kb.connect() + try: + tid = kb.create_task(conn, title="quiet", assignee="worker") + limit = _kb._PROTOCOL_VIOLATION_FAILURE_LIMIT + for i in range(limit - 1): + _drive_protocol_violation(conn, tid, 990000 + i) + assert kb.get_task(conn, tid).status == "ready", ( + f"violation {i + 1}/{limit} should still retry" + ) + + _drive_protocol_violation(conn, tid, 990900) + + task = kb.get_task(conn, tid) + assert task.status == "blocked", ( + f"violation streak at the bound must block, got {task.status}" + ) + events = kb.list_events(conn, tid) + kinds = [e.kind for e in events] + assert kinds.count("protocol_violation") == limit + assert "crashed" not in kinds + gave_up = [e for e in events if e.kind == "gave_up"] + assert len(gave_up) == 1, f"expected exactly one gave_up, got {kinds}" + payload = gave_up[0].payload or {} + assert payload.get("protocol_violations") == limit + assert payload.get("protocol_violation_limit") == limit + # Side channel consumed by dispatch_once — read through the same + # (current) module object the reaper ran in, see _drive_worker_exit. + assert tid in _kb.detect_crashed_workers._last_auto_blocked + finally: + conn.close() + + +def test_protocol_violation_budget_not_consumed_by_other_failures(kanban_home): + """Mixed failure kinds must not consume the violation retry budget. + + Regression for the #61233 review finding: expressed as a plain + ``failure_limit`` over the unified ``consecutive_failures`` counter, the + violation budget was consumed by earlier timeouts / nonzero exits. As a + violation-only streak, a prior real crash must not eat violation + retries, and below-budget violations must leave the unified counter + untouched (so the two budgets stay independent). + """ + import hermes_cli.kanban_db as _kb + conn = kb.connect() + try: + tid = kb.create_task(conn, title="mixed", assignee="worker") + + # One real crash: unified counter ticks to 1 (below + # DEFAULT_FAILURE_LIMIT=2 — task stays ready). + _drive_nonzero_crash(conn, tid, 991000) + task = kb.get_task(conn, tid) + assert task.status == "ready" + assert task.consecutive_failures == 1 + + # Two violations after it: streak 1 and 2 — both retry, unified + # counter untouched. (Pre-fix: the crash consumed the budget and the + # violations blocked well before three of them happened.) + for i, pid in enumerate((991001, 991002)): + _drive_protocol_violation(conn, tid, pid) + task = kb.get_task(conn, tid) + assert task.status == "ready", ( + f"violation {i + 1} after a crash must still retry, " + f"got {task.status}" + ) + assert task.consecutive_failures == 1, ( + "below-budget violations must not tick the unified counter" + ) + + # Third consecutive violation: streak hits the bound — blocked. + _drive_protocol_violation(conn, tid, 991003) + task = kb.get_task(conn, tid) + assert task.status == "blocked" + gave_up = [e for e in kb.list_events(conn, tid) if e.kind == "gave_up"] + assert len(gave_up) == 1 + assert (gave_up[0].payload or {}).get("protocol_violations") == \ + _kb._PROTOCOL_VIOLATION_FAILURE_LIMIT + finally: + conn.close() + + +def test_protocol_violation_streak_resets_on_other_failure_kind(kanban_home): + """A non-violation failure between violations resets the streak. + + The budget counts CONSECUTIVE clean-exit violations: two violations, a + real crash, then two more violations is a streak of 2 — not 4 — so the + fourth violation must still retry; only a third consecutive one blocks. + """ + conn = kb.connect() + try: + tid = kb.create_task(conn, title="reset", assignee="worker") + + _drive_protocol_violation(conn, tid, 993000) + _drive_protocol_violation(conn, tid, 993001) + assert kb.get_task(conn, tid).status == "ready" + + # Real crash breaks the streak (and ticks the unified counter to 1). + _drive_nonzero_crash(conn, tid, 993002) + assert kb.get_task(conn, tid).status == "ready" + + # Streak restarts at 1, 2 — the pre-crash violations no longer count. + _drive_protocol_violation(conn, tid, 993003) + assert kb.get_task(conn, tid).status == "ready", ( + "violation streak must reset after a non-violation failure" + ) + _drive_protocol_violation(conn, tid, 993004) + assert kb.get_task(conn, tid).status == "ready" + + # Third consecutive violation since the crash: blocked. + _drive_protocol_violation(conn, tid, 993005) + assert kb.get_task(conn, tid).status == "blocked" + finally: + conn.close() + + +def test_protocol_violation_respects_max_retries_precedence(kanban_home): + """Per-task ``max_retries`` overrides the violation bound, both ways. + + Same top precedence it has for every other failure kind in + ``_record_task_failure``: ``max_retries=1`` blocks on the FIRST violation + (zero retries — the pre-fix behavior, now opt-in per task); + ``max_retries=5`` keeps retrying past the default bound of 3 and blocks + on the 5th consecutive violation. + """ + conn = kb.connect() + try: + strict = kb.create_task( + conn, title="strict", assignee="worker", max_retries=1, + ) + _drive_protocol_violation(conn, strict, 992000) + task = kb.get_task(conn, strict) + assert task.status == "blocked", ( + f"max_retries=1 must block on the first violation, got {task.status}" + ) + gave_up = [e for e in kb.list_events(conn, strict) if e.kind == "gave_up"] + assert len(gave_up) == 1 + payload = gave_up[0].payload or {} + assert payload.get("protocol_violations") == 1 + assert payload.get("protocol_violation_limit") == 1 + + lenient = kb.create_task( + conn, title="lenient", assignee="worker", max_retries=5, + ) + for i in range(4): + _drive_protocol_violation(conn, lenient, 992100 + i) + assert kb.get_task(conn, lenient).status == "ready", ( + f"violation {i + 1}/5 should retry under max_retries=5" + ) + _drive_protocol_violation(conn, lenient, 992104) + assert kb.get_task(conn, lenient).status == "blocked" + finally: + conn.close() + + def test_detect_crashed_workers_nonzero_exit_uses_default_limit(kanban_home): """A worker that exited non-zero (real error / crash) uses the normal counter path — one failure doesn't trip the breaker. From 3cd8feb63c5c17175bfc635b542c6123c6d420b8 Mon Sep 17 00:00:00 2001 From: slow4cyl <22582211+slow4cyl@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:53:34 -0500 Subject: [PATCH 047/149] docs(kanban): worker-lifecycle + events table reflect the bounded protocol-violation retry Fold in kevinb361's suggested lifecycle wording (#61817 conceded in favor of this PR) and update the second stale site his sweep didn't cover: the task-events table still said the dispatcher 'auto-blocks immediately instead of retrying'. Both now describe the violation-only streak: protocol_violation fires on every violation (its payload marker feeds the budget), below-budget runs return the task to ready, and gave_up + auto-block happen only when the consecutive streak reaches _PROTOCOL_VIOLATION_FAILURE_LIMIT (default 3, per-task max_retries overriding). --- website/docs/user-guide/features/kanban.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md index b505ccc9bf4a..57d66f585180 100644 --- a/website/docs/user-guide/features/kanban.md +++ b/website/docs/user-guide/features/kanban.md @@ -369,10 +369,17 @@ Every profile that works kanban tasks automatically gets the worker lifecycle That final `kanban_complete` / `kanban_block` call is part of the worker protocol. If the worker process exits with status 0 while the task is still -`running`, the dispatcher treats that as a protocol violation, emits a -`protocol_violation` event, and auto-blocks the task on the next tick instead -of respawning it into the same loop. This usually means the model wrote a -plain-text answer and exited without using the Kanban tool surface. +`running`, the dispatcher treats that as a protocol violation and emits a +`protocol_violation` event. Because a protocol violation is **not +deterministic** — the model may emit the missing tool call on a later run — +the dispatcher gives it a **bounded retry** (up to +`_PROTOCOL_VIOLATION_FAILURE_LIMIT` consecutive violations, default 3) before +auto-blocking the task instead of respawning it into the same loop. The budget +counts only *consecutive* clean-exit protocol violations — interleaved +rate-limited requeues are neutral, and any other failure kind resets the +streak — and a per-task `max_retries` overrides the bound. This usually means +the model wrote a plain-text answer and exited without using the Kanban tool +surface. The lifecycle plus the load-bearing reference details (workspace kinds, deliverable `artifacts`, claiming created cards) ship in that system-prompt block, so every worker has them regardless of which profile it runs under — no per-profile skill setup required. @@ -926,7 +933,7 @@ Every transition appends a row to `task_events`. Each row carries an optional `r | `stale` | `{elapsed_seconds, last_heartbeat_at, heartbeat_age_seconds, timeout_seconds, pid, terminated}` | Task ran longer than `kanban.dispatch_stale_timeout_seconds` (default 4 h) AND no `kanban_heartbeat` arrived in the last hour. Dispatcher SIGTERM'd the host-local worker (if any), reset the task to `ready` for re-dispatch. Does NOT tick the failure counter (stale is dispatcher-side absence detection, not a worker fault). Workers running long operations should call `kanban_heartbeat` at least once an hour to avoid this. | | `respawn_guarded` | `{reason}` | Dispatcher refused to re-spawn this ready task this tick. Reasons: `blocker_auth` (last failure was a quota/auth/429 error — wait for the rate window to reset), `recent_success` (a completed run happened in the last hour — wait for review before re-running), `active_pr` (a GitHub PR URL appears in a recent comment — a prior worker already opened a PR). The task stays in `ready`; the next tick gets another chance to spawn. If the underlying condition persists, the normal `consecutive_failures` circuit breaker will auto-block via `gave_up` after `failure_limit` failures. | | `spawn_failed` | `{error, failures}` | One spawn attempt failed (missing PATH, workspace unmountable, …). Counter increments; task returns to `ready` for retry. | -| `protocol_violation` | `{pid, claimer, exit_code}` | Worker exited successfully while the task was still `running`, usually because it answered without calling `kanban_complete` or `kanban_block`. The dispatcher also emits `gave_up` and auto-blocks immediately instead of retrying. | +| `protocol_violation` | `{pid, claimer, exit_code, protocol_violation}` | Worker exited successfully while the task was still `running`, usually because it answered without calling `kanban_complete` or `kanban_block`. Emitted on every violation (the payload's `protocol_violation: true` marker is copied into the run metadata and feeds the violation-only retry budget). Below the budget — up to `_PROTOCOL_VIOLATION_FAILURE_LIMIT` (default 3) *consecutive* violations, per-task `max_retries` overriding — the task simply returns to `ready` for another attempt; when the streak reaches the bound the dispatcher also emits `gave_up` and auto-blocks. | | `gave_up` | `{failures, effective_limit, limit_source, error}` | Circuit breaker fired after N consecutive non-successful attempts. Task auto-blocks with the last error. The effective limit resolves as task `max_retries`, then dispatcher `failure_limit` / `kanban.failure_limit`, then the built-in default. | `hermes kanban tail ` shows these for a single task. `hermes kanban watch` streams them board-wide. From 52cafa6f8e3e8cec6d3528b739380d4442a71cf4 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:36:12 +0530 Subject: [PATCH 048/149] follow-up: integrate agent nudge + dispatcher retry docs and tests - Nudge text now warns that repeated protocol violations will block the task and require manual intervention, so the model understands the consequence of ignoring the nudge. - Kanban docs restructured to clearly separate the two defense layers: agent-side prevention (nudge, from #64350) and dispatcher-side recovery (bounded retry, from this PR). - Two new integration tests verifying the nudge mentions blocking and that the agent-side and dispatcher-side budgets are independent. --- agent/kanban_stop.py | 3 +- tests/agent/test_kanban_stop.py | 38 ++++++++++++++++++++++ website/docs/user-guide/features/kanban.md | 21 ++++++++---- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/agent/kanban_stop.py b/agent/kanban_stop.py index bb97ddd425f0..e7c2eae828a6 100644 --- a/agent/kanban_stop.py +++ b/agent/kanban_stop.py @@ -96,7 +96,8 @@ def build_kanban_stop_nudge( "1. Finish any remaining deliverable (write the required file(s) now).\n" "2. Call `kanban_complete(summary=..., artifacts=[...])` if the work " "is done, OR `kanban_block(reason=...)` if you are blocked.\n\n" - "Never end a turn with only a promise of future action.]" + "Never end a turn with only a promise of future action. Repeated " + "protocol violations will block this task and require manual intervention.]" ) diff --git a/tests/agent/test_kanban_stop.py b/tests/agent/test_kanban_stop.py index 62ba78a42a79..d377f6503ea4 100644 --- a/tests/agent/test_kanban_stop.py +++ b/tests/agent/test_kanban_stop.py @@ -93,3 +93,41 @@ def test_nudge_budget_exhausted(clear_kanban_env): assert build_kanban_stop_nudge(messages=[], attempts=2) is None assert build_kanban_stop_nudge(messages=[], attempts=1, max_attempts=1) is None assert build_kanban_stop_nudge(messages=[], attempts=0, max_attempts=1) is not None + + +# ── Integration: agent nudge + dispatcher bounded retry ────────────── +# These tests verify the two layers compose correctly: the agent-side +# nudge fires first (up to 2 attempts), and if the worker still exits +# without a terminal call, the dispatcher's bounded retry (streak of 3) +# handles it. See also tests/hermes_cli/test_kanban_core_functionality.py +# for the dispatcher-side streak tests. + + +def test_nudge_text_warns_about_blocking(clear_kanban_env): + """The nudge should warn that repeated violations will block the task.""" + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + nudge = build_kanban_stop_nudge(messages=[], attempts=0) + assert nudge is not None + assert "block" in nudge.lower(), ( + "nudge should warn that repeated violations will block the task" + ) + + +def test_nudge_and_dispatcher_budgets_are_independent(clear_kanban_env): + """Agent-side nudge budget (2) and dispatcher-side streak (3) are + separate budgets — the nudge counter does not affect the dispatcher's + violation streak, and vice versa. + + This is a source-level invariant check: the nudge counter + (``_kanban_stop_nudges``) lives on the AIAgent instance and resets + per session, while the dispatcher streak lives in the task_runs DB + table and persists across worker respawns. + """ + clear_kanban_env.setenv("HERMES_KANBAN_TASK", "t_abc") + # Agent-side: 2 nudge attempts per session + assert build_kanban_stop_nudge(messages=[], attempts=0) is not None + assert build_kanban_stop_nudge(messages=[], attempts=1) is not None + assert build_kanban_stop_nudge(messages=[], attempts=2) is None + # Dispatcher-side streak is tracked in the DB, not in the nudge module — + # the nudge module has no knowledge of the streak counter. + assert not hasattr(build_kanban_stop_nudge, "_streak") diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md index 57d66f585180..c8841e9172dd 100644 --- a/website/docs/user-guide/features/kanban.md +++ b/website/docs/user-guide/features/kanban.md @@ -370,12 +370,21 @@ Every profile that works kanban tasks automatically gets the worker lifecycle That final `kanban_complete` / `kanban_block` call is part of the worker protocol. If the worker process exits with status 0 while the task is still `running`, the dispatcher treats that as a protocol violation and emits a -`protocol_violation` event. Because a protocol violation is **not -deterministic** — the model may emit the missing tool call on a later run — -the dispatcher gives it a **bounded retry** (up to -`_PROTOCOL_VIOLATION_FAILURE_LIMIT` consecutive violations, default 3) before -auto-blocking the task instead of respawning it into the same loop. The budget -counts only *consecutive* clean-exit protocol violations — interleaved +`protocol_violation` event. + +**Agent-side prevention:** Before the worker exits, Hermes injects up to two +synthetic nudges when it detects the model is about to stop without a terminal +board tool call. This catches the common case where the model narrates the next +step ("Let me write the report") and stops with `finish_reason=stop`. The nudge +reminds the model to call `kanban_complete` or `kanban_block` immediately. This +guard is active only for dispatcher-spawned workers (`HERMES_KANBAN_TASK` is +set) and can be disabled with `HERMES_KANBAN_STOP_NUDGE=0`. + +**Dispatcher-side recovery:** If the nudges are exhausted or the worker crashes +before reaching the nudge, the dispatcher gives the violation a **bounded retry** +(up to `_PROTOCOL_VIOLATION_FAILURE_LIMIT` consecutive violations, default 3) +before auto-blocking the task instead of respawning it into the same loop. The +budget counts only *consecutive* clean-exit protocol violations — interleaved rate-limited requeues are neutral, and any other failure kind resets the streak — and a per-task `max_retries` overrides the bound. This usually means the model wrote a plain-text answer and exited without using the Kanban tool From 78e844d4465c4d9f6a2cb2f44e5076d31e1c68db Mon Sep 17 00:00:00 2001 From: webtecnica <75556242+webtecnica@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:29:08 -0300 Subject: [PATCH 049/149] fix(agent): validate credential pool after provider auto-detection (#63425) Provider auto-detection (URL-based inference for Anthropic, OpenAI Codex, and xAI endpoints) runs before credential-pool validation in AIAgent init, but #63048 placed the pool validation before auto-detection. When the agent is constructed with provider=None and a recognized endpoint URL, the pool is validated against an empty provider identity and discarded, even though auto-detection correctly resolves the provider moments later. Fix: move the credential-pool validation block to after the URL-based auto-detection chain. The pool is stored on the agent before auto-detection; validation now checks the resolved provider and only nullifies agent._credential_pool when the pool's scoped provider genuinely doesn't match. Regression test covers all three auto-detection paths: - Anthropic (api.anthropic.com) - OpenAI Codex (chatgpt.com/backend-api/codex) - xAI (api.x.ai) Fixes #63425. --- agent/agent_init.py | 30 +-- .../test_63425_credential_pool_auto_detect.py | 178 ++++++++++++++++++ 2 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 tests/run_agent/test_63425_credential_pool_auto_detect.py diff --git a/agent/agent_init.py b/agent/agent_init.py index f9acb3c982ab..0c700c279b98 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -434,18 +434,6 @@ def init_agent( agent.base_url = base_url or "" provider_name = provider.strip().lower() if isinstance(provider, str) and provider.strip() else None agent.provider = provider_name or "" - if credential_pool is not None: - try: - from agent.credential_pool import credential_pool_matches_provider - - if not credential_pool_matches_provider( - credential_pool, - agent.provider, - base_url=agent.base_url, - ): - credential_pool = None - except Exception: - credential_pool = None agent._credential_pool = credential_pool agent.acp_command = acp_command or command agent.acp_args = list(acp_args or args or []) @@ -482,6 +470,24 @@ def init_agent( else: agent.api_mode = "chat_completions" + # Credential-pool validation runs AFTER provider auto-detection so + # a pool scoped to e.g. "anthropic" is not rejected when the agent + # was constructed with provider=None and an anthropic.com URL. + # Regression from #63048 which placed this check before the + # URL-based auto-detection block above (fixed #63425). + if credential_pool is not None: + try: + from agent.credential_pool import credential_pool_matches_provider + + if not credential_pool_matches_provider( + credential_pool, + agent.provider, + base_url=agent.base_url, + ): + agent._credential_pool = None + except Exception: + agent._credential_pool = None + # Eagerly warm the transport cache so import errors surface at init, # not mid-conversation. Also validates the api_mode is registered. try: diff --git a/tests/run_agent/test_63425_credential_pool_auto_detect.py b/tests/run_agent/test_63425_credential_pool_auto_detect.py new file mode 100644 index 000000000000..82be64845152 --- /dev/null +++ b/tests/run_agent/test_63425_credential_pool_auto_detect.py @@ -0,0 +1,178 @@ +"""Reproduction test for issue #63425: Provider auto-detection discards credential pools. + +When AIAgent is constructed with provider=None and a recognized endpoint URL, +the provider auto-detection works but the credential pool is discarded because +pool validation runs before URL-based provider inference. +""" +from types import SimpleNamespace +from unittest.mock import patch, MagicMock + +import pytest + + +def _mock_client(api_key="test-key", base_url="https://api.anthropic.com"): + c = MagicMock() + c.api_key = api_key + c.base_url = base_url + c._default_headers = None + return c + + +class TestCredentialPoolPreservedOnAutoDetect: + """Issue #63425: credential pool should survive provider auto-detection.""" + + def test_anthropic_pool_preserved_with_url_auto_detect(self): + """When provider=None and base_url=api.anthropic.com, the passed + credential_pool should remain attached after auto-detection.""" + from agent.agent_init import init_agent + + # Build a minimal agent-like object (like tests use object.__new__) + from run_agent import AIAgent + agent = object.__new__(AIAgent) + agent._base_url = "" + agent._base_url_lower = "" + agent._base_url_hostname = "" + + pool = SimpleNamespace(provider="anthropic") + + with patch("agent.auxiliary_client.resolve_provider_client", return_value=(None, None)), \ + patch("run_agent.get_tool_definitions", return_value=[]), \ + patch('agent.anthropic_adapter.build_anthropic_client', return_value=MagicMock()), \ + patch('agent.anthropic_adapter.resolve_anthropic_token', return_value=''), \ + patch('agent.anthropic_adapter._is_oauth_token', return_value=False), \ + patch('agent.azure_identity_adapter.is_token_provider', return_value=False), \ + patch('hermes_cli.model_normalize.normalize_model_for_provider', return_value='test-model'), \ + patch('agent.credential_pool.load_pool', return_value=MagicMock()), \ + patch('hermes_cli.config.load_config', return_value={}), \ + patch('hermes_cli.config.get_compatible_custom_providers', return_value=[]), \ + patch('agent.iteration_budget.IterationBudget'), \ + patch('hermes_cli.config.cfg_get', return_value=None): + + init_agent( + agent, + base_url="https://api.anthropic.com", + api_key="test-key", + provider=None, + model="test-model", + credential_pool=pool, + skip_context_files=True, + skip_memory=True, + quiet_mode=True, + ) + + print(f"agent.provider = {agent.provider!r}") + print(f"agent.api_mode = {agent.api_mode!r}") + print(f"agent._credential_pool is pool = {agent._credential_pool is pool}") + + assert agent.provider == "anthropic", ( + f"Provider should be auto-detected as 'anthropic', got {agent.provider!r}" + ) + assert agent.api_mode == "anthropic_messages", ( + f"api_mode should be 'anthropic_messages', got {agent.api_mode!r}" + ) + assert agent._credential_pool is pool, ( + "Credential pool was discarded! agent._credential_pool is NOT the " + "same object that was passed to AIAgent().\n" + f" Expected: {id(pool)}\n" + f" Got: {id(agent._credential_pool)}" + ) + + def test_codex_pool_preserved_with_url_auto_detect(self): + """When provider=None and base_url=chatgpt.com/backend-api/codex, the + passed credential_pool should remain attached.""" + from agent.agent_init import init_agent + from run_agent import AIAgent + agent = object.__new__(AIAgent) + agent._base_url = "" + agent._base_url_lower = "" + agent._base_url_hostname = "" + + pool = SimpleNamespace(provider="openai-codex") + + with patch("agent.auxiliary_client.resolve_provider_client", return_value=(_mock_client("key", "https://chatgpt.com/backend-api/codex"), None)), \ + patch("run_agent.get_tool_definitions", return_value=[]), \ + patch("run_agent.OpenAI", return_value=MagicMock()), \ + patch('agent.anthropic_adapter.build_anthropic_client', return_value=MagicMock()), \ + patch('agent.anthropic_adapter.resolve_anthropic_token', return_value=''), \ + patch('agent.anthropic_adapter._is_oauth_token', return_value=False), \ + patch('agent.azure_identity_adapter.is_token_provider', return_value=False), \ + patch('hermes_cli.model_normalize.normalize_model_for_provider', return_value='test-model'), \ + patch('agent.credential_pool.load_pool', return_value=MagicMock()), \ + patch('hermes_cli.config.load_config', return_value={}), \ + patch('hermes_cli.config.get_compatible_custom_providers', return_value=[]), \ + patch('agent.iteration_budget.IterationBudget'), \ + patch('hermes_cli.config.cfg_get', return_value=None): + + init_agent( + agent, + base_url="https://chatgpt.com/backend-api/codex", + api_key="test-key", + provider=None, + model="gpt-5.5", + credential_pool=pool, + skip_context_files=True, + skip_memory=True, + quiet_mode=True, + ) + + print(f"\nagent.provider = {agent.provider!r}") + print(f"agent.api_mode = {agent.api_mode!r}") + print(f"agent._credential_pool is pool = {agent._credential_pool is pool}") + + assert agent.provider == "openai-codex", ( + f"Provider should be auto-detected as 'openai-codex', got {agent.provider!r}" + ) + assert agent._credential_pool is pool, ( + "Credential pool was discarded! agent._credential_pool is NOT the " + f"same object. Expected: {id(pool)}, Got: {id(agent._credential_pool)}" + ) + + def test_xai_pool_preserved_with_url_auto_detect(self): + """When provider=None and base_url=api.x.ai, the passed + credential_pool should remain attached.""" + from agent.agent_init import init_agent + from run_agent import AIAgent + agent = object.__new__(AIAgent) + agent._base_url = "" + agent._base_url_lower = "" + agent._base_url_hostname = "" + + pool = SimpleNamespace(provider="xai") + + with patch("agent.auxiliary_client.resolve_provider_client", return_value=(_mock_client("key", "https://api.x.ai"), None)), \ + patch("run_agent.get_tool_definitions", return_value=[]), \ + patch("run_agent.OpenAI", return_value=MagicMock()), \ + patch('agent.anthropic_adapter.build_anthropic_client', return_value=MagicMock()), \ + patch('agent.anthropic_adapter.resolve_anthropic_token', return_value=''), \ + patch('agent.anthropic_adapter._is_oauth_token', return_value=False), \ + patch('agent.azure_identity_adapter.is_token_provider', return_value=False), \ + patch('hermes_cli.model_normalize.normalize_model_for_provider', return_value='test-model'), \ + patch('agent.credential_pool.load_pool', return_value=MagicMock()), \ + patch('hermes_cli.config.load_config', return_value={}), \ + patch('hermes_cli.config.get_compatible_custom_providers', return_value=[]), \ + patch('agent.iteration_budget.IterationBudget'), \ + patch('hermes_cli.config.cfg_get', return_value=None): + + init_agent( + agent, + base_url="https://api.x.ai", + api_key="test-key", + provider=None, + model="grok-5", + credential_pool=pool, + skip_context_files=True, + skip_memory=True, + quiet_mode=True, + ) + + print(f"\nagent.provider = {agent.provider!r}") + print(f"agent.api_mode = {agent.api_mode!r}") + print(f"agent._credential_pool is pool = {agent._credential_pool is pool}") + + assert agent.provider == "xai", ( + f"Provider should be auto-detected as 'xai', got {agent.provider!r}" + ) + assert agent._credential_pool is pool, ( + "Credential pool was discarded! agent._credential_pool is NOT the " + f"same object. Expected: {id(pool)}, Got: {id(agent._credential_pool)}" + ) From ffa525754da49033ccd93ed622804560bb5b163d Mon Sep 17 00:00:00 2001 From: dsad Date: Sun, 12 Jul 2026 23:48:52 +0300 Subject: [PATCH 050/149] fix(cron): keep live one-shots when running-set check fails --- cron/jobs.py | 8 +++++++- tests/cron/test_jobs.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cron/jobs.py b/cron/jobs.py index 90c318742e6f..f81e40f2598e 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -209,7 +209,13 @@ def _job_running_in_this_process(job_id: str) -> bool: from cron.scheduler import get_running_job_ids return job_id in get_running_job_ids() except Exception: - return False + logger.warning( + "Cron running-set liveness check failed for job %r; keeping the " + "entry to avoid deleting a possibly live one-shot run", + job_id, + exc_info=True, + ) + return True def _jobs_lock_file() -> Path: diff --git a/tests/cron/test_jobs.py b/tests/cron/test_jobs.py index d7d16e551341..6e0b5633cc5d 100644 --- a/tests/cron/test_jobs.py +++ b/tests/cron/test_jobs.py @@ -1147,6 +1147,39 @@ class TestGetDueJobs: assert get_due_jobs() == [] assert get_job("inflight") is None # stale entry cleaned up + def test_stale_maxed_oneshot_kept_when_running_check_errors( + self, tmp_cron_dir, monkeypatch + ): + """If the running-set lookup fails, do not delete a possibly live run. + + This is the fail-closed sibling of #62002/#62014: the liveness check is + the only signal distinguishing "expired but live" from "stale and dead". + Treating a lookup error as "not running" reopens the data-loss path by + deleting the job record underneath an in-flight one-shot. + """ + import cron.scheduler as scheduler_mod + from cron.jobs import _hermes_now, _oneshot_run_claim_ttl_seconds + + monkeypatch.delenv("HERMES_CRON_TIMEOUT", raising=False) + ttl = _oneshot_run_claim_ttl_seconds() + t0 = _hermes_now() + run_at = (t0 - timedelta(seconds=ttl + 300)).isoformat() + save_jobs([{ + "id": "inflight-error", "name": "flight check", "prompt": "x", + "schedule": {"kind": "once", "run_at": run_at}, + "next_run_at": run_at, "enabled": True, "state": "scheduled", + "repeat": {"times": 1, "completed": 1}, + "run_claim": {"at": run_at, "by": "this-machine"}, + }]) + + def fail_running_set(): + raise RuntimeError("running set unavailable") + + monkeypatch.setattr(scheduler_mod, "get_running_job_ids", fail_running_set) + + assert get_due_jobs() == [] + assert get_job("inflight-error") is not None + def test_run_claim_heartbeat_keeps_long_run_claimed_past_ttl( self, tmp_cron_dir, monkeypatch ): From fd461b58cad4ed64d0121b60481694a597aa1e6c Mon Sep 17 00:00:00 2001 From: dsad Date: Mon, 13 Jul 2026 00:57:31 +0300 Subject: [PATCH 051/149] fix(gateway): fail closed on compression state probe errors --- gateway/run.py | 22 ++++++++++++-- .../test_compression_in_flight_check.py | 30 +++++++++++++++++++ ...st_compression_interrupt_demotion_56391.py | 20 +++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 9b19190338d4..8d63da9c952a 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -5262,8 +5262,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_id = await asyncio.to_thread( self._lookup_session_id_under_store_lock, session_store, session_key ) - except Exception: + except (AttributeError, TypeError): return False + except Exception: + logger.warning( + "Compression in-flight check failed while reading session %s; " + "treating compression as active to avoid interrupting a possible " + "parent-session rotation", + session_key, + exc_info=True, + ) + return True if not session_id: return False session_db = getattr(self, "_session_db", None) @@ -5275,8 +5284,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew raw_db.get_compression_lock_holder, str(session_id) ) return bool(holder) - except Exception: + except (AttributeError, TypeError): return False + except Exception: + logger.warning( + "Compression in-flight check failed while reading lock holder " + "for session %s; treating compression as active to avoid " + "interrupting a possible parent-session rotation", + session_id, + exc_info=True, + ) + return True @staticmethod def _lookup_session_id_under_store_lock(session_store, session_key: str): diff --git a/tests/gateway/test_compression_in_flight_check.py b/tests/gateway/test_compression_in_flight_check.py index 0a33c4bcb7f3..db1d0a1926be 100644 --- a/tests/gateway/test_compression_in_flight_check.py +++ b/tests/gateway/test_compression_in_flight_check.py @@ -60,6 +60,36 @@ async def test_returns_false_when_no_session_store(): assert await runner._session_has_compression_in_flight("k") is False +@pytest.mark.asyncio +async def test_structural_lock_absence_still_fails_open(): + runner = _make_runner(holder_value=None) + runner._session_db._db.get_compression_lock_holder = MagicMock( + side_effect=AttributeError("old SessionDB has no lock helper") + ) + + assert await runner._session_has_compression_in_flight("k") is False + + +@pytest.mark.asyncio +async def test_db_lock_probe_error_fails_closed(): + runner = _make_runner(holder_value=None) + runner._session_db._db.get_compression_lock_holder = MagicMock( + side_effect=RuntimeError("sqlite temporarily unavailable") + ) + + assert await runner._session_has_compression_in_flight("k") is True + + +@pytest.mark.asyncio +async def test_store_lookup_error_fails_closed(): + runner = _make_runner(holder_value=None) + runner.session_store._ensure_loaded_locked = MagicMock( + side_effect=RuntimeError("routing index temporarily unavailable") + ) + + assert await runner._session_has_compression_in_flight("k") is True + + @pytest.mark.asyncio async def test_db_call_runs_off_event_loop(): """Regression core: get_compression_lock_holder MUST execute in non-event-loop thread.""" diff --git a/tests/gateway/test_compression_interrupt_demotion_56391.py b/tests/gateway/test_compression_interrupt_demotion_56391.py index cf9f4287c1f7..ee97bd0107a7 100644 --- a/tests/gateway/test_compression_interrupt_demotion_56391.py +++ b/tests/gateway/test_compression_interrupt_demotion_56391.py @@ -182,6 +182,26 @@ class TestBusyHandlerDemotesInterruptForCompression: parent.interrupt.assert_called_once_with("please stop") + @pytest.mark.asyncio + async def test_lock_probe_error_does_not_interrupt_parent_session(self) -> None: + runner = _make_runner() + adapter = _make_adapter() + event = _make_event(text="follow up while lock state is unavailable") + sk = build_session_key(event.source) + parent = _make_parent_no_subagents() + runner._running_agents[sk] = parent + runner.adapters[event.source.platform] = adapter + runner._session_db._db.get_compression_lock_holder.side_effect = RuntimeError( + "sqlite temporarily unavailable" + ) + + with patch("gateway.run.merge_pending_message_event"): + handled = await runner._handle_active_session_busy_message(event, sk) + + assert handled is True + parent.interrupt.assert_not_called() + assert adapter._pending_messages.get(sk) is event + @pytest.mark.asyncio async def test_pending_sentinel_does_not_trigger_false_positive(self) -> None: runner = _make_runner() From aa56243a8985d98bd1801dd477aec3e81473f87e Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Thu, 4 Jun 2026 19:28:55 -0400 Subject: [PATCH 052/149] perf(cli): skip npm install during update when lockfile is unchanged (#17268) (cherry picked from commit 8fb6d5e910b6fd89bdc698c477cc5f039a0deabd) (cherry picked from commit 27474007b9463d7ce19d981a24aeeac552e79f48) --- hermes_cli/main.py | 41 +++++++++++++ tests/hermes_cli/test_cmd_update.py | 92 ++++++++++++++++++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index f10fef3fd958..349aad749360 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -8126,6 +8126,39 @@ def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: return resolve_uv() or shutil.which("uv") +def _npm_lockfile_changed(hermes_root: Path) -> bool: + lockfile = PROJECT_ROOT / "package-lock.json" + if not lockfile.exists(): + return True + # Also check that node_modules exists; a matching hash with missing + # node_modules means the cache was recorded by another checkout. + if not (PROJECT_ROOT / "node_modules").is_dir(): + return True + try: + current = hashlib.sha256(lockfile.read_bytes()).hexdigest() + # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. + cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] + cache_file = hermes_root / f".npm_lock_hash_{cache_key}" + if not cache_file.exists(): + return True + return cache_file.read_text(encoding="utf-8").strip() != current + except OSError: + return True + + +def _record_npm_lockfile_hash(hermes_root: Path) -> None: + lockfile = PROJECT_ROOT / "package-lock.json" + if not lockfile.exists(): + return + try: + digest = hashlib.sha256(lockfile.read_bytes()).hexdigest() + cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] + cache_file = hermes_root / f".npm_lock_hash_{cache_key}" + cache_file.write_text(digest, encoding="utf-8") + except OSError: + logger.debug("Could not write npm lockfile hash cache") + + def _update_node_dependencies() -> None: from hermes_constants import find_node_executable, with_hermes_node_path @@ -8136,6 +8169,13 @@ def _update_node_dependencies() -> None: if not (PROJECT_ROOT / "package.json").exists(): return + from hermes_constants import get_default_hermes_root + + hermes_root = get_default_hermes_root() + if not _npm_lockfile_changed(hermes_root): + logger.info("npm lockfile unchanged, skipping npm install") + return + # With a single workspace lockfile the root install would cover ALL # workspaces — but apps/desktop pulls in Electron as a devDependency, # and its postinstall downloads a ~200MB binary. Most users don't @@ -8180,6 +8220,7 @@ def _update_node_dependencies() -> None: env=nixos_env, ) if ws_result.returncode == 0: + _record_npm_lockfile_hash(hermes_root) print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)") else: print(" ⚠ npm workspace install failed") diff --git a/tests/hermes_cli/test_cmd_update.py b/tests/hermes_cli/test_cmd_update.py index 56671aa7ecc5..d8643fb43b62 100644 --- a/tests/hermes_cli/test_cmd_update.py +++ b/tests/hermes_cli/test_cmd_update.py @@ -1,5 +1,6 @@ """Tests for cmd_update — branch fallback when remote branch doesn't exist.""" +import hashlib import subprocess from types import SimpleNamespace from unittest.mock import patch @@ -70,6 +71,92 @@ def _patch_managed_uv(request): yield +class TestCmdUpdateNpmLockfileCache: + @staticmethod + def _cache_file(hermes_root, project_root): + cache_key = hashlib.sha256(str(project_root).encode()).hexdigest()[:12] + return hermes_root / f".npm_lock_hash_{cache_key}" + + def test_npm_lockfile_changed_no_cache(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "node_modules").mkdir() + + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_npm_lockfile_changed_matching(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + content = b'{"lockfileVersion": 3}' + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_bytes(content) + (tmp_path / "node_modules").mkdir() + digest = hashlib.sha256(content).hexdigest() + self._cache_file(tmp_path, tmp_path).write_text(digest) + + assert hm._npm_lockfile_changed(tmp_path) is False + + def test_npm_lockfile_changed_mismatch(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "node_modules").mkdir() + self._cache_file(tmp_path, tmp_path).write_text("old-digest") + + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_npm_lockfile_changed_missing_node_modules(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + content = b'{"lockfileVersion": 3}' + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + self._cache_file(tmp_path, tmp_path).write_text(digest) + # node_modules missing: should report changed even though hash matches + + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_record_npm_lockfile_hash(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + content = b'{"lockfileVersion": 3}' + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_bytes(content) + + hm._record_npm_lockfile_hash(tmp_path) + + expected = hashlib.sha256(content).hexdigest() + assert self._cache_file(tmp_path, tmp_path).read_text() == expected + + def test_npm_lockfile_changed_cache_read_error(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "node_modules").mkdir() + # Make cache file a directory to cause OSError on read + self._cache_file(tmp_path, tmp_path).mkdir(parents=True) + + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_update_skips_npm_when_lockfile_unchanged(self, tmp_path, monkeypatch): + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package.json").write_text("{}") + + with patch("shutil.which", return_value="/usr/bin/npm"), \ + patch.object(hm, "_npm_lockfile_changed", return_value=False), \ + patch("subprocess.run") as mock_run: + hm._update_node_dependencies() + + mock_run.assert_not_called() + + class TestCmdUpdatePip: """Regression tests for pip-install update flows.""" @@ -246,7 +333,10 @@ class TestCmdUpdateBranchFallback: ), patch.object(hm, "_sync_with_upstream_if_needed") as sync_mock: cmd_update(mock_args) - sync_mock.assert_called_once_with(["git"], PROJECT_ROOT) + expected_git_cmd = ( + ["git", "-c", "windows.appendAtomically=false"] if hm._is_windows() else ["git"] + ) + sync_mock.assert_called_once_with(expected_git_cmd, PROJECT_ROOT) captured = capsys.readouterr() assert "Already up to date!" in captured.out From d426b9ddfe55cc3bfc8577e50bd041f63970d06b Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Fri, 10 Jul 2026 00:21:24 +0530 Subject: [PATCH 053/149] fix: derive skip-key manifests from npm workspaces config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 2 from @ethernet8023 on #61580: 1. The manifest list was a hardcoded root/ui-tui/web trio — desktop and any future workspace escaped the skip key even though step 1's root install hoists deps for every workspace. The list is now expanded from the root package.json 'workspaces' globs (npm's own source of truth): on the real repo that yields all 8 manifests incl. apps/desktop, apps/bootstrap-installer, apps/shared, and the nested ui-tui/packages/hermes-ink. Unreadable package.json falls back to root manifests only (never skips more than main would install). 2. --prefer-offline dropped entirely (this branch no longer carries #39399): local 3-run benchmarks on the repo's real manifests show the flag is noise on npm ci with a warm cache (root: 0.90s vs 0.84s avg; ws: 4.02s vs 4.00s avg) — npm ci does no resolution and the content-addressed cache already serves tarballs locally. It also carried the stale-resolution risk on the npm install fallback the reviewer flagged. All the real win is the skip itself (0s vs ~5s+). Tests: workspace-glob edit (desktop), literal-listed edit, and new-workspace-under-glob all defeat the skip; verified against the real repo's workspace config (8 manifests picked up). --- hermes_cli/main.py | 63 ++++++++++++++++++++--- tests/hermes_cli/test_cmd_update.py | 80 ++++++++++++++++++++++++++--- 2 files changed, 129 insertions(+), 14 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 349aad749360..16eee2489cc9 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -8126,16 +8126,68 @@ def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: return resolve_uv() or shutil.which("uv") +def _npm_manifest_paths() -> tuple[Path, ...]: + """Manifests whose changes must defeat the update-skip. + + The lockfile alone is NOT a sufficient key: on a local checkout a dev + can edit package.json (root or a workspace) without running npm — the + lockfile is then unchanged but `hermes update` is exactly the step + expected to sync node_modules (via the `npm install` fallback in + _run_npm_install_deterministic). + + The workspace list is pulled from the root package.json's `workspaces` + globs (npm's own source of truth) rather than hardcoded, so adding a + workspace can never silently escape the skip key. The root install + (step 1, --workspaces=false) still hoists shared deps for EVERY + workspace — desktop included — so all of them belong in the key, not + just the ones step 2 installs. Falls back to hashing just root + manifests if package.json is unreadable (never skips more than main + would have installed). + """ + root_pkg = PROJECT_ROOT / "package.json" + paths = [PROJECT_ROOT / "package-lock.json", root_pkg] + try: + workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get( + "workspaces", [] + ) + if isinstance(workspaces, dict): # legacy {"packages": [...]} form + workspaces = workspaces.get("packages", []) + for pattern in workspaces: + for match in sorted(PROJECT_ROOT.glob(str(pattern))): + manifest = match / "package.json" + if manifest.is_file(): + paths.append(manifest) + except (OSError, json.JSONDecodeError, TypeError): + pass + return tuple(paths) + + +def _npm_manifests_digest() -> str | None: + """Combined sha256 over the lockfile + all workspace package.json files. + + Returns None when the lockfile is missing (never skip then). + """ + if not (PROJECT_ROOT / "package-lock.json").exists(): + return None + h = hashlib.sha256() + for p in _npm_manifest_paths(): + h.update(str(p.relative_to(PROJECT_ROOT)).encode()) + try: + h.update(p.read_bytes()) + except OSError: + h.update(b"") + return h.hexdigest() + + def _npm_lockfile_changed(hermes_root: Path) -> bool: - lockfile = PROJECT_ROOT / "package-lock.json" - if not lockfile.exists(): + current = _npm_manifests_digest() + if current is None: return True # Also check that node_modules exists; a matching hash with missing # node_modules means the cache was recorded by another checkout. if not (PROJECT_ROOT / "node_modules").is_dir(): return True try: - current = hashlib.sha256(lockfile.read_bytes()).hexdigest() # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" @@ -8147,11 +8199,10 @@ def _npm_lockfile_changed(hermes_root: Path) -> bool: def _record_npm_lockfile_hash(hermes_root: Path) -> None: - lockfile = PROJECT_ROOT / "package-lock.json" - if not lockfile.exists(): + digest = _npm_manifests_digest() + if digest is None: return try: - digest = hashlib.sha256(lockfile.read_bytes()).hexdigest() cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" cache_file.write_text(digest, encoding="utf-8") diff --git a/tests/hermes_cli/test_cmd_update.py b/tests/hermes_cli/test_cmd_update.py index d8643fb43b62..3be32ab8cf04 100644 --- a/tests/hermes_cli/test_cmd_update.py +++ b/tests/hermes_cli/test_cmd_update.py @@ -89,12 +89,10 @@ class TestCmdUpdateNpmLockfileCache: def test_npm_lockfile_changed_matching(self, tmp_path, monkeypatch): from hermes_cli import main as hm - content = b'{"lockfileVersion": 3}' monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) - (tmp_path / "package-lock.json").write_bytes(content) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') (tmp_path / "node_modules").mkdir() - digest = hashlib.sha256(content).hexdigest() - self._cache_file(tmp_path, tmp_path).write_text(digest) + self._cache_file(tmp_path, tmp_path).write_text(hm._npm_manifests_digest()) assert hm._npm_lockfile_changed(tmp_path) is False @@ -123,14 +121,80 @@ class TestCmdUpdateNpmLockfileCache: def test_record_npm_lockfile_hash(self, tmp_path, monkeypatch): from hermes_cli import main as hm - content = b'{"lockfileVersion": 3}' monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) - (tmp_path / "package-lock.json").write_bytes(content) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') hm._record_npm_lockfile_hash(tmp_path) - expected = hashlib.sha256(content).hexdigest() - assert self._cache_file(tmp_path, tmp_path).read_text() == expected + assert ( + self._cache_file(tmp_path, tmp_path).read_text() + == hm._npm_manifests_digest() + ) + + def test_package_json_only_edit_defeats_skip(self, tmp_path, monkeypatch): + """Reviewer scenario (#61580): dev edits package.json WITHOUT running + npm — lockfile unchanged. `hermes update` must still install (the + npm-install fallback is what syncs node_modules in that state).""" + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "package.json").write_text('{"dependencies": {}}') + (tmp_path / "node_modules").mkdir() + hm._record_npm_lockfile_hash(tmp_path) + assert hm._npm_lockfile_changed(tmp_path) is False + + (tmp_path / "package.json").write_text( + '{"dependencies": {"left-pad": "^1.0.0"}}' + ) + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_workspace_package_json_edit_defeats_skip(self, tmp_path, monkeypatch): + """The manifest list comes from the root package.json `workspaces` + globs (npm's source of truth), so ANY workspace (desktop included) + defeats the skip, not a hardcoded set.""" + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "package.json").write_text( + '{"workspaces": ["apps/*", "ui-tui"]}' + ) + (tmp_path / "ui-tui").mkdir() + (tmp_path / "ui-tui" / "package.json").write_text("{}") + (tmp_path / "apps" / "desktop").mkdir(parents=True) + (tmp_path / "apps" / "desktop" / "package.json").write_text("{}") + (tmp_path / "node_modules").mkdir() + hm._record_npm_lockfile_hash(tmp_path) + assert hm._npm_lockfile_changed(tmp_path) is False + + # A glob-matched workspace (desktop) defeats the skip… + (tmp_path / "apps" / "desktop" / "package.json").write_text( + '{"name": "desktop"}' + ) + assert hm._npm_lockfile_changed(tmp_path) is True + + # …and so does a literal-listed one. + hm._record_npm_lockfile_hash(tmp_path) + assert hm._npm_lockfile_changed(tmp_path) is False + (tmp_path / "ui-tui" / "package.json").write_text('{"name": "x"}') + assert hm._npm_lockfile_changed(tmp_path) is True + + def test_new_workspace_added_defeats_skip(self, tmp_path, monkeypatch): + """Adding a whole new workspace dir under an existing glob changes + the manifest set itself — must also defeat the skip.""" + from hermes_cli import main as hm + + monkeypatch.setattr(hm, "PROJECT_ROOT", tmp_path) + (tmp_path / "package-lock.json").write_text('{"lockfileVersion": 3}') + (tmp_path / "package.json").write_text('{"workspaces": ["apps/*"]}') + (tmp_path / "node_modules").mkdir() + hm._record_npm_lockfile_hash(tmp_path) + assert hm._npm_lockfile_changed(tmp_path) is False + + (tmp_path / "apps" / "newtool").mkdir(parents=True) + (tmp_path / "apps" / "newtool" / "package.json").write_text("{}") + assert hm._npm_lockfile_changed(tmp_path) is True def test_npm_lockfile_changed_cache_read_error(self, tmp_path, monkeypatch): from hermes_cli import main as hm From 71e91f89b51da55c89e064be1198a6c98036a991 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:49:42 +0530 Subject: [PATCH 054/149] test(update): document shared npm cache scope --- hermes_cli/main.py | 9 +++++--- tests/hermes_cli/test_cmd_update.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 16eee2489cc9..45ce0a7a8c26 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -8222,8 +8222,11 @@ def _update_node_dependencies() -> None: from hermes_constants import get_default_hermes_root - hermes_root = get_default_hermes_root() - if not _npm_lockfile_changed(hermes_root): + # This cache describes PROJECT_ROOT/node_modules, which is shared by every + # Hermes profile using this checkout. Keep one per-checkout cache under the + # shared Hermes root rather than rerunning npm once per named profile. + shared_hermes_root = get_default_hermes_root() + if not _npm_lockfile_changed(shared_hermes_root): logger.info("npm lockfile unchanged, skipping npm install") return @@ -8271,7 +8274,7 @@ def _update_node_dependencies() -> None: env=nixos_env, ) if ws_result.returncode == 0: - _record_npm_lockfile_hash(hermes_root) + _record_npm_lockfile_hash(shared_hermes_root) print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)") else: print(" ⚠ npm workspace install failed") diff --git a/tests/hermes_cli/test_cmd_update.py b/tests/hermes_cli/test_cmd_update.py index 3be32ab8cf04..b1458cbd3c2f 100644 --- a/tests/hermes_cli/test_cmd_update.py +++ b/tests/hermes_cli/test_cmd_update.py @@ -220,6 +220,40 @@ class TestCmdUpdateNpmLockfileCache: mock_run.assert_not_called() + def test_update_uses_one_shared_npm_cache_across_profiles( + self, tmp_path, monkeypatch + ): + """The npm cache describes checkout-global node_modules, not a profile.""" + from hermes_cli import main as hm + import hermes_constants + + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / "package.json").write_text("{}") + shared_root = tmp_path / ".hermes" + named_profile = shared_root / "profiles" / "work" + named_profile.mkdir(parents=True) + + monkeypatch.setattr(hm, "PROJECT_ROOT", checkout) + monkeypatch.setattr(hermes_constants.Path, "home", lambda: tmp_path) + monkeypatch.setattr( + hermes_constants, "find_node_executable", lambda _name: "/usr/bin/npm" + ) + + cache_roots = [] + with patch.object( + hm, + "_npm_lockfile_changed", + side_effect=lambda root: cache_roots.append(root) or False, + ): + monkeypatch.setenv("HERMES_HOME", str(shared_root)) + hm._update_node_dependencies() + + monkeypatch.setenv("HERMES_HOME", str(named_profile)) + hm._update_node_dependencies() + + assert cache_roots == [shared_root, shared_root] + class TestCmdUpdatePip: """Regression tests for pip-install update flows.""" From ca907480ae3448880464f557e12e0662ec28cb23 Mon Sep 17 00:00:00 2001 From: HexLab98 Date: Sun, 12 Jul 2026 21:29:55 +0700 Subject: [PATCH 055/149] fix(gateway): allow ws_orphan_reap rows in session recovery (#63207) Whitelist ws_orphan_reap alongside agent_close in find_latest_gateway_session_for_peer so gateway stale-routing self-heal can reopen wrongly-reaped messaging sessions instead of minting empty replacements. Layer A prevention already landed in #60609. --- gateway/session.py | 7 ++++--- hermes_state.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/gateway/session.py b/gateway/session.py index fea3ba4a3c0d..f04090445e75 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -1904,9 +1904,10 @@ class SessionStore: # Stale routing self-heal (#54878): the in-memory entry # points at a session that has ALREADY been ended in # state.db. Drop it and fall through to recovery/create. - # Recovery finder reopens ``agent_close`` rows (preserving - # the transcript) but returns None for other end_reasons - # (e.g. /new), starting a fresh session. + # Recovery finder reopens ``agent_close`` and mistaken + # ``ws_orphan_reap`` rows (preserving the transcript) but + # returns None for other end_reasons (e.g. /new), starting + # a fresh session. logger.warning( "gateway.session: routing key %r -> %s is ended in " "state.db but still live in sessions.json; dropping " diff --git a/hermes_state.py b/hermes_state.py index 79e95ffa66e4..7181266fe1ca 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -2113,9 +2113,10 @@ class SessionDB: pruned after process-level restart bugs. New gateway sessions persist the deterministic ``session_key`` on the durable session row so the mapping can be rebuilt exactly. Rows ended only by older gateway - cleanup's ``agent_close`` bug are treated as recoverable; explicit - conversation boundaries such as /new, /resume switches, and compression - splits are not. + cleanup's ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap`` + (dashboard viewer disconnect before #60609) are treated as recoverable; + explicit conversation boundaries such as /new, /resume switches, and + compression splits are not. """ if not session_key: return None @@ -2125,7 +2126,7 @@ class SessionDB: SELECT * FROM sessions WHERE session_key = ? AND source = ? - AND (ended_at IS NULL OR end_reason = 'agent_close') + AND (ended_at IS NULL OR end_reason IN ('agent_close', 'ws_orphan_reap')) AND (COALESCE(message_count, 0) > 0 OR EXISTS ( SELECT 1 FROM messages WHERE messages.session_id = sessions.id LIMIT 1 )) @@ -2150,7 +2151,7 @@ class SessionDB: AND COALESCE(chat_id, '') = COALESCE(?, '') AND COALESCE(chat_type, '') = COALESCE(?, '') AND COALESCE(thread_id, '') = COALESCE(?, '') - AND (ended_at IS NULL OR end_reason = 'agent_close') + AND (ended_at IS NULL OR end_reason IN ('agent_close', 'ws_orphan_reap')) AND (COALESCE(message_count, 0) > 0 OR EXISTS ( SELECT 1 FROM messages WHERE messages.session_id = sessions.id LIMIT 1 )) From 7452467f5409c8aeedd78adcf81c12f91437b39e Mon Sep 17 00:00:00 2001 From: HexLab98 Date: Mon, 13 Jul 2026 07:34:29 +0700 Subject: [PATCH 056/149] test(gateway): cover ws_orphan_reap session recovery (#63207) Regression tests for find_latest_gateway_session_for_peer and SessionStore stale-routing self-heal when end_reason is ws_orphan_reap. Pin manual approval mode in blocking E2E tests so smart aux-LLM resolution does not flake CI. --- tests/gateway/test_approve_deny_commands.py | 10 +++++++ .../test_session_store_runtime_stale_guard.py | 18 ++++++++++++ tests/test_hermes_state.py | 28 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/tests/gateway/test_approve_deny_commands.py b/tests/gateway/test_approve_deny_commands.py index d715f26d5faf..6ac15ec8a0fd 100644 --- a/tests/gateway/test_approve_deny_commands.py +++ b/tests/gateway/test_approve_deny_commands.py @@ -425,6 +425,16 @@ class TestBlockingApprovalE2E: os.environ.pop("HERMES_GATEWAY_SESSION", None) os.environ.pop("HERMES_EXEC_ASK", None) os.environ.pop("HERMES_SESSION_KEY", None) + # These E2E tests exercise manual gateway blocking; default config is + # approvals.mode=smart which may auto-approve/deny via aux LLM before + # notify_cb runs (flaky on CI when the LLM is slow or unavailable). + self._approval_mode_patch = patch( + "tools.approval._get_approval_mode", return_value="manual" + ) + self._approval_mode_patch.start() + + def teardown_method(self): + self._approval_mode_patch.stop() def test_blocking_approval_approve_once(self): """check_all_command_guards blocks until resolve_gateway_approval is called.""" diff --git a/tests/gateway/test_session_store_runtime_stale_guard.py b/tests/gateway/test_session_store_runtime_stale_guard.py index 57f8c624bf25..aa4415e53453 100644 --- a/tests/gateway/test_session_store_runtime_stale_guard.py +++ b/tests/gateway/test_session_store_runtime_stale_guard.py @@ -142,6 +142,24 @@ class TestRuntimeStaleGuard: # A brand-new session row must NOT have been created. db.create_session.assert_not_called() + def test_stale_ws_orphan_reap_entry_recovered_preserving_session_id(self, tmp_path): + """Stale ``ws_orphan_reap`` entry → recovery reopens the SAME session_id (#63207).""" + source = _source() + db = _db_returning({"sid_stale": {"end_reason": "ws_orphan_reap", "id": "sid_stale"}}) + db.find_latest_gateway_session_for_peer.return_value = { + "id": "sid_stale", + "started_at": (datetime.now() - timedelta(hours=2)).timestamp(), + } + store = _make_store_with_db(tmp_path, db) + key = store._generate_session_key(source) + store._entries[key] = _make_entry(key, "sid_stale") + + result = store.get_or_create_session(source) + + assert result.session_id == "sid_stale" + db.reopen_session.assert_called_once_with("sid_stale") + db.create_session.assert_not_called() + def test_stale_entry_creates_fresh_when_recovery_returns_none(self, tmp_path): """Stale entry, no recoverable row → brand-new session (no silent drop).""" source = _source() diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 2afcd6e4c6b0..eb6aaf279ea3 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -5627,6 +5627,34 @@ def test_gateway_session_peer_round_trip_and_recovery(db): assert recovered["id"] == "gw-session" +def test_gateway_session_recovery_reopens_ws_orphan_reap_rows(db): + """Rows wrongly ended by the TUI ws-orphan reaper must be recoverable (#63207).""" + db.create_session( + "reaped-gw-session", + "telegram", + user_id="user-1", + session_key="agent:main:telegram:dm:chat-1", + chat_id="chat-1", + chat_type="dm", + ) + db.append_message("reaped-gw-session", "user", "hello") + db.end_session("reaped-gw-session", "ws_orphan_reap") + + recovered = db.find_latest_gateway_session_for_peer( + source="telegram", + user_id="user-1", + session_key="agent:main:telegram:dm:chat-1", + chat_id="chat-1", + chat_type="dm", + ) + assert recovered["id"] == "reaped-gw-session" + + db.reopen_session("reaped-gw-session") + row = db.get_session("reaped-gw-session") + assert row["ended_at"] is None + assert row["end_reason"] is None + + def test_gateway_session_recovery_reopens_legacy_agent_close_rows(db): db.create_session( "closed-gw-session", From 2a0dd95ccfbd0f1d50e4d47a17ee402fe9b64165 Mon Sep 17 00:00:00 2001 From: SilentKnight87 <54852388+SilentKnight87@users.noreply.github.com> Date: Sun, 12 Jul 2026 11:19:27 -0400 Subject: [PATCH 057/149] fix(telegram): classify and dedup post-reconnect probe failures (#63243) --- plugins/platforms/telegram/adapter.py | 28 +++++-- .../test_telegram_network_reconnect.py | 74 +++++++++++++++++++ 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index de1dad27dd4c..7b959b1edf61 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -2349,8 +2349,13 @@ class TelegramAdapter(BasePlatformAdapter): wedged httpx pool fails this probe; a healthy one returns well under the timeout. - On any failure, re-enter the reconnect ladder so the existing - MAX_NETWORK_RETRIES path can ultimately escalate to fatal-error. + On connectivity failure, re-enter the reconnect ladder (via + ``_schedule_polling_recovery`` so the in-flight guard and task + bookkeeping apply) and let the existing MAX_NETWORK_RETRIES path + ultimately escalate to fatal-error. Auth/validation failures + (``InvalidToken``, ``BadRequest``, ...) are not connectivity symptoms + and must not trigger reconnect churn — same policy as the heartbeat + loop and the pending-update probe (#62098, #63243). """ HEARTBEAT_PROBE_DELAY = 60 PROBE_TIMEOUT = 10 @@ -2364,8 +2369,9 @@ class TelegramAdapter(BasePlatformAdapter): "[%s] Updater not running %ds after reconnect — treating as wedged", self.name, HEARTBEAT_PROBE_DELAY, ) - await self._handle_polling_network_error( - RuntimeError("Updater not running after reconnect heartbeat") + self._schedule_polling_recovery( + RuntimeError("Updater not running after reconnect heartbeat"), + reason="post-reconnect probe: updater not running", ) return @@ -2373,11 +2379,21 @@ class TelegramAdapter(BasePlatformAdapter): await asyncio.wait_for(self._app.bot.get_me(), PROBE_TIMEOUT) self._send_path_degraded = False except Exception as probe_err: + if not self._looks_like_network_error(probe_err): + logger.warning( + "[%s] Post-reconnect probe hit a non-connectivity error" + " (not retrying): %s", + self.name, _redact_telegram_error_text(probe_err), + ) + return logger.warning( "[%s] Polling heartbeat probe failed %ds after reconnect: %s", - self.name, HEARTBEAT_PROBE_DELAY, probe_err, + self.name, HEARTBEAT_PROBE_DELAY, + _redact_telegram_error_text(probe_err), + ) + self._schedule_polling_recovery( + probe_err, reason="post-reconnect probe failure" ) - await self._handle_polling_network_error(probe_err) def _disarm_ptb_retry_loop(self) -> None: """Synchronously stop PTB's internal polling retry loop. diff --git a/tests/gateway/test_telegram_network_reconnect.py b/tests/gateway/test_telegram_network_reconnect.py index 463877bf0728..c7e274d74ebb 100644 --- a/tests/gateway/test_telegram_network_reconnect.py +++ b/tests/gateway/test_telegram_network_reconnect.py @@ -385,6 +385,11 @@ async def test_heartbeat_probe_reenters_ladder_when_updater_not_running(): await adapter._verify_polling_after_reconnect() mock_app.bot.get_me.assert_not_called() + # Recovery is scheduled through _schedule_polling_recovery (#63243), so + # the ladder runs as the tracked _polling_error_task. + task = adapter._polling_error_task + assert task is not None + await task adapter._handle_polling_network_error.assert_awaited_once() err = adapter._handle_polling_network_error.await_args.args[0] assert isinstance(err, RuntimeError) @@ -421,6 +426,9 @@ async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out(): with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", new=fast_wait_for): await adapter._verify_polling_after_reconnect() + task = adapter._polling_error_task + assert task is not None + await task adapter._handle_polling_network_error.assert_awaited_once() @@ -445,12 +453,78 @@ async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error(): with patch("asyncio.sleep", new_callable=AsyncMock): await adapter._verify_polling_after_reconnect() + task = adapter._polling_error_task + assert task is not None + # _schedule_polling_recovery must also register the ladder in + # _background_tasks so a failed recovery isn't silently GC'd. + assert task in adapter._background_tasks + await task adapter._handle_polling_network_error.assert_awaited_once() assert isinstance( adapter._handle_polling_network_error.await_args.args[0], ConnectionError ) +@pytest.mark.asyncio +async def test_heartbeat_probe_ignores_auth_errors(): + """ + Auth/validation failures from the post-reconnect probe must not enter the + network-reconnect ladder (#63243): a revoked token would otherwise churn + through stop/drain/start_polling cycles that mask the real failure. + """ + adapter = _make_adapter() + + mock_updater = MagicMock() + mock_updater.running = True + + # Name-shaped like PTB's InvalidToken; _looks_like_network_error excludes + # it by class name, matching real PTB semantics. + invalid_token = type("InvalidToken", (Exception,), {})("token revoked") + + mock_app = MagicMock() + mock_app.updater = mock_updater + mock_app.bot.get_me = AsyncMock(side_effect=invalid_token) + adapter._app = mock_app + + adapter._handle_polling_network_error = AsyncMock() + + with patch("asyncio.sleep", new_callable=AsyncMock): + await adapter._verify_polling_after_reconnect() + + assert adapter._polling_error_task is None + adapter._handle_polling_network_error.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_heartbeat_probe_defers_to_inflight_recovery(): + """ + A probe failure while another recovery is mid-flight must not start a + second concurrent stop/drain/start_polling sequence (#63243) — overlapping + recoveries produce dueling getUpdates sessions (self-inflicted 409s). + """ + adapter = _make_adapter() + + mock_updater = MagicMock() + mock_updater.running = True + + mock_app = MagicMock() + mock_app.updater = mock_updater + mock_app.bot.get_me = AsyncMock(side_effect=ConnectionError("pool wedged")) + adapter._app = mock_app + + inflight = MagicMock() + inflight.done.return_value = False + adapter._polling_error_task = inflight + + adapter._handle_polling_network_error = AsyncMock() + + with patch("asyncio.sleep", new_callable=AsyncMock): + await adapter._verify_polling_after_reconnect() + + assert adapter._polling_error_task is inflight + adapter._handle_polling_network_error.assert_not_awaited() + + @pytest.mark.asyncio async def test_heartbeat_probe_skips_when_already_fatal(): """ From 1b5ceec2a0feca55fd350a81bda6873299493d78 Mon Sep 17 00:00:00 2001 From: HexLab98 Date: Sun, 12 Jul 2026 20:27:14 +0700 Subject: [PATCH 058/149] docs: clarify write safety, HERMES_WRITE_SAFE_ROOT, and file-mutation verifier Document that safe-root violations are hard-blocked (not approval-gated), add a security guide section for write_file/patch guards, and link cron and verifier docs so users trust the footer over agent summaries. --- .../docs/reference/environment-variables.md | 18 +++++- website/docs/user-guide/configuration.md | 16 ++++++ website/docs/user-guide/features/cron.md | 4 ++ website/docs/user-guide/security.md | 55 +++++++++++++++++-- .../reference/environment-variables.md | 18 +++++- 5 files changed, 103 insertions(+), 8 deletions(-) diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 53f7a18a0bed..5e6f1fc0de0c 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -745,7 +745,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us | `HERMES_PREFILL_MESSAGES_FILE` | Path to a JSON file of ephemeral prefill messages injected at API-call time. | | `HERMES_ALLOW_PRIVATE_URLS` | `true`/`false` — allow tools to fetch localhost/private-network URLs. Off by default in gateway mode. | | `HERMES_REDACT_SECRETS` | `true`/`false` — control secret redaction in tool output, logs, and chat responses (default: `true`). | -| `HERMES_WRITE_SAFE_ROOT` | Optional directory prefix that restricts `write_file`/`patch` writes; paths outside require approval. Supports multiple directories separated by `os.pathsep` (`:` on Unix, `;` on Windows). | +| `HERMES_WRITE_SAFE_ROOT` | Optional directory prefix that **hard-blocks** `write_file`/`patch` writes outside the listed roots (no approval prompt). Supports multiple directories separated by `os.pathsep` (`:` on Unix, `;` on Windows). See [HERMES_WRITE_SAFE_ROOT](#hermes_write_safe_root) below. | | `HERMES_DISABLE_LAZY_INSTALLS` | Internal bridge var set automatically in the official Docker image to prevent runtime dependency installs into the immutable `/opt/hermes` tree. The user-facing equivalent is `security.allow_lazy_installs: false` in `config.yaml`; do not set this in `.env`. | | `HERMES_DISABLE_FILE_STATE_GUARD` | Set to `1` to turn off the "file changed since you read it" guard on `patch`/`write_file`. | | `HERMES_CORE_TOOLS` | Comma-separated override for the canonical core tool list (advanced; rarely needed). | @@ -760,6 +760,22 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us | `HERMES_AGENT_LOGO` | Override the ASCII banner logo at CLI startup. | | `DELEGATION_MAX_CONCURRENT_CHILDREN` | Max parallel subagents per `delegate_task` batch (default: `3`, floor of 1, no ceiling). Also configurable via `delegation.max_concurrent_children` in `config.yaml` — the config value takes priority. | +### HERMES_WRITE_SAFE_ROOT {#hermes_write_safe_root} + +When this variable is set, `write_file` and `patch` may only target paths inside the listed directory prefix(es). Any path outside those roots is **rejected immediately** — the write does not go through the dangerous-command approval system and there is no prompt to override it. + +The official Docker image sets `HERMES_WRITE_SAFE_ROOT=/opt/data` alongside `HERMES_HOME=/opt/data` so the agent cannot escape the mounted data volume. + +**Do not add this to `~/.hermes/.env` unless you intend to sandbox writes.** A common mistake is pointing it at a project directory while expecting the agent to edit `~/.hermes/cron/jobs.json`, `~/.hermes/skills/`, or scripts under a profile — those paths are outside the sandbox and every `write_file`/`patch` to them fails. The tool error currently reads `Write denied: '…' is a protected system/credential file.` for all blocked writes (including safe-root violations); check `echo $HERMES_WRITE_SAFE_ROOT` when you see that message. + +To allow both a workspace and Hermes state, list both prefixes (order does not matter): + +```bash +export HERMES_WRITE_SAFE_ROOT=/path/to/project:/home/you/.hermes +``` + +Unset the variable or remove it from `.env` to restore normal writes (still subject to the credential-path denylist — see [File write safety](../user-guide/security.md#file-write-safety)). + ## Interface | Variable | Description | diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index e65399a67d85..1d2e3b672979 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1447,6 +1447,22 @@ Example footer: Set `file_mutation_verifier: false` (or `HERMES_FILE_MUTATION_VERIFIER=0`) to suppress the footer. The verifier only fires when real failures are outstanding at turn end — a model that retries a failed patch and succeeds within the same turn will not trigger it for that file. +**Trust the verifier over the model's summary.** The footer means the listed files were **not** modified on disk, even if the assistant's closing message says the task is done. Common causes: + +- **Write denied** — path is on the credential denylist or outside `HERMES_WRITE_SAFE_ROOT` (see [File write safety](./security.md#file-write-safety)) +- **Patch mismatch** — `old_string` did not match the file on disk +- **Syntax gate** — candidate content failed JSON/YAML/TOML validation before write + +Example footer when writes are blocked: + +``` +⚠️ File-mutation verifier: 2 file(s) were NOT modified this turn despite any wording above that may suggest otherwise. Run `git status` or `read_file` to confirm. + • ~/.hermes/cron/jobs.json — [patch] Write denied: '…' is a protected system/credential file. + • ~/.hermes/scripts/monitor.py — [write_file] Write denied: '…' is a protected system/credential file. +``` + +If writes to Hermes state (cron jobs, skills, scripts under `~/.hermes/`) are failing, check whether `HERMES_WRITE_SAFE_ROOT` is set in your environment. For cron changes, use the `cronjob` tool or `hermes cron edit` instead of patching `jobs.json` directly. + ### UI language for static messages The `display.language` setting translates a small set of static user-facing messages — the CLI approval prompt, a handful of gateway slash-command replies (e.g. restart-drain notices, "approval expired", "goal cleared"). It does **not** translate agent responses, log lines, tool output, error tracebacks, or slash-command descriptions — those stay in English. If you want the agent itself to reply in another language, just tell it in your prompt or system message. diff --git a/website/docs/user-guide/features/cron.md b/website/docs/user-guide/features/cron.md index a65a4bca1e78..3a07d5bde04a 100644 --- a/website/docs/user-guide/features/cron.md +++ b/website/docs/user-guide/features/cron.md @@ -732,6 +732,10 @@ The referenced jobs' most recent completed outputs are injected above the prompt Jobs are stored in `~/.hermes/cron/jobs.json`. Output from job runs is saved to `~/.hermes/cron/output/{job_id}/{timestamp}.md`. +:::tip +Ask the agent to manage jobs through the `cronjob` tool, `hermes cron edit`, or `/cron` — not by patching `jobs.json` directly. Direct edits can fail silently when [file write safety](../security.md#file-write-safety) blocks the path (for example when `HERMES_WRITE_SAFE_ROOT` is set), and the [file-mutation verifier](../configuration.md#file-mutation-verifier) footer is the authoritative signal that nothing was saved. +::: + Jobs may store `model` and `provider` as `null`. When those fields are omitted, Hermes resolves them at execution time from the global configuration. They only appear in the job record when a per-job override is set. The storage uses atomic file writes so interrupted writes do not leave a partially written job file behind. diff --git a/website/docs/user-guide/security.md b/website/docs/user-guide/security.md index 71d1b131d2ed..d2dd8306512b 100644 --- a/website/docs/user-guide/security.md +++ b/website/docs/user-guide/security.md @@ -10,15 +10,16 @@ Hermes Agent is designed with a defense-in-depth security model. This page cover ## Overview -The security model has seven layers: +The security model has eight layers: 1. **User authorization** — who can talk to the agent (allowlists, DM pairing) 2. **Dangerous command approval** — human-in-the-loop for destructive operations -3. **Container isolation** — Docker/Singularity/Modal sandboxing with hardened settings -4. **MCP credential filtering** — environment variable isolation for MCP subprocesses -5. **Context file scanning** — prompt injection detection in project files -6. **Cross-session isolation** — sessions cannot access each other's data or state; cron job storage paths are hardened against path traversal attacks -7. **Input sanitization** — working directory parameters in terminal tool backends are validated against an allowlist to prevent shell injection +3. **File write safety** — denylist and optional write sandbox for `write_file`/`patch` +4. **Container isolation** — Docker/Singularity/Modal sandboxing with hardened settings +5. **MCP credential filtering** — environment variable isolation for MCP subprocesses +6. **Context file scanning** — prompt injection detection in project files +7. **Cross-session isolation** — sessions cannot access each other's data or state; cron job storage paths are hardened against path traversal attacks +8. **Input sanitization** — working directory parameters in terminal tool backends are validated against an allowlist to prevent shell injection ## Dangerous Command Approval @@ -232,6 +233,48 @@ These patterns are loaded at startup and silently approved in all future session Use `hermes config edit` to review or remove patterns from your permanent allowlist. ::: +## File Write Safety {#file-write-safety} + +Before `write_file` or `patch` touches disk, Hermes checks the target path against a denylist and an optional sandbox. Blocked writes return an error to the agent immediately — **there is no approval prompt** and no way to override from the chat UI. The model may still claim the edit succeeded; when `display.file_mutation_verifier` is on (default), trust the [file-mutation verifier footer](./configuration.md#file-mutation-verifier) over the assistant's closing summary. + +### Protected paths (always blocked) + +These categories are always denied, even when `HERMES_WRITE_SAFE_ROOT` is unset: + +| Category | Examples | +|----------|----------| +| OS credential stores | `~/.ssh/`, `~/.aws/`, `~/.kube/`, `/etc/sudoers`, `~/.netrc` | +| Hermes credential stores | `auth.json`, `.env`, `.anthropic_oauth.json`, `mcp-tokens/`, `pairing/` under HERMES_HOME (active profile and global root) | +| Project secret files | `.env`, `.env.local`, `.env.production`, `.envrc` anywhere on disk | + +Sensitive paths inside the safe root are still blocked — pointing `HERMES_WRITE_SAFE_ROOT` at `$HOME` does not allow writing `~/.ssh/id_rsa`. + +The tool error message is always `Write denied: '…' is a protected system/credential file.` even when the actual reason is the safe-root sandbox below. If the path does not look like a credential file, check `echo $HERMES_WRITE_SAFE_ROOT`. + +### HERMES_WRITE_SAFE_ROOT (optional sandbox) + +When set, `write_file` and `patch` may only target paths inside the listed directory prefix(es). Anything outside is **hard-blocked** — not routed through dangerous-command approval. + +- Set automatically in the [official Docker image](https://github.com/NousResearch/hermes-agent) (`HERMES_WRITE_SAFE_ROOT=/opt/data`) +- Supports multiple roots separated by `:` on Unix or `;` on Windows +- **Do not add to `~/.hermes/.env` casually.** If you set it to a project directory, the agent cannot write to `~/.hermes/cron/jobs.json`, profile skills, or other Hermes state outside that prefix + +To allow both a workspace and Hermes home: + +```bash +export HERMES_WRITE_SAFE_ROOT=/path/to/project:/home/you/.hermes +``` + +Unset the variable to restore unrestricted writes (subject to the protected-path denylist). Full reference: [HERMES_WRITE_SAFE_ROOT](../reference/environment-variables.md#hermes_write_safe_root). + +### Cron and other Hermes state + +Do not ask the agent to `patch` `~/.hermes/cron/jobs.json` directly. Use the `cronjob` tool, [`hermes cron`](./features/cron.md), or `/cron` — they update the job store through the supported API. The same applies to other Hermes control files when write safety blocks direct edits. + +:::note Defense-in-depth, not a hard boundary +Write guards apply to `write_file` and `patch` only. The `terminal` tool runs as the same OS user and can still `cat` or overwrite denied paths via shell commands. The denylist reduces accidental damage and gives models a clear stop signal; it does not sandbox a hostile or compromised agent. +::: + ## User Authorization (Gateway) When running the messaging gateway, Hermes controls who can interact with the bot through a layered authorization system. diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md index 8de0ad1a93ec..c405e6312de5 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md @@ -559,7 +559,7 @@ Graph 事件（Teams 会议、日历、聊天等）的入站变更通知监听 | `HERMES_PREFILL_MESSAGES_FILE` | 包含在 API 调用时注入的临时预填消息的 JSON 文件路径。 | | `HERMES_ALLOW_PRIVATE_URLS` | `true`/`false`——允许工具获取 localhost/私有网络 URL。gateway 模式下默认关闭。 | | `HERMES_REDACT_SECRETS` | `true`/`false`——控制工具输出、日志和聊天响应中的密钥脱敏（默认：`true`）。 | -| `HERMES_WRITE_SAFE_ROOT` | 可选目录前缀，限制 `write_file`/`patch` 写入；超出范围的路径需要审批。支持多个目录，使用 `os.pathsep` 分隔（Unix 为 `:`，Windows 为 `;`）。 | +| `HERMES_WRITE_SAFE_ROOT` | 可选目录前缀，**硬阻止** `write_file`/`patch` 写入列出的根目录之外的路径（无审批提示）。支持多个目录，使用 `os.pathsep` 分隔（Unix 为 `:`，Windows 为 `;`）。详见下方 [HERMES_WRITE_SAFE_ROOT](#hermes_write_safe_root)。 | | `HERMES_DISABLE_LAZY_INSTALLS` | 官方 Docker 镜像中自动设置的内部桥接变量，用于阻止运行时将依赖安装到不可变的 `/opt/hermes` 树。面向用户的等价配置是 `config.yaml` 中的 `security.allow_lazy_installs: false`；不要在 `.env` 中手动设置此变量。 | | `HERMES_DISABLE_FILE_STATE_GUARD` | 设为 `1` 可关闭 `patch`/`write_file` 上的"文件自上次读取后已更改"保护。 | | `HERMES_CORE_TOOLS` | 规范核心工具列表的逗号分隔覆盖（高级；极少需要）。 | @@ -574,6 +574,22 @@ Graph 事件（Teams 会议、日历、聊天等）的入站变更通知监听 | `HERMES_AGENT_LOGO` | 覆盖 CLI 启动时的 ASCII 横幅 logo。 | | `DELEGATION_MAX_CONCURRENT_CHILDREN` | 每个 `delegate_task` 批次的最大并行子 agent 数（默认：`3`，下限为 1，无上限）。也可通过 `config.yaml` 中的 `delegation.max_concurrent_children` 配置——config 值优先。 | +### HERMES_WRITE_SAFE_ROOT {#hermes_write_safe_root} + +设置此变量后，`write_file` 和 `patch` 只能写入列出的目录前缀内的路径。超出这些根目录的路径会被**立即拒绝**——不会进入危险命令审批流程，也没有聊天界面可以覆盖。 + +官方 Docker 镜像会设置 `HERMES_WRITE_SAFE_ROOT=/opt/data` 与 `HERMES_HOME=/opt/data`，防止 agent 逃出挂载的数据卷。 + +**除非有意沙箱化写入，否则不要将此变量加入 `~/.hermes/.env`。** 常见错误是将其指向项目目录，却期望 agent 编辑 `~/.hermes/cron/jobs.json`、`~/.hermes/skills/` 或 profile 下的脚本——这些路径在沙箱外，每次 `write_file`/`patch` 都会失败。当前所有被阻止的写入（包括 safe-root 违规）都显示 `Write denied: '…' is a protected system/credential file.`；看到此消息时请检查 `echo $HERMES_WRITE_SAFE_ROOT`。 + +若需同时允许工作区和 Hermes 状态目录，列出两个前缀（顺序无关）： + +```bash +export HERMES_WRITE_SAFE_ROOT=/path/to/project:/home/you/.hermes +``` + +取消设置或从 `.env` 中移除此变量可恢复常规写入（仍受凭证路径拒绝列表约束——见[文件写入安全](../user-guide/security.md#file-write-safety)）。 + ## 界面 | 变量 | 描述 | From 55d826cceff1a024d141085cacf8a2fad02cd9f7 Mon Sep 17 00:00:00 2001 From: HexLab98 Date: Sun, 12 Jul 2026 20:34:26 +0700 Subject: [PATCH 059/149] fix(file-safety): distinguish safe-root write denial from credential blocks Return actionable errors when HERMES_WRITE_SAFE_ROOT blocks a path instead of labeling every denial as a protected credential file. Wire the helper through write_file, patch, delete/move, and the Copilot ACP shim; sync docs examples. --- agent/copilot_acp_client.py | 9 +++-- agent/file_safety.py | 35 ++++++++++++++----- tests/agent/test_copilot_acp_client.py | 7 +++- tests/tools/test_file_write_safety.py | 32 +++++++++++++++++ tools/file_operations.py | 21 ++++++----- .../docs/reference/environment-variables.md | 2 +- website/docs/user-guide/configuration.md | 4 +-- website/docs/user-guide/security.md | 2 +- .../reference/environment-variables.md | 2 +- 9 files changed, 87 insertions(+), 27 deletions(-) diff --git a/agent/copilot_acp_client.py b/agent/copilot_acp_client.py index ce3ec2c5c400..5e095af3902b 100644 --- a/agent/copilot_acp_client.py +++ b/agent/copilot_acp_client.py @@ -26,7 +26,7 @@ from openai.types.chat.chat_completion_message_tool_call import ( Function, ) -from agent.file_safety import get_read_block_error, is_write_denied +from agent.file_safety import get_read_block_error, get_write_denied_error from agent.redact import redact_sensitive_text from tools.environments.local import hermes_subprocess_env @@ -727,10 +727,9 @@ class CopilotACPClient: elif method == "fs/write_text_file": try: path = _ensure_path_within_cwd(str(params.get("path") or ""), cwd) - if is_write_denied(str(path)): - raise PermissionError( - f"Write denied: '{path}' is a protected system/credential file." - ) + denied = get_write_denied_error(str(path)) + if denied: + raise PermissionError(denied) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(str(params.get("content") or "")) response = { diff --git a/agent/file_safety.py b/agent/file_safety.py index d7e20ee5f0b9..2f4b3666e28d 100644 --- a/agent/file_safety.py +++ b/agent/file_safety.py @@ -95,16 +95,16 @@ def get_safe_write_roots() -> set[str]: return roots -def is_write_denied(path: str) -> bool: - """Return True if path is blocked by the write denylist or safe root.""" +def _classify_write_denial(path: str) -> Optional[str]: + """Return ``'credential'``, ``'safe_root'``, or ``None`` if writes are allowed.""" home = os.path.realpath(os.path.expanduser("~")) resolved = os.path.realpath(os.path.expanduser(str(path))) if resolved in build_write_denied_paths(home): - return True + return "credential" for prefix in build_write_denied_prefixes(home): if resolved.startswith(prefix): - return True + return "credential" mcp_tokens_dir_name = "mcp-tokens" @@ -121,13 +121,13 @@ def is_write_denied(path: str) -> bool: try: mcp_real = os.path.realpath(os.path.join(base_real, mcp_tokens_dir_name)) if resolved == mcp_real or resolved.startswith(mcp_real + os.sep): - return True + return "credential" except Exception: pass try: pairing_real = os.path.realpath(os.path.join(base_real, "pairing")) if resolved == pairing_real or resolved.startswith(pairing_real + os.sep): - return True + return "credential" except Exception: pass @@ -139,9 +139,28 @@ def is_write_denied(path: str) -> bool: allowed = True break if not allowed: - return True + return "safe_root" - return False + return None + + +def is_write_denied(path: str) -> bool: + """Return True if path is blocked by the write denylist or safe root.""" + return _classify_write_denial(path) is not None + + +def get_write_denied_error(path: str, *, verb: str = "Write") -> Optional[str]: + """Return a user/model-facing error when writes to ``path`` are blocked.""" + denial = _classify_write_denial(path) + if denial is None: + return None + if denial == "safe_root": + roots_display = os.pathsep.join(sorted(get_safe_write_roots())) + return ( + f"{verb} denied: '{path}' is outside HERMES_WRITE_SAFE_ROOT " + f"({roots_display}). Unset the variable or add this path's directory prefix." + ) + return f"{verb} denied: '{path}' is a protected system/credential file." # Common secret-bearing project-local environment file basenames. diff --git a/tests/agent/test_copilot_acp_client.py b/tests/agent/test_copilot_acp_client.py index 5f2d3c234fe3..1d1d28cc5421 100644 --- a/tests/agent/test_copilot_acp_client.py +++ b/tests/agent/test_copilot_acp_client.py @@ -209,7 +209,11 @@ class CopilotACPClientSafetyTests(unittest.TestCase): target = home / ".ssh" / "id_rsa" target.parent.mkdir(parents=True, exist_ok=True) - with patch("agent.copilot_acp_client.is_write_denied", return_value=True, create=True): + with patch( + "agent.copilot_acp_client.get_write_denied_error", + return_value="Write denied: protected", + create=True, + ): response = self._dispatch( { "jsonrpc": "2.0", @@ -248,6 +252,7 @@ class CopilotACPClientSafetyTests(unittest.TestCase): ) self.assertIn("error", response) + self.assertIn("HERMES_WRITE_SAFE_ROOT", str(response["error"])) self.assertFalse(outside.exists()) diff --git a/tests/tools/test_file_write_safety.py b/tests/tools/test_file_write_safety.py index ae766a7a723e..655ba407b652 100644 --- a/tests/tools/test_file_write_safety.py +++ b/tests/tools/test_file_write_safety.py @@ -168,6 +168,38 @@ class TestMultipleSafeWriteRoots: assert _is_write_denied(str(inside)) is False +class TestGetWriteDeniedError: + """get_write_denied_error() should distinguish credential vs safe-root blocks.""" + + def test_credential_path_message(self): + from agent.file_safety import get_write_denied_error + + err = get_write_denied_error(os.path.expanduser("~/.ssh/id_rsa")) + assert err is not None + assert "protected system/credential file" in err + assert "HERMES_WRITE_SAFE_ROOT" not in err + + def test_safe_root_message(self, tmp_path: Path, monkeypatch): + from agent.file_safety import get_write_denied_error + + safe_root = tmp_path / "workspace" + outside = tmp_path / "outside.txt" + os.makedirs(safe_root, exist_ok=True) + + monkeypatch.setenv("HERMES_WRITE_SAFE_ROOT", str(safe_root)) + err = get_write_denied_error(str(outside)) + assert err is not None + assert "outside HERMES_WRITE_SAFE_ROOT" in err + assert str(safe_root) in err + assert "protected system/credential file" not in err + + def test_allowed_path_returns_none(self, tmp_path: Path): + from agent.file_safety import get_write_denied_error + + target = tmp_path / "ok.txt" + assert get_write_denied_error(str(target)) is None + + class TestCheckSensitivePathMacOSBypass: """Verify _check_sensitive_path blocks /private/etc paths (issue #8734).""" diff --git a/tools/file_operations.py b/tools/file_operations.py index 76446befaa5e..994723cf583b 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -37,6 +37,7 @@ from tools.binary_extensions import BINARY_EXTENSIONS from agent.file_safety import ( build_write_denied_paths, build_write_denied_prefixes, + get_write_denied_error, is_write_denied as _shared_is_write_denied, ) @@ -1289,8 +1290,9 @@ class ShellFileOperations(FileOperations): def _python_delete(self, path: str, recursive: bool) -> WriteResult: path = self._expand_path(path) - if _is_write_denied(path): - return WriteResult(error=f"Delete denied: {path} is a protected path") + denied = get_write_denied_error(path, verb="Delete") + if denied: + return WriteResult(error=denied) # We can't shell out to ``rm`` here — it doesn't exist on Windows # ``cmd.exe`` or PowerShell, so this code path is what's left when @@ -1335,8 +1337,9 @@ class ShellFileOperations(FileOperations): src = self._expand_path(src) dst = self._expand_path(dst) for p in (src, dst): - if _is_write_denied(p): - return WriteResult(error=f"Move denied: {p} is a protected path") + denied = get_write_denied_error(p, verb="Move") + if denied: + return WriteResult(error=denied) result = self._exec( f"mv {self._escape_shell_arg(src)} {self._escape_shell_arg(dst)}" ) @@ -1383,8 +1386,9 @@ class ShellFileOperations(FileOperations): path = self._expand_path(path) # Block writes to sensitive paths - if _is_write_denied(path): - return WriteResult(error=f"Write denied: '{path}' is a protected system/credential file.") + denied = get_write_denied_error(path) + if denied: + return WriteResult(error=denied) # ── Fail-closed pre-write syntax gate ─────────────────────────── # Validate the CANDIDATE content BEFORE any bytes touch disk — @@ -1566,8 +1570,9 @@ class ShellFileOperations(FileOperations): path = self._expand_path(path) # Block writes to sensitive paths - if _is_write_denied(path): - return PatchResult(error=f"Write denied: '{path}' is a protected system/credential file.") + denied = get_write_denied_error(path) + if denied: + return PatchResult(error=denied) # Read current content read_cmd = f"cat {self._escape_shell_arg(path)} 2>/dev/null" diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 5e6f1fc0de0c..0a449252b634 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -766,7 +766,7 @@ When this variable is set, `write_file` and `patch` may only target paths inside The official Docker image sets `HERMES_WRITE_SAFE_ROOT=/opt/data` alongside `HERMES_HOME=/opt/data` so the agent cannot escape the mounted data volume. -**Do not add this to `~/.hermes/.env` unless you intend to sandbox writes.** A common mistake is pointing it at a project directory while expecting the agent to edit `~/.hermes/cron/jobs.json`, `~/.hermes/skills/`, or scripts under a profile — those paths are outside the sandbox and every `write_file`/`patch` to them fails. The tool error currently reads `Write denied: '…' is a protected system/credential file.` for all blocked writes (including safe-root violations); check `echo $HERMES_WRITE_SAFE_ROOT` when you see that message. +**Do not add this to `~/.hermes/.env` unless you intend to sandbox writes.** A common mistake is pointing it at a project directory while expecting the agent to edit `~/.hermes/cron/jobs.json`, `~/.hermes/skills/`, or scripts under a profile — those paths are outside the sandbox and every `write_file`/`patch` to them fails with an `outside HERMES_WRITE_SAFE_ROOT` error. To allow both a workspace and Hermes state, list both prefixes (order does not matter): diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 1d2e3b672979..40748ce23197 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1457,8 +1457,8 @@ Example footer when writes are blocked: ``` ⚠️ File-mutation verifier: 2 file(s) were NOT modified this turn despite any wording above that may suggest otherwise. Run `git status` or `read_file` to confirm. - • ~/.hermes/cron/jobs.json — [patch] Write denied: '…' is a protected system/credential file. - • ~/.hermes/scripts/monitor.py — [write_file] Write denied: '…' is a protected system/credential file. + • ~/.hermes/cron/jobs.json — [patch] Write denied: '…' is outside HERMES_WRITE_SAFE_ROOT (/path/to/project) + • ~/.hermes/scripts/monitor.py — [write_file] Write denied: '…' is outside HERMES_WRITE_SAFE_ROOT (/path/to/project) ``` If writes to Hermes state (cron jobs, skills, scripts under `~/.hermes/`) are failing, check whether `HERMES_WRITE_SAFE_ROOT` is set in your environment. For cron changes, use the `cronjob` tool or `hermes cron edit` instead of patching `jobs.json` directly. diff --git a/website/docs/user-guide/security.md b/website/docs/user-guide/security.md index d2dd8306512b..a5488b6250c8 100644 --- a/website/docs/user-guide/security.md +++ b/website/docs/user-guide/security.md @@ -249,7 +249,7 @@ These categories are always denied, even when `HERMES_WRITE_SAFE_ROOT` is unset: Sensitive paths inside the safe root are still blocked — pointing `HERMES_WRITE_SAFE_ROOT` at `$HOME` does not allow writing `~/.ssh/id_rsa`. -The tool error message is always `Write denied: '…' is a protected system/credential file.` even when the actual reason is the safe-root sandbox below. If the path does not look like a credential file, check `echo $HERMES_WRITE_SAFE_ROOT`. +Safe-root violations return `Write denied: '…' is outside HERMES_WRITE_SAFE_ROOT (…)`. Credential-path blocks use `Write denied: '…' is a protected system/credential file.` ### HERMES_WRITE_SAFE_ROOT (optional sandbox) diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md index c405e6312de5..4b15a846f147 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/environment-variables.md @@ -580,7 +580,7 @@ Graph 事件（Teams 会议、日历、聊天等）的入站变更通知监听 官方 Docker 镜像会设置 `HERMES_WRITE_SAFE_ROOT=/opt/data` 与 `HERMES_HOME=/opt/data`，防止 agent 逃出挂载的数据卷。 -**除非有意沙箱化写入，否则不要将此变量加入 `~/.hermes/.env`。** 常见错误是将其指向项目目录，却期望 agent 编辑 `~/.hermes/cron/jobs.json`、`~/.hermes/skills/` 或 profile 下的脚本——这些路径在沙箱外，每次 `write_file`/`patch` 都会失败。当前所有被阻止的写入（包括 safe-root 违规）都显示 `Write denied: '…' is a protected system/credential file.`；看到此消息时请检查 `echo $HERMES_WRITE_SAFE_ROOT`。 +**除非有意沙箱化写入，否则不要将此变量加入 `~/.hermes/.env`。** 常见错误是将其指向项目目录，却期望 agent 编辑 `~/.hermes/cron/jobs.json`、`~/.hermes/skills/` 或 profile 下的脚本——这些路径在沙箱外，每次 `write_file`/`patch` 都会失败并返回 `outside HERMES_WRITE_SAFE_ROOT` 错误。 若需同时允许工作区和 Hermes 状态目录，列出两个前缀（顺序无关）： From 320e886f3dd5b432bc85fcc651076ea5addec36a Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:26:24 +0530 Subject: [PATCH 060/149] test(file-safety): add integration tests for safe-root denial messages Exercises the actual ShellFileOperations.write_file and patch_replace code paths (not just the helper in isolation) to verify that safe-root denials surface 'outside HERMES_WRITE_SAFE_ROOT' and credential-path denials surface 'protected system/credential file'. Adapted from PR #55615 by @liuhao1024. --- tests/tools/test_file_write_safety.py | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/tools/test_file_write_safety.py b/tests/tools/test_file_write_safety.py index 655ba407b652..7cf5c0468f6d 100644 --- a/tests/tools/test_file_write_safety.py +++ b/tests/tools/test_file_write_safety.py @@ -200,6 +200,75 @@ class TestGetWriteDeniedError: assert get_write_denied_error(str(target)) is None +class TestSafeRootDenialMessageIntegration: + """Regression tests verifying that file-tools surface the correct denial + message when HERMES_WRITE_SAFE_ROOT blocks a path. + + Prior to this fix, ALL write denials returned the same "protected + system/credential file" message regardless of root cause. These tests + exercise the actual write_file / patch_replace code path, not just + the get_write_denied_error() helper in isolation. + """ + + @pytest.fixture + def ops(self, tmp_path: Path): + from tools.environments.local import LocalEnvironment + from tools.file_operations import ShellFileOperations + env = LocalEnvironment(cwd=str(tmp_path)) + return ShellFileOperations(env, cwd=str(tmp_path)) + + def test_write_file_safe_root_outside_shows_safe_root_message( + self, ops, tmp_path: Path, monkeypatch + ): + safe_root = tmp_path / "workspace" + safe_root.mkdir() + outside = tmp_path / "other" / "file.txt" + outside.parent.mkdir() + monkeypatch.setenv("HERMES_WRITE_SAFE_ROOT", str(safe_root)) + + res = ops.write_file(str(outside), "content") + assert res.error is not None + assert "outside HERMES_WRITE_SAFE_ROOT" in res.error + assert str(safe_root) in res.error + assert "credential" not in res.error + assert not outside.exists() + + def test_patch_replace_safe_root_outside_shows_safe_root_message( + self, ops, tmp_path: Path, monkeypatch + ): + safe_root = tmp_path / "workspace" + safe_root.mkdir() + outside = tmp_path / "other" / "file.txt" + outside.parent.mkdir() + outside.write_text("old content") + monkeypatch.setenv("HERMES_WRITE_SAFE_ROOT", str(safe_root)) + + res = ops.patch_replace(str(outside), "old", "new") + assert res.error is not None + assert "outside HERMES_WRITE_SAFE_ROOT" in res.error + assert "credential" not in res.error + + def test_write_file_credential_path_shows_credential_message( + self, ops, tmp_path: Path + ): + res = ops.write_file("/etc/shadow", "content") + assert res.error is not None + assert "protected system/credential file" in res.error + assert "outside" not in res.error + + def test_write_file_allowed_path_returns_no_error( + self, ops, tmp_path: Path, monkeypatch + ): + safe_root = tmp_path / "workspace" + safe_root.mkdir() + inside = safe_root / "file.txt" + monkeypatch.setenv("HERMES_WRITE_SAFE_ROOT", str(safe_root)) + + res = ops.write_file(str(inside), "content") + assert res.error is None + assert inside.read_text() == "content" + + class TestCheckSensitivePathMacOSBypass: """Verify _check_sensitive_path blocks /private/etc paths (issue #8734).""" From e16743b0d5d1899785a5dfa87d4f5f8d8e503226 Mon Sep 17 00:00:00 2001 From: Jaret Bottoms <25535214+jbbottoms@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:15:21 -0500 Subject: [PATCH 061/149] fix(telegram): diagnose blocked-loop init hangs, unbind DoH from system DNS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #63309 hang class — gateway stuck at 'Connecting to Telegram (attempt 1/8)' with no retry, no timeout, for minutes — can only occur when the event loop thread itself is blocked in a synchronous call: _await_with_thread_deadline's timer fires off-loop, but its expiry hand-off (call_soon_threadsafe) still needs the loop to run, and the gateway's outer wait_for is a pure loop timer. When the loop is pinned, every layer goes silent simultaneously and the process wedges with no evidence of where. Two changes: 1. Loop-blocked watchdog in _await_with_thread_deadline: a second daemon timer fires one grace period (5s) after the deadline; if the loop still hasn't processed the expiry, it logs a WARNING from the timer thread and faulthandler-dumps all thread stacks to stderr — converting the silent hang into a trace that names the exact blocking frame. A threading.Event set by the expiry callback (and on normal exit) keeps completed awaits from ever being misreported. 2. discover_fallback_ips: the system-resolver leg runs socket.getaddrinfo in a worker thread with no timeout, and asyncio.gather waited on it unboundedly — a wedged OS resolver stalled discovery for minutes between the two startup log lines. Its result only feeds a log message, so it no longer gates discovery: DoH legs (already client-bounded) are gathered alone and the system leg is awaited with a _DOH_TIMEOUT cap, best-effort. Refs #63309 Tests: 3 watchdog regressions (blocked-loop dump fires; responsive-loop timeout does not; completed await does not) + 2 hung-resolver regressions (DoH results returned promptly; worst-case seed fallback stays bounded). --- plugins/platforms/telegram/adapter.py | 52 +++++++++++++ .../platforms/telegram/telegram_network.py | 20 +++-- tests/gateway/test_telegram_init_deadline.py | 76 +++++++++++++++++++ tests/gateway/test_telegram_network.py | 51 +++++++++++++ 4 files changed, 194 insertions(+), 5 deletions(-) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index 7b959b1edf61..05506c79da48 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -9,6 +9,7 @@ Uses python-telegram-bot library for: import asyncio import dataclasses +import faulthandler import inspect import json import logging @@ -45,6 +46,35 @@ def _consume_abandoned_task(task: asyncio.Task) -> None: logger.debug("Abandoned Telegram init task failed after timeout", exc_info=True) +# Grace period after the wall-clock deadline fires: if the event loop still +# hasn't processed the expiry callback by then, the loop thread itself is +# blocked in a synchronous call — the exact state in which every asyncio-based +# timeout (including this helper's own expiry hand-off) goes silent, so the +# gateway hangs at "attempt 1/8" with no further output (#63309). +_LOOP_BLOCKED_DUMP_GRACE = 5.0 + + +def _dump_loop_blocked_diagnostics(timeout: float, grace: float) -> None: + """Emit diagnostics from the deadline timer thread when the loop is stuck. + + Runs OFF the event loop, so it works precisely when the loop cannot. The + faulthandler dump names the frame the loop thread is blocked in — the one + piece of information #63309-class hangs otherwise never surface. + """ + logger.warning( + "[Telegram] init deadline (%.0fs) expired but the event loop has not " + "processed the expiry after a further %.0fs — the loop thread appears " + "BLOCKED in a synchronous call, which is why no timeout fires (#63309). " + "Dumping all thread stacks to stderr to identify the blocking frame.", + timeout, + grace, + ) + try: + faulthandler.dump_traceback(all_threads=True) + except Exception: + logger.debug("faulthandler traceback dump failed", exc_info=True) + + async def _await_with_thread_deadline(awaitable, timeout: float, *, on_abandon=None): """Await with a wall-clock deadline that does not depend on loop timers. @@ -66,17 +96,34 @@ async def _await_with_thread_deadline(awaitable, timeout: float, *, on_abandon=N task = asyncio.ensure_future(awaitable) loop = asyncio.get_running_loop() deadline = loop.create_future() + # Set the moment the loop actually runs the expiry callback (or the helper + # exits normally). threading.Event so the watchdog thread can read it + # without touching asyncio state from off-loop. + loop_processed_expiry = threading.Event() def _mark_expired() -> None: + loop_processed_expiry.set() if not deadline.done(): deadline.set_result(None) def _expire_from_thread() -> None: loop.call_soon_threadsafe(_mark_expired) + def _watchdog_check() -> None: + # The deadline fired _LOOP_BLOCKED_DUMP_GRACE ago but the loop never + # ran _mark_expired: the loop thread is stuck in a synchronous call. + # Diagnose from this thread — the loop can't. + if not loop_processed_expiry.is_set(): + _dump_loop_blocked_diagnostics(timeout, _LOOP_BLOCKED_DUMP_GRACE) + timer = threading.Timer(max(timeout, 0.0), _expire_from_thread) timer.daemon = True timer.start() + watchdog = threading.Timer( + max(timeout, 0.0) + _LOOP_BLOCKED_DUMP_GRACE, _watchdog_check + ) + watchdog.daemon = True + watchdog.start() try: done, _ = await asyncio.wait( {task, deadline}, @@ -99,6 +146,11 @@ async def _await_with_thread_deadline(awaitable, timeout: float, *, on_abandon=N raise asyncio.TimeoutError() finally: timer.cancel() + watchdog.cancel() + # cancel() cannot stop a Timer whose callback is already running; + # setting the event closes that race so a completed await can never + # be misreported as a blocked loop. + loop_processed_expiry.set() async def _run_abandon_cleanup(on_abandon) -> None: diff --git a/plugins/platforms/telegram/telegram_network.py b/plugins/platforms/telegram/telegram_network.py index 319f4d2accf8..a0fd14ebb5d3 100644 --- a/plugins/platforms/telegram/telegram_network.py +++ b/plugins/platforms/telegram/telegram_network.py @@ -205,14 +205,24 @@ async def discover_fallback_ips() -> list[str]: """ async with httpx.AsyncClient(timeout=httpx.Timeout(_DOH_TIMEOUT)) as client: doh_tasks = [_query_doh_provider(client, p) for p in _DOH_PROVIDERS] - system_dns_task = asyncio.to_thread(_resolve_system_dns) - results = await asyncio.gather(system_dns_task, *doh_tasks, return_exceptions=True) + system_dns_task = asyncio.ensure_future(asyncio.to_thread(_resolve_system_dns)) + results = await asyncio.gather(*doh_tasks, return_exceptions=True) - # results[0] = system DNS IPs (set), results[1:] = DoH IP lists - system_ips: set[str] = results[0] if isinstance(results[0], set) else set() + # The system-resolver leg runs socket.getaddrinfo in a worker thread with + # no timeout of its own — a wedged OS resolver (broken VPN/DNS) can sit for + # minutes. Its result only feeds the no-usable-answers log line below, so + # it must never gate discovery: bound it and move on (#63309). The DoH legs + # are already bounded by the client timeout above. + system_ips: set[str] = set() + try: + system_result = await asyncio.wait_for(system_dns_task, timeout=_DOH_TIMEOUT) + if isinstance(system_result, set): + system_ips = system_result + except Exception: + logger.debug("System-DNS resolution for %s did not complete in time", _TELEGRAM_API_HOST) doh_ips: list[str] = [] - for r in results[1:]: + for r in results: if isinstance(r, list): doh_ips.extend(r) diff --git a/tests/gateway/test_telegram_init_deadline.py b/tests/gateway/test_telegram_init_deadline.py index ca697276e8b5..5e3d045ba48f 100644 --- a/tests/gateway/test_telegram_init_deadline.py +++ b/tests/gateway/test_telegram_init_deadline.py @@ -199,3 +199,79 @@ async def test_shutdown_abandoned_app_handles_none_and_missing_requests(): app.shutdown = AsyncMock(side_effect=RuntimeError("still running")) app.bot = None await tg_adapter._shutdown_abandoned_app(app) # must not raise + + +@pytest.mark.asyncio +async def test_blocked_loop_after_expiry_dumps_diagnostics(monkeypatch): + """#63309: when the loop thread is stuck in a synchronous call, the expiry + callback never runs and every asyncio timeout goes silent. The off-loop + watchdog must detect that state and emit diagnostics from its own thread.""" + import asyncio as _asyncio + import time as _time + + dumps = [] + monkeypatch.setattr( + tg_adapter, + "_dump_loop_blocked_diagnostics", + lambda timeout, grace: dumps.append((timeout, grace)), + ) + monkeypatch.setattr(tg_adapter, "_LOOP_BLOCKED_DUMP_GRACE", 0.15) + + hung = _asyncio.get_running_loop().create_future() # never completes + task = _asyncio.ensure_future( + tg_adapter._await_with_thread_deadline(hung, timeout=0.05) + ) + # Let the helper start its deadline + watchdog timers… + await _asyncio.sleep(0) + # …then block the event loop straight through deadline (0.05s) AND the + # watchdog grace (0.15s): call_soon_threadsafe stays queued, exactly like + # a sync call pinning the loop during Application.initialize(). + _time.sleep(0.5) + with pytest.raises(_asyncio.TimeoutError): + await task + + assert dumps == [(0.05, 0.15)] + hung.cancel() + + +@pytest.mark.asyncio +async def test_responsive_loop_expiry_does_not_dump(monkeypatch): + """A normal timeout on a responsive loop must not trigger the watchdog.""" + import asyncio as _asyncio + + dumps = [] + monkeypatch.setattr( + tg_adapter, + "_dump_loop_blocked_diagnostics", + lambda timeout, grace: dumps.append((timeout, grace)), + ) + monkeypatch.setattr(tg_adapter, "_LOOP_BLOCKED_DUMP_GRACE", 0.1) + + hung = _asyncio.get_running_loop().create_future() + with pytest.raises(_asyncio.TimeoutError): + await tg_adapter._await_with_thread_deadline(hung, timeout=0.05) + # Give the (cancelled) watchdog window time to have fired if it were going to. + await _asyncio.sleep(0.3) + assert dumps == [] + hung.cancel() + + +@pytest.mark.asyncio +async def test_completed_await_never_reports_blocked_loop(monkeypatch): + """Success before the deadline must cancel the watchdog (no false dump).""" + import asyncio as _asyncio + + dumps = [] + monkeypatch.setattr( + tg_adapter, + "_dump_loop_blocked_diagnostics", + lambda timeout, grace: dumps.append((timeout, grace)), + ) + monkeypatch.setattr(tg_adapter, "_LOOP_BLOCKED_DUMP_GRACE", 0.05) + + async def _quick(): + return "ok" + + assert await tg_adapter._await_with_thread_deadline(_quick(), timeout=0.2) == "ok" + await _asyncio.sleep(0.4) + assert dumps == [] diff --git a/tests/gateway/test_telegram_network.py b/tests/gateway/test_telegram_network.py index cede492fe5bd..8406973b9483 100644 --- a/tests/gateway/test_telegram_network.py +++ b/tests/gateway/test_telegram_network.py @@ -705,3 +705,54 @@ class TestDiscoverFallbackIps: ips = await tnet.discover_fallback_ips() assert ips == ["149.154.167.220"] + + @pytest.mark.asyncio + async def test_hung_system_dns_does_not_gate_doh_results(self, monkeypatch): + """#63309: socket.getaddrinfo has no timeout of its own — a wedged OS + resolver must not stall discovery. DoH answers must come back promptly + even while the system-DNS worker thread is still hanging.""" + import time as _time + + self._patch_doh(monkeypatch, { + "https://dns.google": (200, _doh_answer("149.154.167.220")), + "https://cloudflare-dns.com": (200, _doh_answer()), + }, system_dns_ips=["149.154.166.110"]) + monkeypatch.setattr(tnet, "_DOH_TIMEOUT", 0.2) + + def _hung_getaddrinfo(*a, **kw): + _time.sleep(1.5) # far beyond the discovery bound + raise OSError("resolver wedged") + + monkeypatch.setattr(tnet.socket, "getaddrinfo", _hung_getaddrinfo) + + start = _time.monotonic() + ips = await tnet.discover_fallback_ips() + elapsed = _time.monotonic() - start + + assert ips == ["149.154.167.220"] + assert elapsed < 1.0, f"discovery gated on hung system DNS ({elapsed:.2f}s)" + + @pytest.mark.asyncio + async def test_hung_system_dns_with_no_doh_answers_bounded_seed_fallback(self, monkeypatch): + """Worst case — resolver wedged AND no DoH answers — must still return + the seed list within the bound instead of hanging connect().""" + import time as _time + + self._patch_doh(monkeypatch, { + "https://dns.google": (200, {"Status": 0}), + "https://cloudflare-dns.com": (200, {"garbage": True}), + }, system_dns_ips=["149.154.166.110"]) + monkeypatch.setattr(tnet, "_DOH_TIMEOUT", 0.2) + + def _hung_getaddrinfo(*a, **kw): + _time.sleep(1.5) + raise OSError("resolver wedged") + + monkeypatch.setattr(tnet.socket, "getaddrinfo", _hung_getaddrinfo) + + start = _time.monotonic() + ips = await tnet.discover_fallback_ips() + elapsed = _time.monotonic() - start + + assert ips == tnet._SEED_FALLBACK_IPS + assert elapsed < 1.0, f"seed fallback gated on hung system DNS ({elapsed:.2f}s)" From cd537187611769ebb6a1aa9460265e9ef5694606 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 12 Jul 2026 20:23:04 +0000 Subject: [PATCH 062/149] fix(cron): prevent long-running scheduled scripts from running twice --- cron/scheduler.py | 73 +++++++++- tests/cron/test_script_claim_heartbeat.py | 165 ++++++++++++++++++++++ 2 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 tests/cron/test_script_claim_heartbeat.py diff --git a/cron/scheduler.py b/cron/scheduler.py index 9d8dc7507757..a50704ea10be 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -1976,6 +1976,7 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option _DEFAULT_SCRIPT_TIMEOUT = 3600 # seconds (1 hour) # Backward-compatible module override used by tests and emergency monkeypatches. _SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT +_RUN_CLAIM_HEARTBEAT_SECONDS = 60.0 def _get_script_timeout() -> int: @@ -2135,6 +2136,71 @@ def _run_job_script(script_path: str) -> tuple[bool, str]: return False, f"Script execution failed: {exc}" +def _run_job_script_with_claim_heartbeat( + job: dict, script_path: str +) -> tuple[bool, str]: + """Run a cron script while keeping its owned one-shot claim fresh. + + Script execution is synchronous and may legitimately outlive the stale + claim TTL. Without a concurrent heartbeat, another scheduler process can + mistake the live run for a dead owner and dispatch the same one-shot again. + Recurring jobs and unclaimed/manual runs have no durable one-shot claim and + therefore use the ordinary script path without starting a thread. + + The claim owner is captured from the dispatched job and never re-read from + storage. ``heartbeat_run_claim`` compares that stable owner before every + refresh, so a stale runner cannot extend a replacement owner's claim. + """ + schedule = job.get("schedule") + claim = job.get("run_claim") + owner = str(claim.get("by") or "") if isinstance(claim, dict) else "" + if not ( + isinstance(schedule, dict) + and schedule.get("kind") == "once" + and owner + ): + return _run_job_script(script_path) + + job_id = str(job.get("id") or "") + stop = threading.Event() + heartbeat_context = contextvars.copy_context() + + def _heartbeat_loop() -> None: + while not stop.wait(_RUN_CLAIM_HEARTBEAT_SECONDS): + try: + heartbeat_run_claim(job_id, expected_owner=owner) + except Exception: + logger.debug( + "Job '%s': script run_claim heartbeat failed", + job_id, + exc_info=True, + ) + + heartbeat_thread = threading.Thread( + target=heartbeat_context.run, + args=(_heartbeat_loop,), + name="cron-script-claim-heartbeat", + daemon=True, + ) + try: + heartbeat_thread.start() + except Exception: + logger.debug( + "Job '%s': could not start script run_claim heartbeat", + job_id, + exc_info=True, + ) + return _run_job_script(script_path) + + try: + return _run_job_script(script_path) + finally: + stop.set() + # Event.wait() wakes immediately. Keep completion bounded if the + # heartbeat is already waiting on another process's jobs-file lock. + heartbeat_thread.join(timeout=1.0) + + def _parse_wake_gate(script_output: str) -> bool: """Parse the last non-empty stdout line of a cron job's pre-check script as a wake gate. @@ -2542,7 +2608,7 @@ def run_job( _prior_cwd = None try: - ok, output = _run_job_script(script_path) + ok, output = _run_job_script_with_claim_heartbeat(job, script_path) finally: if _prior_cwd is not None: try: @@ -2689,7 +2755,7 @@ def run_job( prerun_script = None script_path = job.get("script") if script_path: - prerun_script = _run_job_script(script_path) + prerun_script = _run_job_script_with_claim_heartbeat(job, script_path) _ran_ok, _script_output = prerun_script if _ran_ok and not _parse_wake_gate(_script_output): logger.info( @@ -3171,7 +3237,6 @@ def run_job( _run_claim_owner = ( str(_run_claim.get("by") or "") if isinstance(_run_claim, dict) else "" ) - _CLAIM_HEARTBEAT_SECONDS = 60.0 _last_claim_heartbeat = time.monotonic() def _heartbeat_run_claim_if_due(): @@ -3179,7 +3244,7 @@ def run_job( if not _is_oneshot or not _run_claim_owner: return _mono = time.monotonic() - if _mono - _last_claim_heartbeat < _CLAIM_HEARTBEAT_SECONDS: + if _mono - _last_claim_heartbeat < _RUN_CLAIM_HEARTBEAT_SECONDS: return _last_claim_heartbeat = _mono try: diff --git a/tests/cron/test_script_claim_heartbeat.py b/tests/cron/test_script_claim_heartbeat.py new file mode 100644 index 000000000000..0f4c3fd863c9 --- /dev/null +++ b/tests/cron/test_script_claim_heartbeat.py @@ -0,0 +1,165 @@ +"""Regression coverage for one-shot claims during blocking cron scripts.""" + +from datetime import datetime, timedelta, timezone +import threading +from unittest.mock import MagicMock, patch + +import pytest + + +@pytest.mark.parametrize( + ("no_agent", "script_output"), + [ + (True, "watchdog complete"), + (False, '{"wakeAgent": false}'), + ], + ids=("script-only-job", "pre-agent-script"), +) +def test_long_running_script_refreshes_owned_claim_in_profile_store( + tmp_path, monkeypatch, no_agent, script_output +): + """Both blocking script paths keep their one-shot claim alive. + + The real store update runs on the heartbeat thread. A second store holds + the same job ID, proving the thread inherited the active profile's + ContextVar instead of falling back to another profile's default paths. + """ + import cron.jobs as jobs + import cron.scheduler as scheduler + + profile_home = tmp_path / "profile" + default_cron = tmp_path / "default" / "cron" + default_cron.mkdir(parents=True) + profile_home.mkdir() + + monkeypatch.setattr(jobs, "CRON_DIR", default_cron) + monkeypatch.setattr(jobs, "JOBS_FILE", default_cron / "jobs.json") + monkeypatch.setattr(jobs, "OUTPUT_DIR", default_cron / "output") + monkeypatch.setattr(scheduler, "_RUN_CLAIM_HEARTBEAT_SECONDS", 0.01) + + original_timestamp = "2026-07-12T12:00:00+00:00" + original_time = datetime.fromisoformat(original_timestamp) + claim_ttl = jobs._oneshot_run_claim_ttl_seconds() + current_time = [original_time + timedelta(seconds=claim_ttl - 60)] + monkeypatch.setattr(jobs, "_hermes_now", lambda: current_time[0]) + + def _job() -> dict: + return { + "id": "long-script", + "name": "long script", + "prompt": "inspect the script output", + "script": "watchdog.py", + "no_agent": no_agent, + "schedule": { + "kind": "once", + "run_at": original_timestamp, + }, + "next_run_at": original_timestamp, + "enabled": True, + "run_claim": { + "at": original_timestamp, + "by": "dispatch-owner", + }, + } + + # Safe fallback store: if ContextVars are not propagated to the heartbeat + # thread, this record would be modified instead of the profile record. + jobs.save_jobs([_job()]) + with jobs.use_cron_store(profile_home): + jobs.save_jobs([_job()]) + claimed_job = jobs.get_job("long-script") + + heartbeat_seen = threading.Event() + real_heartbeat = jobs.heartbeat_run_claim + second_scheduler_scan = {} + + def _observed_heartbeat(job_id: str, *, expected_owner: str) -> bool: + updated = real_heartbeat(job_id, expected_owner=expected_owner) + # A different scheduler scans after the ORIGINAL claim's TTL while the + # script is still blocked. The refreshed claim must keep the job out of + # the due set and preserve its durable record. + current_time[0] = original_time + timedelta(seconds=claim_ttl + 10) + second_scheduler_scan["due"] = jobs.get_due_jobs() + second_scheduler_scan["record_present"] = jobs.get_job(job_id) is not None + heartbeat_seen.set() + return updated + + def _blocking_script(_script_path: str) -> tuple[bool, str]: + assert heartbeat_seen.wait(timeout=2), ( + "claim was not refreshed while script blocked" + ) + return True, script_output + + monkeypatch.setattr(scheduler, "heartbeat_run_claim", _observed_heartbeat) + monkeypatch.setattr(scheduler, "_run_job_script", _blocking_script) + + with ( + jobs.use_cron_store(profile_home), + patch("hermes_state.SessionDB", return_value=MagicMock()), + ): + success, _doc, _response, error = scheduler.run_job(claimed_job) + profile_claim = jobs.get_job("long-script")["run_claim"] + + assert success is True + assert error is None + assert profile_claim["at"] != original_timestamp + assert profile_claim["by"] == "dispatch-owner" + assert second_scheduler_scan == {"due": [], "record_present": True} + assert jobs.get_job("long-script")["run_claim"] == { + "at": original_timestamp, + "by": "dispatch-owner", + } + + +def test_script_heartbeat_uses_captured_claim_owner(tmp_path, monkeypatch): + """A stale script runner cannot refresh a replacement owner's claim.""" + import cron.jobs as jobs + import cron.scheduler as scheduler + + profile_home = tmp_path / "profile" + profile_home.mkdir() + original_timestamp = "2026-07-12T12:00:00+00:00" + replacement_timestamp = "2026-07-12T12:00:30+00:00" + job = { + "id": "reclaimed-script", + "script": "watchdog.py", + "schedule": {"kind": "once", "run_at": original_timestamp}, + "run_claim": {"at": original_timestamp, "by": "original-owner"}, + } + + with jobs.use_cron_store(profile_home): + jobs.save_jobs([ + { + **job, + "run_claim": { + "at": replacement_timestamp, + "by": "replacement-owner", + }, + } + ]) + + heartbeat_seen = threading.Event() + real_heartbeat = jobs.heartbeat_run_claim + + def _observed_heartbeat(job_id: str, *, expected_owner: str) -> bool: + updated = real_heartbeat(job_id, expected_owner=expected_owner) + heartbeat_seen.set() + return updated + + def _blocking_script(_script_path: str) -> tuple[bool, str]: + assert heartbeat_seen.wait(timeout=2) + return True, "done" + + monkeypatch.setattr(scheduler, "_RUN_CLAIM_HEARTBEAT_SECONDS", 0.01) + monkeypatch.setattr(scheduler, "heartbeat_run_claim", _observed_heartbeat) + monkeypatch.setattr(scheduler, "_run_job_script", _blocking_script) + + with jobs.use_cron_store(profile_home): + assert scheduler._run_job_script_with_claim_heartbeat(job, "watchdog.py") == ( + True, + "done", + ) + assert jobs.get_job("reclaimed-script")["run_claim"] == { + "at": replacement_timestamp, + "by": "replacement-owner", + } From ca559a78523e9370bc2b46e689c5b7bb8ceb36a1 Mon Sep 17 00:00:00 2001 From: Rory Ford Date: Fri, 12 Jun 2026 21:04:41 +1000 Subject: [PATCH 063/149] fix(gateway): never prune sessions when active-process check fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prune_old_entries' active-process guard failed open: when has_active_processes_fn raised, the except block logged at debug and fell through to the age check, so sessions with live background processes attached could still be pruned — violating the documented invariant that such sessions are never dropped. Add a continue so an exception in the safety check fails safe (the entry is kept). Commit 6b408e131 fixed the session_key/session_id mismatch in this same guard but left the exception path failing open. Co-Authored-By: Claude Fable 5 --- gateway/session.py | 4 +++ tests/gateway/test_session_store_prune.py | 31 +++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/gateway/session.py b/gateway/session.py index f04090445e75..962d186c3513 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -2179,6 +2179,10 @@ class SessionStore: "has_active_processes_fn raised during prune for %s: %s", entry.session_key, exc, ) + # Fail safe: if we can't tell whether a background + # process is attached, keep the entry rather than + # risk orphaning live work. + continue if entry.updated_at < cutoff: removed_keys.append(key) for key in removed_keys: diff --git a/tests/gateway/test_session_store_prune.py b/tests/gateway/test_session_store_prune.py index b331d305e375..888d3a9a0a5d 100644 --- a/tests/gateway/test_session_store_prune.py +++ b/tests/gateway/test_session_store_prune.py @@ -165,6 +165,37 @@ class TestPruneBasics: assert removed == 1 assert "active" not in store._entries + def test_prune_keeps_entry_when_active_check_raises(self, tmp_path): + """A failing active-process check must fail safe, not fail open. + + If has_active_processes_fn raises, we can't tell whether a live + background process is attached — so the entry must be kept. + Previously the except block logged and fell through to the age + check, pruning the session anyway. + """ + def _broken(session_key: str) -> bool: + raise RuntimeError("process registry unavailable") + + store = _make_store(tmp_path, has_active_processes_fn=_broken) + store._entries["old"] = _entry("old", age_days=1000) + + removed = store.prune_old_entries(max_age_days=90) + + assert removed == 0 + assert "old" in store._entries + + def test_prune_removes_old_entry_when_active_check_returns_false(self, tmp_path): + """Sibling guard: a callback that cleanly reports no active process + must still allow the old entry to be pruned. + """ + store = _make_store(tmp_path, has_active_processes_fn=lambda key: False) + store._entries["old"] = _entry("old", age_days=1000) + + removed = store.prune_old_entries(max_age_days=90) + + assert removed == 1 + assert "old" not in store._entries + def test_prune_does_not_write_disk_when_no_removals(self, tmp_path): """If nothing is evictable, _save() should NOT be called.""" store = _make_store(tmp_path) From 17cfa0f0a543058b8ade8d4467338e524179b0bd Mon Sep 17 00:00:00 2001 From: Ziliang Peng Date: Fri, 22 May 2026 10:09:17 -0700 Subject: [PATCH 064/149] fix(background_review): inherit parent's reasoning_config to preserve Anthropic cache namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #17276 painstakingly pinned `_cached_system_prompt`, `session_start`, `session_id`, and the toolset config on the background-review fork so its outbound request body would byte-match the parent's and hit Anthropic's exact-prefix cache. The contributor measured a ~26% end-to-end cost reduction on Sonnet 4.5. That optimization is currently being silently undone by a missing `reasoning_config` kwarg. The fork's `AIAgent(...)` call omits it, so the fork's `reasoning_config` defaults to `None`. `anthropic_adapter.build_anthropic_kwargs` (line ~2165) then short-circuits the `thinking` / `output_config` block, and the fork's request body lands in a DIFFERENT Anthropic cache namespace from the parent's. Result on the wire: 0 `cache_read_input_tokens`, full `cache_creation_input_tokens` of the entire parent prefix — every single background review. 7 days of midagent.db traffic from one host running stock Hermes against Anthropic Sonnet: ``` Background-review FIRST calls (the moment a review fork is born): count = 68 cache_write tokens = 7,004,297 cache_read tokens = 1,016,335 Cost on Sonnet ($3.75/M write vs $0.30/M read): Spent on these writes: $26.27 Cost if they had hit parent cache instead: $2.10 WASTED: $24.16 / week / user ``` That is from one user. Multiply by Hermes's installed base for the full impact. Tested against api.anthropic.com directly (see refs/api-tests/ in the attached investigation repo if needed): | pair | cache_r | cache_w | |---------------------------------------------|---------|---------| | parent fresh | 0 | 24,047 | | parent same again | 24,047 | 0 | | fork: appends 2 new tail msgs, thinking ON | 24,047 | 22 | | fork: appends 2 new tail msgs, thinking OFF | 0 | 24,047 | Same fork-shape request, only difference is `thinking`. With the fix, the fork hits the parent's full prefix and only writes the delta (the `Review the conversation above…` prompt block, ~3-5K tokens). One line in `agent/background_review.py`: pass `reasoning_config=getattr(agent, "reasoning_config", None)` to the `AIAgent(...)` constructor of the review fork. A short comment block above it explains why so the next person who reads this code doesn't re-introduce the regression. `tests/run_agent/test_background_review_cache_parity.py` already covers the system-prompt / session-id / toolset-config parity contracts that PR #17276 introduced. I added: * a `reasoning_config` attribute to `_make_agent_stub` so the stub has a non-None parent value the test can verify is propagated. * `test_review_fork_inherits_parent_reasoning_config()` — asserts the fork's `AIAgent(...)` kwargs carry the parent's `reasoning_config`. Pre-fix this test fails with `None vs expected {'enabled': True, 'effort': 'medium'}`; post-fix all 4 tests in the file pass. ``` $ python -m pytest tests/run_agent/test_background_review_cache_parity.py -v test_review_fork_inherits_parent_cached_system_prompt PASSED test_review_fork_pins_session_start_and_session_id PASSED test_review_fork_inherits_parent_toolset_config PASSED test_review_fork_inherits_parent_reasoning_config PASSED ← new ``` Also runs against the broader background-review test suite: `test_background_review.py` (4), `test_background_review_summary.py` (8), `test_background_review_toolset_restriction.py` (3) — 19/19 pass. `agent/curator.py:1691` has the same omission for the umbrella-curation fork, but curator's prompt is "curate all skills" — it shares no prefix with any user conversation, so cache-parity is a non-issue there. Worth auditing if the curator ever takes a parent conversation as input, but not part of this PR. The `agent/auxiliary_client.py:1006` `reasoning_config=None` hardcode is intentional (title/summary one-shots on short prompts — per-call cost of namespace flip is negligible) and is also out of scope. --- agent/background_review.py | 4 ++ .../test_background_review_cache_parity.py | 49 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/agent/background_review.py b/agent/background_review.py index bf78f679236d..e1d4cbf1e89c 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -696,6 +696,10 @@ def _run_review_in_thread( if isinstance(_rt.get("command"), str) and _rt["command"]: _fork_kwargs["acp_command"] = _rt["command"] _fork_kwargs["acp_args"] = _rt.get("args") or [] + # Match parent's reasoning config so the fork's ``thinking`` / + # ``output_config`` are byte-identical in the request body — + # Anthropic's cache key is namespaced by ``thinking`` presence. + _fork_kwargs["reasoning_config"] = getattr(agent, "reasoning_config", None) review_agent = AIAgent( model=_rt.get("model") or agent.model, max_iterations=16, diff --git a/tests/run_agent/test_background_review_cache_parity.py b/tests/run_agent/test_background_review_cache_parity.py index 58a2dfa4812f..fd7ef4ff58b5 100644 --- a/tests/run_agent/test_background_review_cache_parity.py +++ b/tests/run_agent/test_background_review_cache_parity.py @@ -41,6 +41,9 @@ def _make_agent_stub(agent_cls): # Non-None so the test catches a missing-kwarg regression. agent.enabled_toolsets = ["memory", "skills", "terminal"] agent.disabled_toolsets = ["spotify", "feishu_doc"] + # Non-None so the test catches reasoning_config NOT being inherited — + # which would put the fork into a different Anthropic cache namespace. + agent.reasoning_config = {"enabled": True, "effort": "medium"} return agent @@ -237,3 +240,49 @@ def test_review_fork_inherits_parent_toolset_config(): f"disabled_toolsets mismatch: {captured.get('disabled_toolsets')!r} " f"vs expected {agent.disabled_toolsets!r}" ) + + +def test_review_fork_inherits_parent_reasoning_config(): + """``reasoning_config`` parity: fork must inherit parent's value so the request body's ``thinking`` / ``output_config`` match (Anthropic cache is namespaced by ``thinking`` presence).""" + import run_agent + + agent = _make_agent_stub(run_agent.AIAgent) + + captured = {} + + class _Recorder: + def __init__(self, *args, **kwargs): + captured["reasoning_config"] = kwargs.get("reasoning_config") + self._cached_system_prompt = None + self._memory_write_origin = None + self._memory_write_context = None + self._memory_store = None + self._memory_enabled = None + self._user_profile_enabled = None + self._memory_nudge_interval = None + self._skill_nudge_interval = None + self.suppress_status_output = None + self.session_start = None + self.session_id = None + + def run_conversation(self, *args, **kwargs): + raise RuntimeError("stop after recording — don't actually call the API") + + def shutdown_memory_provider(self): + pass + + def close(self): + pass + + with patch.object(run_agent, "AIAgent", _Recorder), \ + patch("threading.Thread", _SyncThread): + agent._spawn_background_review( + messages_snapshot=[], + review_memory=True, + review_skills=False, + ) + + assert captured.get("reasoning_config") == agent.reasoning_config, ( + f"reasoning_config mismatch: {captured.get('reasoning_config')!r} " + f"vs expected {agent.reasoning_config!r}" + ) From 8ef006933ec05eacae74c01ef9bf4f3d7f21bb0d Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:05:17 +0530 Subject: [PATCH 065/149] fix(background_review): gate reasoning_config inheritance on not-routed + dedupe recorder stubs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up to the reasoning_config cache-parity fix: - Only inherit the parent's reasoning_config when the fork runs on the parent's model (not routed). On the routed aux path (auxiliary.background_review.{provider,model}) the cache is cold regardless, so parity buys nothing, and the parent's effort vocabulary can be invalid for the routed model/provider: OpenRouter extra_body.reasoning.effort is forwarded unclamped (chat_completions.py) and codex_responses only maps max/ultra for gpt-5.6 — an exotic parent effort routed to a strict provider could 400 the review. Mirrors the existing 'not _routed' gate on _cached_system_prompt / session_start three lines below. - Add a routed-path regression test asserting reasoning_config is omitted from the fork kwargs when _resolve_review_runtime returns routed=True. - Extract the four copy-pasted recorder stubs in test_background_review_cache_parity.py into a single _make_recorder_class() factory so a new fork attribute needs one stub edit, not four. --- agent/background_review.py | 11 +- .../test_background_review_cache_parity.py | 234 +++++++++--------- 2 files changed, 132 insertions(+), 113 deletions(-) diff --git a/agent/background_review.py b/agent/background_review.py index e1d4cbf1e89c..c2ea87bd94e2 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -699,7 +699,16 @@ def _run_review_in_thread( # Match parent's reasoning config so the fork's ``thinking`` / # ``output_config`` are byte-identical in the request body — # Anthropic's cache key is namespaced by ``thinking`` presence. - _fork_kwargs["reasoning_config"] = getattr(agent, "reasoning_config", None) + # Same-model path only: when routed to a different aux model the + # cache is cold regardless (parity buys nothing) and the parent's + # effort vocabulary may not be valid for the routed model/provider + # (e.g. OpenRouter ``extra_body.reasoning.effort`` is forwarded + # unclamped; codex_responses passes ``max``/``ultra`` through + # unmapped except on gpt-5.6/xAI). Let the routed fork use + # provider defaults — matching the ``not _routed`` gate on + # _cached_system_prompt below. + if not _routed: + _fork_kwargs["reasoning_config"] = getattr(agent, "reasoning_config", None) review_agent = AIAgent( model=_rt.get("model") or agent.model, max_iterations=16, diff --git a/tests/run_agent/test_background_review_cache_parity.py b/tests/run_agent/test_background_review_cache_parity.py index fd7ef4ff58b5..b7e27245d265 100644 --- a/tests/run_agent/test_background_review_cache_parity.py +++ b/tests/run_agent/test_background_review_cache_parity.py @@ -58,28 +58,52 @@ class _SyncThread: self._target() -class _ReviewAgentRecorder: - """Stand-in for the review-fork AIAgent that records the prompt assignment.""" +def _make_recorder_class(captured=None, record_on_run=()): + """Build a Recorder class standing in for the review-fork AIAgent. - def __init__(self, *args, **kwargs): - self._cached_system_prompt = None - self._memory_write_origin = None - self._memory_write_context = None - self._memory_store = None - self._memory_enabled = None - self._user_profile_enabled = None - self._memory_nudge_interval = None - self._skill_nudge_interval = None - self.suppress_status_output = None + Keeps the stub attribute list in ONE place: when + ``_spawn_background_review`` starts touching a new fork attribute, only + this factory needs the extra stub — not one copy per test. - def run_conversation(self, *args, **kwargs): - raise RuntimeError("stop after recording state — don't actually call the API") + ``captured`` (dict): if given, ``__init__`` stores the full constructor + kwargs under ``captured["init_kwargs"]`` so tests can assert on both + kwarg values and kwarg *presence*. + ``record_on_run``: instance attribute names copied into ``captured`` when + ``run_conversation`` fires — for values the production code assigns + after construction. + """ - def shutdown_memory_provider(self): - pass + class _Recorder: + def __init__(self, *args, **kwargs): + if captured is not None: + captured["init_kwargs"] = dict(kwargs) + self._cached_system_prompt = None + self._memory_write_origin = None + self._memory_write_context = None + self._memory_store = None + self._memory_enabled = None + self._user_profile_enabled = None + self._memory_nudge_interval = None + self._skill_nudge_interval = None + self.suppress_status_output = None + self.session_start = None + self.session_id = None - def close(self): - pass + def run_conversation(self, *args, **kwargs): + if captured is not None: + for _name in record_on_run: + captured[_name] = getattr(self, _name) + raise RuntimeError( + "stop after recording — don't actually call the API" + ) + + def shutdown_memory_provider(self): + pass + + def close(self): + pass + + return _Recorder def test_review_fork_inherits_parent_cached_system_prompt(): @@ -97,27 +121,21 @@ def test_review_fork_inherits_parent_cached_system_prompt(): captured = {} parent_prompt = agent._cached_system_prompt - # Hook the assignment site: record what gets put on the review agent. - real_recorder_init = _ReviewAgentRecorder.__init__ + _Recorder = _make_recorder_class() - def _recorder_init(self, *args, **kwargs): - real_recorder_init(self, *args, **kwargs) - # The actual production code assigns _cached_system_prompt AFTER __init__, - # so we need to capture it on attribute set. Use a property-style sentinel - # via __setattr__ on this instance. - - with patch.object(run_agent, "AIAgent", _ReviewAgentRecorder), \ + with patch.object(run_agent, "AIAgent", _Recorder), \ patch("threading.Thread", _SyncThread): - # Wrap the recorder's __setattr__ so we can see the _cached_system_prompt - # write that _spawn_background_review performs after construction. - orig_setattr = _ReviewAgentRecorder.__setattr__ + # The production code assigns _cached_system_prompt AFTER __init__, + # so wrap the recorder's __setattr__ to see that post-construction + # write from _spawn_background_review. + orig_setattr = _Recorder.__setattr__ def _spy_setattr(self, name, value): if name == "_cached_system_prompt": captured["written_prompt"] = value orig_setattr(self, name, value) - with patch.object(_ReviewAgentRecorder, "__setattr__", _spy_setattr): + with patch.object(_Recorder, "__setattr__", _spy_setattr): agent._spawn_background_review( messages_snapshot=[], review_memory=True, @@ -147,31 +165,9 @@ def test_review_fork_pins_session_start_and_session_id(): agent = _make_agent_stub(run_agent.AIAgent) captured = {} - - class _Recorder: - def __init__(self, *args, **kwargs): - self._cached_system_prompt = None - self._memory_write_origin = None - self._memory_write_context = None - self._memory_store = None - self._memory_enabled = None - self._user_profile_enabled = None - self._memory_nudge_interval = None - self._skill_nudge_interval = None - self.suppress_status_output = None - self.session_start = None - self.session_id = None - - def run_conversation(self, *args, **kwargs): - captured["session_start"] = self.session_start - captured["session_id"] = self.session_id - raise RuntimeError("stop after recording") - - def shutdown_memory_provider(self): - pass - - def close(self): - pass + _Recorder = _make_recorder_class( + captured, record_on_run=("session_start", "session_id") + ) with patch.object(run_agent, "AIAgent", _Recorder), \ patch("threading.Thread", _SyncThread): @@ -198,31 +194,7 @@ def test_review_fork_inherits_parent_toolset_config(): agent = _make_agent_stub(run_agent.AIAgent) captured = {} - - class _Recorder: - def __init__(self, *args, **kwargs): - captured["enabled_toolsets"] = kwargs.get("enabled_toolsets") - captured["disabled_toolsets"] = kwargs.get("disabled_toolsets") - self._cached_system_prompt = None - self._memory_write_origin = None - self._memory_write_context = None - self._memory_store = None - self._memory_enabled = None - self._user_profile_enabled = None - self._memory_nudge_interval = None - self._skill_nudge_interval = None - self.suppress_status_output = None - self.session_start = None - self.session_id = None - - def run_conversation(self, *args, **kwargs): - raise RuntimeError("stop after recording — don't actually call the API") - - def shutdown_memory_provider(self): - pass - - def close(self): - pass + _Recorder = _make_recorder_class(captured) with patch.object(run_agent, "AIAgent", _Recorder), \ patch("threading.Thread", _SyncThread): @@ -232,47 +204,30 @@ def test_review_fork_inherits_parent_toolset_config(): review_skills=False, ) - assert captured.get("enabled_toolsets") == agent.enabled_toolsets, ( - f"enabled_toolsets mismatch: {captured.get('enabled_toolsets')!r} " + init_kwargs = captured.get("init_kwargs", {}) + assert init_kwargs.get("enabled_toolsets") == agent.enabled_toolsets, ( + f"enabled_toolsets mismatch: {init_kwargs.get('enabled_toolsets')!r} " f"vs expected {agent.enabled_toolsets!r}" ) - assert captured.get("disabled_toolsets") == agent.disabled_toolsets, ( - f"disabled_toolsets mismatch: {captured.get('disabled_toolsets')!r} " + assert init_kwargs.get("disabled_toolsets") == agent.disabled_toolsets, ( + f"disabled_toolsets mismatch: {init_kwargs.get('disabled_toolsets')!r} " f"vs expected {agent.disabled_toolsets!r}" ) def test_review_fork_inherits_parent_reasoning_config(): - """``reasoning_config`` parity: fork must inherit parent's value so the request body's ``thinking`` / ``output_config`` match (Anthropic cache is namespaced by ``thinking`` presence).""" + """``reasoning_config`` parity on the default (non-routed) path. + + The fork must inherit the parent's value so the request body's + ``thinking`` / ``output_config`` match — Anthropic's cache is + namespaced by ``thinking`` presence. + """ import run_agent agent = _make_agent_stub(run_agent.AIAgent) captured = {} - - class _Recorder: - def __init__(self, *args, **kwargs): - captured["reasoning_config"] = kwargs.get("reasoning_config") - self._cached_system_prompt = None - self._memory_write_origin = None - self._memory_write_context = None - self._memory_store = None - self._memory_enabled = None - self._user_profile_enabled = None - self._memory_nudge_interval = None - self._skill_nudge_interval = None - self.suppress_status_output = None - self.session_start = None - self.session_id = None - - def run_conversation(self, *args, **kwargs): - raise RuntimeError("stop after recording — don't actually call the API") - - def shutdown_memory_provider(self): - pass - - def close(self): - pass + _Recorder = _make_recorder_class(captured) with patch.object(run_agent, "AIAgent", _Recorder), \ patch("threading.Thread", _SyncThread): @@ -282,7 +237,62 @@ def test_review_fork_inherits_parent_reasoning_config(): review_skills=False, ) - assert captured.get("reasoning_config") == agent.reasoning_config, ( - f"reasoning_config mismatch: {captured.get('reasoning_config')!r} " + init_kwargs = captured.get("init_kwargs", {}) + assert init_kwargs.get("reasoning_config") == agent.reasoning_config, ( + f"reasoning_config mismatch: {init_kwargs.get('reasoning_config')!r} " f"vs expected {agent.reasoning_config!r}" ) + + +def test_routed_review_fork_does_not_inherit_reasoning_config(): + """Routed aux path: the fork must NOT inherit the parent's reasoning_config. + + When ``auxiliary.background_review.{provider,model}`` routes the review + to a different model, cache parity is moot (the cache is cold on that + model regardless) and the parent's effort vocabulary may be invalid for + the routed model/provider (OpenRouter ``extra_body.reasoning.effort`` is + forwarded unclamped; codex_responses passes ``max``/``ultra`` through + unmapped except on gpt-5.6/xAI). The routed fork must fall back to + provider defaults, mirroring the ``not _routed`` gate on + ``_cached_system_prompt`` inheritance. + """ + import run_agent + import agent.background_review as bg_review + + agent_stub = _make_agent_stub(run_agent.AIAgent) + + captured = {} + _Recorder = _make_recorder_class(captured) + + routed_runtime = { + "provider": "openrouter", + "model": "aux-cheap-model", + "api_key": "test-key", + "base_url": None, + "api_mode": None, + "credential_pool": None, + "request_overrides": {}, + "max_tokens": None, + "command": None, + "args": [], + "routed": True, + } + + with patch.object(run_agent, "AIAgent", _Recorder), \ + patch.object(bg_review, "_resolve_review_runtime", + return_value=routed_runtime), \ + patch("threading.Thread", _SyncThread): + agent_stub._spawn_background_review( + messages_snapshot=[], + review_memory=True, + review_skills=False, + ) + + init_kwargs = captured.get("init_kwargs", {}) + assert "reasoning_config" not in init_kwargs, ( + f"Routed review fork was passed the parent's reasoning_config " + f"({init_kwargs.get('reasoning_config')!r}). On the routed path the " + "cache is cold (no parity benefit) and the parent's effort value may " + "be invalid for the routed model/provider — it must be omitted so " + "the fork uses provider defaults." + ) From 202be02ac9a1b6ac8b640e9b2e0917fae6806f4b Mon Sep 17 00:00:00 2001 From: Roseyco-management Date: Mon, 13 Jul 2026 03:57:50 +0100 Subject: [PATCH 066/149] test(telegram): define polling progress contract --- plugins/platforms/telegram/adapter.py | 42 +++++ .../gateway/test_telegram_polling_progress.py | 171 ++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 tests/gateway/test_telegram_polling_progress.py diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index 05506c79da48..e617b7b0bd23 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -633,6 +633,10 @@ class TelegramAdapter(BasePlatformAdapter): self._polling_error_task: Optional[asyncio.Task] = None self._polling_conflict_count: int = 0 self._polling_network_error_count: int = 0 + self._polling_generation: int = 0 + self._polling_progress_event = asyncio.Event() + self._polling_progress_accepting: bool = False + self._polling_progress_verifier_task: Optional[asyncio.Task] = None self._polling_error_callback_ref = None self._polling_heartbeat_task: Optional[asyncio.Task] = None # Consecutive heartbeat probes that saw queued updates the running @@ -1944,6 +1948,43 @@ class TelegramAdapter(BasePlatformAdapter): self.name, exc_info=True, ) + def _begin_polling_generation(self) -> tuple[int, asyncio.Event]: + """Start accepting progress for a new getUpdates polling generation.""" + verifier = self._polling_progress_verifier_task + if verifier is not None and not verifier.done(): + verifier.cancel() + self._polling_progress_verifier_task = None + self._polling_generation += 1 + self._polling_progress_event = asyncio.Event() + self._polling_progress_accepting = True + self._send_path_degraded = True + return self._polling_generation, self._polling_progress_event + + def _record_polling_progress(self, generation: int) -> None: + """Record successful getUpdates I/O for the current generation only.""" + if not self._polling_progress_accepting: + return + if generation != self._polling_generation: + return + self._polling_progress_event.set() + self._polling_network_error_count = 0 + self._send_path_degraded = False + + def _instrument_polling_request(self, request): + """Wrap one dedicated PTB getUpdates request with progress tracking.""" + do_request = request.do_request + + async def _do_request(*args, **kwargs): + generation = self._polling_generation + result = await do_request(*args, **kwargs) + status_code, _ = result + if 200 <= status_code < 300: + self._record_polling_progress(generation) + return result + + request.do_request = _do_request + return request + def _get_general_request_drain_lock(self) -> asyncio.Lock: lock = getattr(self, "_general_request_drain_lock", None) if lock is None: @@ -3246,6 +3287,7 @@ class TelegramAdapter(BasePlatformAdapter): **request_kwargs, httpx_kwargs=_with_limits() ) + get_updates_request = self._instrument_polling_request(get_updates_request) builder = builder.request(request).get_updates_request(get_updates_request) self._app = builder.build() self._bot = self._app.bot diff --git a/tests/gateway/test_telegram_polling_progress.py b/tests/gateway/test_telegram_polling_progress.py new file mode 100644 index 000000000000..717eaaf868a5 --- /dev/null +++ b/tests/gateway/test_telegram_polling_progress.py @@ -0,0 +1,171 @@ +"""Behavior contract for generation-safe Telegram polling progress.""" + +import asyncio + +import pytest + +from gateway.config import PlatformConfig +from plugins.platforms.telegram import adapter as tg_adapter +from plugins.platforms.telegram.adapter import TelegramAdapter + + +class _ControlledRequest: + """Minimal PTB request double with controllable completion.""" + + instances = [] + + def __init__(self, *args, result=None, error=None, entered=None, release=None, **kwargs): + self.result = result + self.error = error + self.entered = entered + self.release = release + self.args = args + self.kwargs = kwargs + type(self).instances.append(self) + + async def do_request(self, *args, **kwargs): + if self.entered is not None: + self.entered.set() + if self.release is not None: + await self.release.wait() + if self.error is not None: + raise self.error + return self.result + + +def _make_adapter() -> TelegramAdapter: + return TelegramAdapter(PlatformConfig(enabled=True, token="test-token")) + + +@pytest.mark.asyncio +async def test_current_polling_generation_success_records_progress(): + adapter = _make_adapter() + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 3 + request = _ControlledRequest(result=(200, b'{"ok":true}')) + + instrumented = adapter._instrument_polling_request(request) + result = await instrumented.do_request("https://api.telegram.org/getUpdates") + + assert instrumented is request + assert result == (200, b'{"ok":true}') + assert progress.is_set() + assert adapter._polling_network_error_count == 0 + assert adapter._send_path_degraded is False + assert generation > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_type", [RuntimeError, asyncio.CancelledError]) +async def test_unsuccessful_polling_request_does_not_record_progress(error_type): + adapter = _make_adapter() + _, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 3 + request = adapter._instrument_polling_request( + _ControlledRequest(error=error_type("request did not complete")) + ) + + with pytest.raises(error_type): + await request.do_request("https://api.telegram.org/getUpdates") + + assert not progress.is_set() + assert adapter._polling_network_error_count == 3 + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_http_error_response_does_not_record_polling_progress(): + adapter = _make_adapter() + _, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 3 + request = adapter._instrument_polling_request( + _ControlledRequest(result=(500, b"bad")) + ) + + result = await request.do_request("https://api.telegram.org/getUpdates") + + assert result == (500, b"bad") + assert not progress.is_set() + assert adapter._polling_network_error_count == 3 + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_general_request_success_cannot_record_polling_progress(monkeypatch): + class _StopConnect(Exception): + pass + + class _Builder: + def __init__(self): + self.general_request = None + self.polling_request = None + + def token(self, _token): + return self + + def request(self, request): + self.general_request = request + return self + + def get_updates_request(self, request): + self.polling_request = request + return self + + def build(self): + raise _StopConnect + + builder = _Builder() + + class _Application: + @staticmethod + def builder(): + return builder + + _ControlledRequest.instances = [] + + async def _no_fallback_ips(): + return [] + + monkeypatch.setattr(tg_adapter, "Application", _Application) + monkeypatch.setattr(tg_adapter, "HTTPXRequest", _ControlledRequest) + monkeypatch.setattr(tg_adapter, "discover_fallback_ips", _no_fallback_ips) + monkeypatch.setattr(tg_adapter, "resolve_proxy_url", lambda *args, **kwargs: None) + + adapter = _make_adapter() + monkeypatch.setattr(adapter, "_acquire_platform_lock", lambda *args, **kwargs: True) + monkeypatch.setattr(adapter, "_fallback_ips", lambda: []) + _, progress = adapter._begin_polling_generation() + + assert await adapter.connect() is False + assert builder.general_request is _ControlledRequest.instances[0] + assert builder.polling_request is _ControlledRequest.instances[1] + + builder.general_request.result = (200, b'{"ok":true}') + result = await builder.general_request.do_request("https://api.telegram.org/sendMessage") + + assert result == (200, b'{"ok":true}') + assert not progress.is_set() + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_late_previous_generation_completion_cannot_heal_current_generation(): + adapter = _make_adapter() + generation_1, _ = adapter._begin_polling_generation() + entered = asyncio.Event() + release = asyncio.Event() + request = adapter._instrument_polling_request( + _ControlledRequest(result=(200, b'{"ok":true}'), entered=entered, release=release) + ) + + completion = asyncio.create_task(request.do_request("getUpdates")) + await entered.wait() + generation_2, progress_2 = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + release.set() + + assert await completion == (200, b'{"ok":true}') + assert generation_2 == generation_1 + 1 + assert not progress_2.is_set() + assert adapter._polling_network_error_count == 4 + assert adapter._send_path_degraded is True From b8295cf6f737a9ff1a4696cef5fab9d010e27d3d Mon Sep 17 00:00:00 2001 From: Roseyco-management Date: Mon, 13 Jul 2026 04:27:20 +0100 Subject: [PATCH 067/149] fix(telegram): gate polling health on getUpdates progress --- plugins/platforms/telegram/adapter.py | 443 +++++++++---- tests/gateway/test_telegram_conflict.py | 59 +- .../test_telegram_network_reconnect.py | 82 ++- .../gateway/test_telegram_polling_progress.py | 623 +++++++++++++++++- .../gateway/test_telegram_send_path_health.py | 62 +- tests/test_telegram_polling_progress_ptb.py | 292 ++++++++ 6 files changed, 1347 insertions(+), 214 deletions(-) create mode 100644 tests/test_telegram_polling_progress_ptb.py diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index e617b7b0bd23..3a986283239c 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -17,6 +17,7 @@ import os import html as _html import re import threading +from contextvars import ContextVar from datetime import datetime, timezone from typing import Dict, List, Optional, Set, Any @@ -474,6 +475,16 @@ _UPDATER_STOP_TIMEOUT = 15.0 # reconnect ladder from stalling indefinitely and allows the heartbeat loop to # trigger its own recovery path. Refs: NousResearch/hermes-agent#59614 _UPDATER_START_TIMEOUT = 30.0 +# A generation is not healthy until the dedicated getUpdates request returns +# successfully. This exceeds a normal long-poll cycle for healthy idle bots. +_POLLING_PROGRESS_TIMEOUT = 60.0 +_POLLING_GENERATION_CONTEXT: ContextVar[Optional[int]] = ContextVar( + "telegram_polling_generation", default=None +) + + +class _PollingLifecycleAbort(RuntimeError): + """Internal control flow for polling startup fenced by teardown.""" class TelegramAdapter(BasePlatformAdapter): @@ -637,6 +648,7 @@ class TelegramAdapter(BasePlatformAdapter): self._polling_progress_event = asyncio.Event() self._polling_progress_accepting: bool = False self._polling_progress_verifier_task: Optional[asyncio.Task] = None + self._polling_teardown_started: bool = False self._polling_error_callback_ref = None self._polling_heartbeat_task: Optional[asyncio.Task] = None # Consecutive heartbeat probes that saw queued updates the running @@ -652,10 +664,9 @@ class TelegramAdapter(BasePlatformAdapter): # error_callback ever fires and the gateway silently stops receiving # messages with the process still alive (#55769). self._polling_not_running_count: int = 0 - # After sustained reconnect storms the PTB httpx pool can return - # SendResult(success=True) for sends that never actually transmit. - # _handle_polling_network_error sets this; _verify_polling_after_reconnect - # clears it once getMe() confirms the Bot client is healthy. + # A polling generation stays degraded until the dedicated getUpdates + # request makes successful progress. start_polling() return and getMe() + # success on the general request path are not polling-health signals. # While True, send() short-circuits to a failure so callers # (cron live-adapter branch) fall through to standalone delivery. self._send_path_degraded: bool = False @@ -1950,11 +1961,20 @@ class TelegramAdapter(BasePlatformAdapter): def _begin_polling_generation(self) -> tuple[int, asyncio.Event]: """Start accepting progress for a new getUpdates polling generation.""" - verifier = self._polling_progress_verifier_task + if getattr(self, "_polling_teardown_started", False): + self._polling_progress_accepting = False + self._send_path_degraded = True + progress = getattr(self, "_polling_progress_event", None) + if progress is None: + progress = asyncio.Event() + self._polling_progress_event = progress + return getattr(self, "_polling_generation", 0), progress + + verifier = getattr(self, "_polling_progress_verifier_task", None) if verifier is not None and not verifier.done(): verifier.cancel() self._polling_progress_verifier_task = None - self._polling_generation += 1 + self._polling_generation = getattr(self, "_polling_generation", 0) + 1 self._polling_progress_event = asyncio.Event() self._polling_progress_accepting = True self._send_path_degraded = True @@ -1962,12 +1982,15 @@ class TelegramAdapter(BasePlatformAdapter): def _record_polling_progress(self, generation: int) -> None: """Record successful getUpdates I/O for the current generation only.""" + if getattr(self, "_polling_teardown_started", False): + return if not self._polling_progress_accepting: return if generation != self._polling_generation: return self._polling_progress_event.set() self._polling_network_error_count = 0 + self._polling_conflict_count = 0 self._send_path_degraded = False def _instrument_polling_request(self, request): @@ -1975,16 +1998,100 @@ class TelegramAdapter(BasePlatformAdapter): do_request = request.do_request async def _do_request(*args, **kwargs): - generation = self._polling_generation + generation = _POLLING_GENERATION_CONTEXT.get() result = await do_request(*args, **kwargs) - status_code, _ = result - if 200 <= status_code < 300: - self._record_polling_progress(generation) + status_code, payload = result + if generation is not None and 200 <= status_code < 300: + try: + # Use the request's own parser so health observation agrees + # exactly with PTB's authoritative response handling (e.g. + # UTF-8 replacement decoding and BOM rejection). + envelope = request.parse_json_payload(payload) + except Exception: + # Instrumentation is observational: PTB still parses the + # untouched payload and owns the resulting exception. + pass + else: + if ( + isinstance(envelope, dict) + and envelope.get("ok") is True + and "result" in envelope + ): + self._record_polling_progress(generation) return result request.do_request = _do_request return request + async def _start_polling_once( + self, + app, + *, + drop_pending_updates: bool, + error_callback, + ) -> None: + """Start one generation and verify real getUpdates progress.""" + if getattr(self, "_polling_teardown_started", False): + raise _PollingLifecycleAbort("Telegram polling teardown started") + generation, progress = self._begin_polling_generation() + if not self._polling_progress_accepting: + raise _PollingLifecycleAbort("Telegram polling teardown started") + + def _generation_error_callback(error: Exception) -> None: + if getattr(self, "_polling_teardown_started", False): + return + if generation != self._polling_generation: + return + if error_callback is not None: + callback_context_token = _POLLING_GENERATION_CONTEXT.set(None) + try: + error_callback(error) + finally: + _POLLING_GENERATION_CONTEXT.reset(callback_context_token) + + context_token = _POLLING_GENERATION_CONTEXT.set(generation) + try: + await asyncio.wait_for( + app.updater.start_polling( + allowed_updates=Update.ALL_TYPES, + drop_pending_updates=drop_pending_updates, + error_callback=_generation_error_callback, + ), + timeout=_UPDATER_START_TIMEOUT, + ) + finally: + _POLLING_GENERATION_CONTEXT.reset(context_token) + if getattr(self, "_polling_teardown_started", False): + self._polling_progress_accepting = False + self._send_path_degraded = True + raise _PollingLifecycleAbort("Telegram polling teardown started") + self._schedule_polling_progress_verifier(generation, progress) + + def _schedule_polling_progress_verifier( + self, generation: int, progress: asyncio.Event + ) -> None: + """Own exactly one tracked verifier for the current generation.""" + if getattr(self, "_polling_teardown_started", False): + self._polling_progress_accepting = False + self._send_path_degraded = True + return + previous = getattr(self, "_polling_progress_verifier_task", None) + if previous is not None and not previous.done(): + previous.cancel() + + task = asyncio.get_running_loop().create_task( + self._verify_polling_after_reconnect(generation, progress) + ) + self._polling_progress_verifier_task = task + self._background_tasks.add(task) + + def _clear_finished_verifier(finished: asyncio.Task) -> None: + self._background_tasks.discard(finished) + if self._polling_progress_verifier_task is finished: + self._polling_progress_verifier_task = None + + task.add_done_callback(_clear_finished_verifier) + def _get_general_request_drain_lock(self) -> asyncio.Lock: lock = getattr(self, "_general_request_drain_lock", None) if lock is None: @@ -2037,6 +2144,8 @@ class TelegramAdapter(BasePlatformAdapter): adapter: the gateway process stays alive and the existing reconnect ladder (``_handle_polling_network_error``) recovers in the background. """ + if getattr(self, "_polling_teardown_started", False): + return if self.has_fatal_error: return if self._polling_error_task and not self._polling_error_task.done(): @@ -2088,6 +2197,8 @@ class TelegramAdapter(BasePlatformAdapter): network error was scheduled for background recovery instead of raising (keeping the gateway process alive). """ + if getattr(self, "_polling_teardown_started", False): + return False if not (self._app and self._app.updater): raise RuntimeError("Telegram application/updater not initialized") try: @@ -2097,16 +2208,17 @@ class TelegramAdapter(BasePlatformAdapter): # TimeoutError (OSError subclass), so the except below classifies # it via _looks_like_network_error and schedules background # recovery instead of blocking connect() indefinitely. - await asyncio.wait_for( - self._app.updater.start_polling( - allowed_updates=Update.ALL_TYPES, - drop_pending_updates=drop_pending_updates, - error_callback=error_callback, - ), - timeout=_UPDATER_START_TIMEOUT, + await self._start_polling_once( + self._app, + drop_pending_updates=drop_pending_updates, + error_callback=error_callback, ) return True + except _PollingLifecycleAbort: + return False except Exception as err: + if getattr(self, "_polling_teardown_started", False): + return False if self._looks_like_polling_conflict(err): logger.warning( "[%s] Telegram polling bootstrap conflict; gateway stays alive " @@ -2135,6 +2247,8 @@ class TelegramAdapter(BasePlatformAdapter): MAX_NETWORK_RETRIES attempts, then mark the adapter retryable-fatal so the supervisor restarts the gateway process. """ + if getattr(self, "_polling_teardown_started", False): + return if self.has_fatal_error: return @@ -2164,6 +2278,9 @@ class TelegramAdapter(BasePlatformAdapter): ) await asyncio.sleep(delay) + if getattr(self, "_polling_teardown_started", False): + return + # Capture a stable local reference: self._app can be reassigned to None # by a concurrent disconnect() while we're suspended across the awaits # below, and re-reading self._app after that point would silently swap @@ -2194,64 +2311,39 @@ class TelegramAdapter(BasePlatformAdapter): except Exception: pass + if getattr(self, "_polling_teardown_started", False): + return await self._drain_polling_connections() + if getattr(self, "_polling_teardown_started", False): + return + try: if not app: raise RuntimeError("Telegram application was torn down during reconnect") - # Guard start_polling() with a timeout: when the connection pool is - # in a degraded state (e.g., after _drain_polling_connections()), the - # httpx client may hold a stale socket that neither connects nor times - # out within PTB's internal flow. Bounding start_polling() prevents - # the reconnect ladder from stalling indefinitely and allows the - # heartbeat loop to trigger its own recovery path. - # Refs: NousResearch/hermes-agent#59614 - try: - await asyncio.wait_for( - app.updater.start_polling( - allowed_updates=Update.ALL_TYPES, - drop_pending_updates=False, - error_callback=self._polling_error_callback_ref, - ), - timeout=_UPDATER_START_TIMEOUT, - ) - except asyncio.TimeoutError: - raise RuntimeError( - "start_polling() timed out — connection pool may be wedged" - ) + await self._start_polling_once( + app, + drop_pending_updates=False, + error_callback=self._polling_error_callback_ref, + ) logger.info( - "[%s] Telegram polling resumed after network error (attempt %d)", + "[%s] Telegram polling restarted after network error (attempt %d); " + "health pending getUpdates progress", self.name, attempt, ) - self._polling_network_error_count = 0 - # start_polling() succeeding IS the recovery signal: the long-poll - # connection is live again, so clear the degraded flag immediately - # rather than blocking all outbound sends for the full - # HEARTBEAT_PROBE_DELAY window. The deferred probe below is a - # defensive re-check — if it later detects a silent wedge (PTB - # running=True but consumer task dead) it re-enters the ladder, - # which re-sets _send_path_degraded. Without this clear here, a - # clean reconnect leaves the flag stuck True until the 60s probe - # (or forever, if the probe is never scheduled), blocking the send - # path even though the bot has fully recovered. See #35205. - self._send_path_degraded = False - # start_polling() returning is necessary but not sufficient: - # PTB's Updater can be left in a state where `running` is True - # but the underlying long-poll task is wedged on a stale httpx - # connection and never makes progress. No error_callback fires - # in that state, so the reconnect ladder won't advance on its - # own. Schedule a deferred probe to detect the wedge and - # re-enter the ladder if needed. - if not self.has_fatal_error: - probe = asyncio.ensure_future(self._verify_polling_after_reconnect()) - self._background_tasks.add(probe) - probe.add_done_callback(self._background_tasks.discard) + except _PollingLifecycleAbort: + return except Exception as retry_err: + if getattr(self, "_polling_teardown_started", False): + return safe_retry_error = _redact_telegram_error_text(retry_err) logger.warning("[%s] Telegram polling reconnect failed: %s", self.name, safe_retry_error) # start_polling failed — polling is dead and no further error # callbacks will fire, so schedule the next retry ourselves. - if not self.has_fatal_error: + if ( + not self.has_fatal_error + and not getattr(self, "_polling_teardown_started", False) + ): task = asyncio.ensure_future( self._handle_polling_network_error(retry_err) ) @@ -2281,9 +2373,9 @@ class TelegramAdapter(BasePlatformAdapter): ``_handle_polling_network_error`` — the same path triggered by PTB's own ``error_callback`` — which drains the dead pool and restarts polling. - Unlike ``_verify_polling_after_reconnect`` (a one-shot probe scheduled - only after an explicit reconnect), this loop runs for the full lifetime - of the polling connection, so it catches a socket that wedges during + Unlike the generation verifier (a one-shot progress deadline after + every polling start), this loop runs for the full lifetime of the + polling connection, so it catches a socket that wedges later during steady-state operation without any prior error event. """ HEARTBEAT_INTERVAL = 90 # seconds between probes @@ -2292,6 +2384,8 @@ class TelegramAdapter(BasePlatformAdapter): while True: try: await asyncio.sleep(HEARTBEAT_INTERVAL) + if getattr(self, "_polling_teardown_started", False): + return if self.has_fatal_error: return bot = self._app.bot if self._app else None @@ -2348,6 +2442,8 @@ class TelegramAdapter(BasePlatformAdapter): report a queue against a live consumer. We detect the stopped updater directly and feed the same ladder (#55769). """ + if getattr(self, "_polling_teardown_started", False): + return # Only meaningful in polling mode; in webhook mode Telegram pushes # updates and holds no server-side queue. if self._webhook_mode: @@ -2381,6 +2477,8 @@ class TelegramAdapter(BasePlatformAdapter): ) if self._polling_not_running_count >= 2: self._polling_not_running_count = 0 + if getattr(self, "_polling_teardown_started", False): + return logger.warning( "[%s] Telegram updater is not running (long-poll task " "gone); triggering polling restart", @@ -2415,6 +2513,8 @@ class TelegramAdapter(BasePlatformAdapter): ) if self._polling_pending_stuck_count >= 2: self._polling_pending_stuck_count = 0 + if getattr(self, "_polling_teardown_started", False): + return logger.warning( "[%s] getUpdates consumer appears wedged (queue not draining); " "triggering polling restart", @@ -2427,66 +2527,96 @@ class TelegramAdapter(BasePlatformAdapter): ) ) - async def _verify_polling_after_reconnect(self) -> None: - """Heartbeat probe scheduled after a successful reconnect. + async def _verify_polling_after_reconnect( + self, + generation: Optional[int] = None, + progress: Optional[asyncio.Event] = None, + ) -> None: + """Require getUpdates progress, using getMe only to classify failure. - PTB's Updater can survive a botched stop()+start_polling() cycle - with `running=True` but a wedged consumer task. No error callback - fires, so the reconnect ladder doesn't advance on its own. This - probe detects the wedge by: - - 1. Sleeping HEARTBEAT_PROBE_DELAY so a healthy long-poll has time - to complete at least one cycle. - 2. Verifying `Updater.running` is still True. - 3. Probing the bot endpoint with a tight asyncio timeout. A - wedged httpx pool fails this probe; a healthy one returns - well under the timeout. - - On connectivity failure, re-enter the reconnect ladder (via - ``_schedule_polling_recovery`` so the in-flight guard and task - bookkeeping apply) and let the existing MAX_NETWORK_RETRIES path - ultimately escalate to fatal-error. Auth/validation failures - (``InvalidToken``, ``BadRequest``, ...) are not connectivity symptoms - and must not trigger reconnect churn — same policy as the heartbeat - loop and the pending-update probe (#62098, #63243). + The generation-bound event is set only by a successful response on the + dedicated getUpdates request. A general-path getMe success can classify + connectivity, but cannot heal polling health. Connectivity failures + enter the guarded recovery ladder; auth/validation errors do not churn. """ - HEARTBEAT_PROBE_DELAY = 60 PROBE_TIMEOUT = 10 - - await asyncio.sleep(HEARTBEAT_PROBE_DELAY) - - if self.has_fatal_error: + if getattr(self, "_polling_teardown_started", False): return - if not (self._app and self._app.updater and self._app.updater.running): + if generation is None: + generation = self._polling_generation + if progress is None: + progress = self._polling_progress_event + + try: + await asyncio.wait_for( + progress.wait(), timeout=_POLLING_PROGRESS_TIMEOUT + ) + except asyncio.TimeoutError: + pass + + if getattr(self, "_polling_teardown_started", False): + return + if progress.is_set() or self.has_fatal_error: + return + if not self._polling_progress_accepting: + return + if generation != self._polling_generation: + return + if progress is not self._polling_progress_event: + return + + app = self._app + if not (app and app.updater and app.updater.running): logger.warning( - "[%s] Updater not running %ds after reconnect — treating as wedged", - self.name, HEARTBEAT_PROBE_DELAY, + "[%s] Updater made no getUpdates progress and is not running", + self.name, ) self._schedule_polling_recovery( - RuntimeError("Updater not running after reconnect heartbeat"), - reason="post-reconnect probe: updater not running", + RuntimeError("Updater not running after polling progress deadline"), + reason="polling progress verifier: updater not running", ) return try: - await asyncio.wait_for(self._app.bot.get_me(), PROBE_TIMEOUT) - self._send_path_degraded = False + await asyncio.wait_for(app.bot.get_me(), PROBE_TIMEOUT) except Exception as probe_err: + if getattr(self, "_polling_teardown_started", False): + return + if self.has_fatal_error or not self._polling_progress_accepting: + return + if generation != self._polling_generation: + return + if progress is not self._polling_progress_event or progress.is_set(): + return if not self._looks_like_network_error(probe_err): logger.warning( - "[%s] Post-reconnect probe hit a non-connectivity error" + "[%s] Polling progress verifier hit a non-connectivity error" " (not retrying): %s", self.name, _redact_telegram_error_text(probe_err), ) return logger.warning( - "[%s] Polling heartbeat probe failed %ds after reconnect: %s", - self.name, HEARTBEAT_PROBE_DELAY, - _redact_telegram_error_text(probe_err), + "[%s] Polling progress verifier connectivity probe failed: %s", + self.name, _redact_telegram_error_text(probe_err), ) self._schedule_polling_recovery( - probe_err, reason="post-reconnect probe failure" + probe_err, + reason="polling progress verifier connectivity failure", ) + return + + if getattr(self, "_polling_teardown_started", False): + return + if self.has_fatal_error or not self._polling_progress_accepting: + return + if generation != self._polling_generation: + return + if progress is not self._polling_progress_event or progress.is_set(): + return + self._schedule_polling_recovery( + RuntimeError("getUpdates made no progress before verifier deadline"), + reason="polling progress verifier: general path healthy but getUpdates stalled", + ) def _disarm_ptb_retry_loop(self) -> None: """Synchronously stop PTB's internal polling retry loop. @@ -2552,6 +2682,8 @@ class TelegramAdapter(BasePlatformAdapter): ) async def _handle_polling_conflict(self, error: Exception) -> None: + if getattr(self, "_polling_teardown_started", False): + return if self.has_fatal_error and self.fatal_error_code == "telegram_polling_conflict": return # Transient 409 Conflict errors arise when the previous gateway process @@ -2607,7 +2739,11 @@ class TelegramAdapter(BasePlatformAdapter): pass await asyncio.sleep(RETRY_DELAY) + if getattr(self, "_polling_teardown_started", False): + return await self._drain_polling_connections() + if getattr(self, "_polling_teardown_started", False): + return # Capture a stable local reference: self._app can be reassigned to # None by a concurrent disconnect() while we're suspended across @@ -2619,31 +2755,22 @@ class TelegramAdapter(BasePlatformAdapter): try: if not app: raise RuntimeError("Telegram application was torn down during conflict reconnect") - # Same watchdog bound as the network-error ladder: an - # exhausted pool hangs start_polling() on the conflict path - # identically (#59614). Timeout converts to RuntimeError so - # the except below logs a readable message and schedules the - # next conflict attempt instead of wedging attempt N forever. - try: - await asyncio.wait_for( - app.updater.start_polling( - allowed_updates=Update.ALL_TYPES, - drop_pending_updates=False, - error_callback=self._polling_error_callback_ref, - ), - timeout=_UPDATER_START_TIMEOUT, - ) - except asyncio.TimeoutError: - raise RuntimeError( - "start_polling() timed out — connection pool may be wedged" - ) + await self._start_polling_once( + app, + drop_pending_updates=False, + error_callback=self._polling_error_callback_ref, + ) logger.info( - "[%s] Telegram polling resumed after conflict retry %d/%d", + "[%s] Telegram polling restarted after conflict retry %d/%d; " + "health pending getUpdates progress", self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES, ) - self._polling_conflict_count = 0 # reset counter on success + return + except _PollingLifecycleAbort: return except Exception as retry_err: + if getattr(self, "_polling_teardown_started", False): + return logger.warning( "[%s] Telegram polling retry %d/%d failed: %s. " "Scheduling next attempt.", @@ -2655,7 +2782,10 @@ class TelegramAdapter(BasePlatformAdapter): # a fatal error leaves the adapter in a limbo state: the # gateway process is alive and reports "connected" but # no messages are received or sent. - if self._polling_conflict_count < MAX_CONFLICT_RETRIES: + if ( + self._polling_conflict_count < MAX_CONFLICT_RETRIES + and not getattr(self, "_polling_teardown_started", False) + ): # We are inside a running coroutine, so the running loop is # guaranteed to exist. asyncio.get_event_loop() is deprecated # and raises "RuntimeError: There is no current event loop in @@ -2669,6 +2799,9 @@ class TelegramAdapter(BasePlatformAdapter): return # Fall through to fatal on the last retry. + if getattr(self, "_polling_teardown_started", False): + return + # Exhausted all retries — declare a fatal error so the gateway # runner can surface this clearly and the user knows to act. message = ( @@ -3125,6 +3258,14 @@ class TelegramAdapter(BasePlatformAdapter): TELEGRAM_WEBHOOK_PORT Local listen port (default 8443) TELEGRAM_WEBHOOK_SECRET Secret token for update verification """ + # Explicit connect() is the only operation allowed to reopen polling + # after a completed, serialized teardown. Background recovery never + # clears this fence. + self._polling_teardown_started = False + # Mode selection is re-evaluated on every explicit connection. Keep + # webhook state false unless this connection starts its webhook. + self._webhook_mode = False + if not TELEGRAM_AVAILABLE: logger.error( "[%s] python-telegram-bot not installed. Run: pip install python-telegram-bot", @@ -3420,6 +3561,8 @@ class TelegramAdapter(BasePlatformAdapter): drop_pending_updates=not is_reconnect, ) self._webhook_mode = True + self._polling_progress_accepting = False + self._send_path_degraded = False logger.info( "[%s] Webhook server listening on 0.0.0.0:%d%s", self.name, webhook_port, webhook_path, @@ -3436,6 +3579,8 @@ class TelegramAdapter(BasePlatformAdapter): loop = asyncio.get_running_loop() def _polling_error_callback(error: Exception) -> None: + if getattr(self, "_polling_teardown_started", False): + return if self._polling_error_task and not self._polling_error_task.done(): return if self._looks_like_polling_conflict(error): @@ -3569,7 +3714,8 @@ class TelegramAdapter(BasePlatformAdapter): collect(task) for task in list(self._pending_text_batch_tasks.values()): collect(task) - collect(self._polling_error_task) + collect(getattr(self, "_polling_error_task", None)) + collect(getattr(self, "_polling_progress_verifier_task", None)) for task in pending_tasks: task.cancel() @@ -3582,8 +3728,10 @@ class TelegramAdapter(BasePlatformAdapter): self._pending_photo_batches.clear() self._pending_text_batch_tasks.clear() self._pending_text_batches.clear() - if self._polling_error_task is not current_task: + if getattr(self, "_polling_error_task", None) is not current_task: self._polling_error_task = None + if getattr(self, "_polling_progress_verifier_task", None) is not current_task: + self._polling_progress_verifier_task = None async def disconnect(self) -> None: """Stop polling/webhook, cancel pending delayed deliveries, and disconnect.""" @@ -3591,6 +3739,42 @@ class TelegramAdapter(BasePlatformAdapter): # that wins the race against teardown and prevents new delayed tasks # from being scheduled by late update handlers. self._mark_disconnected() + self._polling_teardown_started = True + self._polling_progress_accepting = False + self._polling_generation = getattr(self, "_polling_generation", 0) + 1 + self._polling_progress_event = asyncio.Event() + self._send_path_degraded = True + + # Recovery can be suspended in stop/drain/start while disconnect begins. + # Cancel and await both polling lifecycle owners immediately after the + # fence, before any other teardown await lets them start a new generation. + current_task = asyncio.current_task() + lifecycle_tasks: list[asyncio.Task] = [] + lifecycle_seen: set[int] = set() + for task in ( + getattr(self, "_polling_error_task", None), + getattr(self, "_polling_progress_verifier_task", None), + ): + if not task or task.done() or task is current_task: + continue + marker = id(task) + if marker in lifecycle_seen: + continue + lifecycle_seen.add(marker) + task.cancel() + if asyncio.isfuture(task) or asyncio.iscoroutine(task): + lifecycle_tasks.append(task) + if lifecycle_tasks: + await asyncio.gather(*lifecycle_tasks, return_exceptions=True) + if getattr(self, "_polling_error_task", None) is not current_task: + self._polling_error_task = None + if getattr(self, "_polling_progress_verifier_task", None) is not current_task: + self._polling_progress_verifier_task = None + + # Cancellation callbacks may have run while awaited; the teardown fence + # remains authoritative regardless of their finalizers. + self._polling_progress_accepting = False + self._send_path_degraded = True # Cancel deferred post-connect housekeeping (command-menu / DM-topic / # status-indicator Bot API calls) so it cannot fire into a half-torn-down @@ -3604,10 +3788,11 @@ class TelegramAdapter(BasePlatformAdapter): # Cancel the heartbeat before tearing down the app so the probe task # cannot fire get_me() into a half-shutdown bot client. - if self._polling_heartbeat_task and not self._polling_heartbeat_task.done(): - self._polling_heartbeat_task.cancel() + polling_heartbeat_task = getattr(self, "_polling_heartbeat_task", None) + if polling_heartbeat_task and not polling_heartbeat_task.done(): + polling_heartbeat_task.cancel() try: - await self._polling_heartbeat_task + await polling_heartbeat_task except asyncio.CancelledError: pass self._polling_heartbeat_task = None diff --git a/tests/gateway/test_telegram_conflict.py b/tests/gateway/test_telegram_conflict.py index 5868a0c56ff5..e00a0f1d33c6 100644 --- a/tests/gateway/test_telegram_conflict.py +++ b/tests/gateway/test_telegram_conflict.py @@ -137,14 +137,13 @@ async def test_polling_conflict_retries_before_fatal(monkeypatch): # First conflict: should retry, NOT be fatal captured["error_callback"](conflict("Conflict: terminated by other getUpdates request")) - await asyncio.sleep(0) - await asyncio.sleep(0) - # Give the scheduled task a chance to run - for _ in range(10): - await asyncio.sleep(0) + await adapter._polling_error_task assert adapter.has_fatal_error is False, "First conflict should not be fatal" - assert adapter._polling_conflict_count == 0, "Count should reset after successful retry" + assert adapter._polling_conflict_count == 1, ( + "Count must remain until the retried generation makes getUpdates progress" + ) + assert adapter._send_path_degraded is True # connect() now starts a lifetime _polling_heartbeat_loop task. With # asyncio.sleep mocked to instant above, it must not be left running or it @@ -152,6 +151,54 @@ async def test_polling_conflict_retries_before_fatal(monkeypatch): await _cancel_heartbeat(adapter) +@pytest.mark.asyncio +async def test_current_generation_conflicts_accumulate_after_start_returns(monkeypatch): + """A later async 409 must advance the retry ladder after PTB start returns.""" + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="***")) + callbacks = [] + conflict_tasks = [] + + async def capture_start(**kwargs): + callbacks.append(kwargs["error_callback"]) + + updater = SimpleNamespace( + start_polling=AsyncMock(side_effect=capture_start), + stop=AsyncMock(), + running=False, + ) + app = SimpleNamespace(updater=updater) + adapter._app = app + adapter._drain_polling_connections = AsyncMock() + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + + def dispatch_conflict(error): + conflict_tasks.append( + asyncio.create_task(adapter._handle_polling_conflict(error)) + ) + + adapter._polling_error_callback_ref = dispatch_conflict + await adapter._start_polling_once( + app, + drop_pending_updates=False, + error_callback=dispatch_conflict, + ) + conflict = type("Conflict", (Exception,), {}) + + try: + callbacks[0](conflict("first async conflict")) + await conflict_tasks[-1] + assert adapter._polling_conflict_count == 1 + + callbacks[1](conflict("second async conflict")) + await conflict_tasks[-1] + assert adapter._polling_conflict_count == 2 + finally: + verifier = adapter._polling_progress_verifier_task + if verifier is not None and not verifier.done(): + verifier.cancel() + await asyncio.gather(verifier, return_exceptions=True) + + @pytest.mark.asyncio async def test_polling_conflict_becomes_fatal_after_retries(monkeypatch): """After exhausting retries, the conflict should become fatal.""" diff --git a/tests/gateway/test_telegram_network_reconnect.py b/tests/gateway/test_telegram_network_reconnect.py index c7e274d74ebb..c1c10726755d 100644 --- a/tests/gateway/test_telegram_network_reconnect.py +++ b/tests/gateway/test_telegram_network_reconnect.py @@ -35,6 +35,7 @@ def _ensure_telegram_mock(): _ensure_telegram_mock() +from plugins.platforms.telegram import adapter as tg_adapter # noqa: E402 from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: E402 @@ -50,6 +51,13 @@ def _make_adapter() -> TelegramAdapter: return TelegramAdapter(PlatformConfig(enabled=True, token="test-token")) +async def _complete_current_polling_generation(adapter: TelegramAdapter) -> None: + verifier = adapter._polling_progress_verifier_task + adapter._record_polling_progress(adapter._polling_generation) + if verifier is not None: + await verifier + + @pytest.mark.asyncio async def test_reconnect_self_schedules_on_start_polling_failure(): """ @@ -157,9 +165,9 @@ async def test_reconnect_chained_retry_updates_polling_error_task(): @pytest.mark.asyncio -async def test_reconnect_success_resets_error_count(): +async def test_reconnect_success_waits_for_progress_to_reset_error_count(): """ - When start_polling() succeeds, _polling_network_error_count should reset to 0. + start_polling() return alone cannot reset the network-error count. """ adapter = _make_adapter() adapter._polling_network_error_count = 3 @@ -177,7 +185,12 @@ async def test_reconnect_success_resets_error_count(): with patch("asyncio.sleep", new_callable=AsyncMock): await adapter._handle_polling_network_error(Exception("Bad Gateway")) + assert adapter._polling_network_error_count == 4 + assert adapter._send_path_degraded is True + + await _complete_current_polling_generation(adapter) assert adapter._polling_network_error_count == 0 + assert adapter._send_path_degraded is False # Clean up the heartbeat-probe task scheduled after a successful reconnect. pending = [t for t in adapter._background_tasks if not t.done()] @@ -263,6 +276,8 @@ async def test_reconnect_drains_polling_request_only(): # Reconnect must still succeed mock_app.updater.start_polling.assert_called_once() + assert adapter._polling_network_error_count == 2 + await _complete_current_polling_generation(adapter) assert adapter._polling_network_error_count == 0 @@ -283,6 +298,8 @@ async def test_reconnect_continues_if_drain_fails(): # start_polling must still be called despite drain failure mock_app.updater.start_polling.assert_called_once() + assert adapter._polling_network_error_count == 2 + await _complete_current_polling_generation(adapter) assert adapter._polling_network_error_count == 0 @@ -339,10 +356,9 @@ async def test_drain_helper_noop_without_app(): @pytest.mark.asyncio -async def test_heartbeat_probe_no_op_when_polling_healthy(): +async def test_polling_verifier_exits_on_matching_progress(monkeypatch): """ - Probe scheduled after a successful reconnect: Updater.running=True and - bot.get_me() returns quickly → recovery confirmed, no further action. + Matching getUpdates progress exits without probing the general path. """ adapter = _make_adapter() @@ -355,19 +371,20 @@ async def test_heartbeat_probe_no_op_when_polling_healthy(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + adapter._record_polling_progress(generation) + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) - mock_app.bot.get_me.assert_awaited_once() + mock_app.bot.get_me.assert_not_awaited() adapter._handle_polling_network_error.assert_not_awaited() @pytest.mark.asyncio -async def test_heartbeat_probe_reenters_ladder_when_updater_not_running(): +async def test_heartbeat_probe_reenters_ladder_when_updater_not_running(monkeypatch): """ - If Updater.running has flipped to False by the heartbeat delay, treat - as wedged: re-enter the reconnect ladder. + If Updater.running is False at the progress deadline, re-enter recovery. """ adapter = _make_adapter() @@ -380,9 +397,10 @@ async def test_heartbeat_probe_reenters_ladder_when_updater_not_running(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) mock_app.bot.get_me.assert_not_called() # Recovery is scheduled through _schedule_polling_recovery (#63243), so @@ -397,7 +415,7 @@ async def test_heartbeat_probe_reenters_ladder_when_updater_not_running(): @pytest.mark.asyncio -async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out(): +async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out(monkeypatch): """ If bot.get_me() hangs longer than PROBE_TIMEOUT, treat as wedged. Simulates the connection-pool wedge that motivated this fix. @@ -416,15 +434,16 @@ async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) async def fast_wait_for(coro, timeout): if asyncio.iscoroutine(coro): coro.close() raise asyncio.TimeoutError() - with patch("asyncio.sleep", new_callable=AsyncMock): - with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", new=fast_wait_for): - await adapter._verify_polling_after_reconnect() + with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", new=fast_wait_for): + await adapter._verify_polling_after_reconnect(generation, progress) task = adapter._polling_error_task assert task is not None @@ -433,7 +452,7 @@ async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out(): @pytest.mark.asyncio -async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error(): +async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error(monkeypatch): """ Any exception raised by bot.get_me() (NetworkError, ConnectionError, etc.) should re-enter the reconnect ladder with the original exception. @@ -449,9 +468,10 @@ async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) task = adapter._polling_error_task assert task is not None @@ -466,7 +486,7 @@ async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error(): @pytest.mark.asyncio -async def test_heartbeat_probe_ignores_auth_errors(): +async def test_heartbeat_probe_ignores_auth_errors(monkeypatch): """ Auth/validation failures from the post-reconnect probe must not enter the network-reconnect ladder (#63243): a revoked token would otherwise churn @@ -487,16 +507,17 @@ async def test_heartbeat_probe_ignores_auth_errors(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) assert adapter._polling_error_task is None adapter._handle_polling_network_error.assert_not_awaited() @pytest.mark.asyncio -async def test_heartbeat_probe_defers_to_inflight_recovery(): +async def test_heartbeat_probe_defers_to_inflight_recovery(monkeypatch): """ A probe failure while another recovery is mid-flight must not start a second concurrent stop/drain/start_polling sequence (#63243) — overlapping @@ -517,16 +538,17 @@ async def test_heartbeat_probe_defers_to_inflight_recovery(): adapter._polling_error_task = inflight adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) assert adapter._polling_error_task is inflight adapter._handle_polling_network_error.assert_not_awaited() @pytest.mark.asyncio -async def test_heartbeat_probe_skips_when_already_fatal(): +async def test_heartbeat_probe_skips_when_already_fatal(monkeypatch): """ If the adapter is already in fatal-error state by the time the probe delay elapses, the probe should bail without further action. @@ -539,9 +561,10 @@ async def test_heartbeat_probe_skips_when_already_fatal(): adapter._app = mock_app adapter._handle_polling_network_error = AsyncMock() + generation, progress = adapter._begin_polling_generation() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0) - with patch("asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() + await adapter._verify_polling_after_reconnect(generation, progress) mock_app.bot.get_me.assert_not_called() adapter._handle_polling_network_error.assert_not_awaited() @@ -1060,4 +1083,3 @@ async def test_handle_polling_network_error_updater_stop_timeout(): # The reconnect ladder must have advanced past the hung stop(). assert drain_called, "_drain_polling_connections was not called after stop() timeout" assert start_polling_called, "start_polling was not called after stop() timeout" - diff --git a/tests/gateway/test_telegram_polling_progress.py b/tests/gateway/test_telegram_polling_progress.py index 717eaaf868a5..fdf00e17c99b 100644 --- a/tests/gateway/test_telegram_polling_progress.py +++ b/tests/gateway/test_telegram_polling_progress.py @@ -1,6 +1,8 @@ """Behavior contract for generation-safe Telegram polling progress.""" import asyncio +import json +from unittest.mock import AsyncMock, MagicMock import pytest @@ -14,6 +16,11 @@ class _ControlledRequest: instances = [] + @staticmethod + def parse_json_payload(payload): + """Match PTB's response authority used by the progress observer.""" + return json.loads(payload.decode("utf-8", "replace")) + def __init__(self, *args, result=None, error=None, entered=None, release=None, **kwargs): self.result = result self.error = error @@ -37,18 +44,183 @@ def _make_adapter() -> TelegramAdapter: return TelegramAdapter(PlatformConfig(enabled=True, token="test-token")) +def _mock_polling_app(*, get_me=None): + app = MagicMock() + app.updater = MagicMock() + app.updater.running = True + app.updater.stop = AsyncMock() + app.updater.start_polling = AsyncMock() + app.bot = MagicMock() + app.bot.get_me = get_me or AsyncMock(return_value=MagicMock()) + app.running = False + app.shutdown = AsyncMock() + return app + + +class _LifecycleBuilder: + def __init__(self, app): + self.app = app + self.polling_request = None + + def token(self, _token): + return self + + def request(self, _request): + return self + + def get_updates_request(self, request): + self.polling_request = request + return self + + def build(self): + return self.app + + +def _lifecycle_app(): + app = MagicMock() + app.updater = MagicMock() + app.updater.running = True + app.updater.start_polling = AsyncMock() + app.updater.start_webhook = AsyncMock() + app.updater.stop = AsyncMock() + app.bot = MagicMock() + app.bot.delete_webhook = AsyncMock() + app.initialize = AsyncMock() + app.start = AsyncMock() + app.stop = AsyncMock() + app.shutdown = AsyncMock() + app.running = True + return app + + +def _configure_lifecycle_connect(monkeypatch, adapter, apps): + builders = [_LifecycleBuilder(app) for app in apps] + remaining = iter(builders) + + class _Application: + @staticmethod + def builder(): + return next(remaining) + + async def _no_fallback_ips(): + return [] + + monkeypatch.setattr(tg_adapter, "Application", _Application) + monkeypatch.setattr(tg_adapter, "HTTPXRequest", _ControlledRequest) + monkeypatch.setattr(tg_adapter, "discover_fallback_ips", _no_fallback_ips) + monkeypatch.setattr(tg_adapter, "resolve_proxy_url", lambda *args, **kwargs: None) + monkeypatch.setattr(adapter, "_acquire_platform_lock", lambda *args, **kwargs: True) + monkeypatch.setattr(adapter, "_release_platform_lock", MagicMock()) + monkeypatch.setattr(adapter, "_fallback_ips", lambda: []) + monkeypatch.setattr(adapter, "_start_post_connect_housekeeping", MagicMock()) + return builders + + +async def _cancel_task(task): + if task is None or task.done(): + return + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def _request_for_generation(generation, request, *args): + """Run a direct request double under the production polling context.""" + generation_context = tg_adapter._POLLING_GENERATION_CONTEXT + token = generation_context.set(generation) + try: + return await request.do_request(*args) + finally: + generation_context.reset(token) + + +@pytest.mark.asyncio +async def test_polling_disconnect_webhook_reconnect_heals_webhook_send_path(monkeypatch): + adapter = _make_adapter() + polling_app = _lifecycle_app() + webhook_app = _lifecycle_app() + _configure_lifecycle_connect(monkeypatch, adapter, [polling_app, webhook_app]) + monkeypatch.delenv("TELEGRAM_WEBHOOK_URL", raising=False) + monkeypatch.delenv("TELEGRAM_WEBHOOK_SECRET", raising=False) + + assert await adapter.connect() is True + assert adapter._webhook_mode is False + assert adapter._send_path_degraded is True + await adapter.disconnect() + + monkeypatch.setenv("TELEGRAM_WEBHOOK_URL", "https://example.test/telegram") + monkeypatch.setenv("TELEGRAM_WEBHOOK_SECRET", "test-secret") + try: + assert await adapter.connect(is_reconnect=True) is True + webhook_app.updater.start_webhook.assert_awaited_once() + assert adapter._webhook_mode is True + assert adapter._polling_progress_accepting is False + assert adapter._send_path_degraded is False + finally: + await adapter.disconnect() + + +@pytest.mark.asyncio +async def test_webhook_disconnect_polling_reconnect_resets_mode_and_waits_for_progress( + monkeypatch, +): + adapter = _make_adapter() + webhook_app = _lifecycle_app() + polling_app = _lifecycle_app() + builders = _configure_lifecycle_connect( + monkeypatch, adapter, [webhook_app, polling_app] + ) + heartbeat_started = asyncio.Event() + heartbeat_modes = [] + + async def heartbeat(): + heartbeat_modes.append(adapter._webhook_mode) + heartbeat_started.set() + await asyncio.Event().wait() + + monkeypatch.setattr(adapter, "_polling_heartbeat_loop", heartbeat) + monkeypatch.setenv("TELEGRAM_WEBHOOK_URL", "https://example.test/telegram") + monkeypatch.setenv("TELEGRAM_WEBHOOK_SECRET", "test-secret") + + assert await adapter.connect() is True + assert adapter._webhook_mode is True + assert adapter._polling_heartbeat_task is None + await adapter.disconnect() + + monkeypatch.delenv("TELEGRAM_WEBHOOK_URL") + monkeypatch.delenv("TELEGRAM_WEBHOOK_SECRET") + try: + assert await adapter.connect(is_reconnect=True) is True + assert adapter._webhook_mode is False + assert adapter._polling_heartbeat_task is not None + assert not adapter._polling_heartbeat_task.done() + await asyncio.wait_for(heartbeat_started.wait(), timeout=1) + assert heartbeat_modes == [False] + assert adapter._send_path_degraded is True + + generation = adapter._polling_generation + polling_request = builders[1].polling_request + polling_request.result = (200, b'{"ok":true,"result":[]}') + await _request_for_generation(generation, polling_request, "getUpdates") + await asyncio.wait_for(adapter._polling_progress_verifier_task, timeout=1) + assert adapter._send_path_degraded is False + finally: + await adapter.disconnect() + + @pytest.mark.asyncio async def test_current_polling_generation_success_records_progress(): adapter = _make_adapter() generation, progress = adapter._begin_polling_generation() adapter._polling_network_error_count = 3 - request = _ControlledRequest(result=(200, b'{"ok":true}')) + request = _ControlledRequest(result=(200, b'{"ok":true,"result":[]}')) instrumented = adapter._instrument_polling_request(request) - result = await instrumented.do_request("https://api.telegram.org/getUpdates") + result = await _request_for_generation( + generation, instrumented, "https://api.telegram.org/getUpdates" + ) assert instrumented is request - assert result == (200, b'{"ok":true}') + assert result == (200, b'{"ok":true,"result":[]}') assert progress.is_set() assert adapter._polling_network_error_count == 0 assert adapter._send_path_degraded is False @@ -59,14 +231,16 @@ async def test_current_polling_generation_success_records_progress(): @pytest.mark.parametrize("error_type", [RuntimeError, asyncio.CancelledError]) async def test_unsuccessful_polling_request_does_not_record_progress(error_type): adapter = _make_adapter() - _, progress = adapter._begin_polling_generation() + generation, progress = adapter._begin_polling_generation() adapter._polling_network_error_count = 3 request = adapter._instrument_polling_request( _ControlledRequest(error=error_type("request did not complete")) ) with pytest.raises(error_type): - await request.do_request("https://api.telegram.org/getUpdates") + await _request_for_generation( + generation, request, "https://api.telegram.org/getUpdates" + ) assert not progress.is_set() assert adapter._polling_network_error_count == 3 @@ -76,13 +250,15 @@ async def test_unsuccessful_polling_request_does_not_record_progress(error_type) @pytest.mark.asyncio async def test_http_error_response_does_not_record_polling_progress(): adapter = _make_adapter() - _, progress = adapter._begin_polling_generation() + generation, progress = adapter._begin_polling_generation() adapter._polling_network_error_count = 3 request = adapter._instrument_polling_request( _ControlledRequest(result=(500, b"bad")) ) - result = await request.do_request("https://api.telegram.org/getUpdates") + result = await _request_for_generation( + generation, request, "https://api.telegram.org/getUpdates" + ) assert result == (500, b"bad") assert not progress.is_set() @@ -155,17 +331,444 @@ async def test_late_previous_generation_completion_cannot_heal_current_generatio entered = asyncio.Event() release = asyncio.Event() request = adapter._instrument_polling_request( - _ControlledRequest(result=(200, b'{"ok":true}'), entered=entered, release=release) + _ControlledRequest( + result=(200, b'{"ok":true,"result":[]}'), + entered=entered, + release=release, + ) ) - completion = asyncio.create_task(request.do_request("getUpdates")) + completion = asyncio.create_task( + _request_for_generation(generation_1, request, "getUpdates") + ) await entered.wait() generation_2, progress_2 = adapter._begin_polling_generation() adapter._polling_network_error_count = 4 release.set() - assert await completion == (200, b'{"ok":true}') + assert await completion == (200, b'{"ok":true,"result":[]}') assert generation_2 == generation_1 + 1 assert not progress_2.is_set() assert adapter._polling_network_error_count == 4 assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_old_polling_child_keeps_generation_when_request_entry_is_delayed(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + release_old_request = asyncio.Event() + start_count = 0 + old_child = None + request = adapter._instrument_polling_request( + _ControlledRequest(result=(200, b'{"ok":true,"result":[]}')) + ) + + async def start_polling(**_kwargs): + nonlocal start_count, old_child + start_count += 1 + if start_count == 1: + + async def delayed_old_request(): + await release_old_request.wait() + return await request.do_request("getUpdates") + + old_child = asyncio.create_task(delayed_old_request()) + + adapter._app.updater.start_polling = start_polling + + await adapter._start_polling_once( + adapter._app, + drop_pending_updates=False, + error_callback=MagicMock(), + ) + generation_1 = adapter._polling_generation + await adapter._start_polling_once( + adapter._app, + drop_pending_updates=False, + error_callback=MagicMock(), + ) + generation_2 = adapter._polling_generation + progress_2 = adapter._polling_progress_event + verifier_2 = adapter._polling_progress_verifier_task + + try: + release_old_request.set() + assert await old_child == (200, b'{"ok":true,"result":[]}') + + assert generation_2 == generation_1 + 1 + assert not progress_2.is_set() + assert adapter._send_path_degraded is True + finally: + await _cancel_task(verifier_2) + + +@pytest.mark.asyncio +async def test_error_callback_is_bound_to_its_polling_generation(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + callbacks = [] + delegated = MagicMock() + + async def capture_start(**kwargs): + callbacks.append(kwargs["error_callback"]) + + adapter._app.updater.start_polling = capture_start + await adapter._start_polling_once( + adapter._app, + drop_pending_updates=False, + error_callback=delegated, + ) + await adapter._start_polling_once( + adapter._app, + drop_pending_updates=False, + error_callback=delegated, + ) + verifier = adapter._polling_progress_verifier_task + stale_error = ConnectionError("stale generation") + current_error = ConnectionError("current generation") + + try: + callbacks[0](stale_error) + delegated.assert_not_called() + + callbacks[1](current_error) + delegated.assert_called_once_with(current_error) + finally: + await _cancel_task(verifier) + + +@pytest.mark.asyncio +async def test_cold_start_waits_for_get_updates_progress_before_healing(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + + started = await adapter._start_polling_resilient( + drop_pending_updates=True, + error_callback=MagicMock(), + ) + + verifier = adapter._polling_progress_verifier_task + assert started is True + assert adapter._app.updater.running is True + assert adapter._send_path_degraded is True + assert verifier is not None and not verifier.done() + assert verifier in adapter._background_tasks + assert [task for task in adapter._background_tasks if not task.done()] == [verifier] + await _cancel_task(verifier) + + +@pytest.mark.asyncio +async def test_matching_get_updates_progress_heals_and_stops_verifier(monkeypatch): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + recovery = MagicMock() + monkeypatch.setattr(adapter, "_schedule_polling_recovery", recovery) + + await adapter._start_polling_resilient( + drop_pending_updates=False, + error_callback=MagicMock(), + ) + verifier = adapter._polling_progress_verifier_task + request = adapter._instrument_polling_request( + _ControlledRequest(result=(200, b'{"ok":true,"result":[]}')) + ) + await _request_for_generation( + adapter._polling_generation, request, "getUpdates" + ) + await asyncio.wait_for(verifier, timeout=1) + + assert adapter._send_path_degraded is False + assert verifier.done() + recovery.assert_not_called() + + +@pytest.mark.asyncio +async def test_general_path_success_without_get_updates_progress_recovers_once(monkeypatch): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + recovery = MagicMock() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0.01, raising=False) + monkeypatch.setattr(adapter, "_schedule_polling_recovery", recovery) + + await adapter._start_polling_resilient( + drop_pending_updates=False, + error_callback=MagicMock(), + ) + verifier = adapter._polling_progress_verifier_task + await asyncio.wait_for(verifier, timeout=1) + + assert adapter._app.bot.get_me.await_count == 1 + recovery.assert_called_once() + error = recovery.call_args.args[0] + assert isinstance(error, RuntimeError) + assert str(error) == "getUpdates made no progress before verifier deadline" + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("probe_error", "should_recover"), + [ + (ConnectionError("pool wedged"), True), + (type("InvalidToken", (Exception,), {})("token revoked"), False), + ], +) +async def test_general_path_error_only_recovers_connectivity_failures( + monkeypatch, probe_error, should_recover +): + adapter = _make_adapter() + adapter._app = _mock_polling_app(get_me=AsyncMock(side_effect=probe_error)) + recovery = MagicMock() + monkeypatch.setattr(tg_adapter, "_POLLING_PROGRESS_TIMEOUT", 0.01, raising=False) + monkeypatch.setattr(adapter, "_schedule_polling_recovery", recovery) + + await adapter._start_polling_resilient( + drop_pending_updates=False, + error_callback=MagicMock(), + ) + await asyncio.wait_for(adapter._polling_progress_verifier_task, timeout=1) + + assert recovery.called is should_recover + if should_recover: + assert recovery.call_args.args[0] is probe_error + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retry_kind", ["network", "conflict"]) +async def test_retry_start_requires_matching_progress_to_heal(monkeypatch, retry_kind): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + adapter._polling_error_callback_ref = MagicMock() + adapter._polling_network_error_count = 3 + monkeypatch.setattr(tg_adapter.asyncio, "sleep", AsyncMock()) + + if retry_kind == "network": + await adapter._handle_polling_network_error(ConnectionError("offline")) + assert adapter._polling_network_error_count == 4 + else: + await adapter._handle_polling_conflict( + RuntimeError("Conflict: terminated by other getUpdates request") + ) + assert adapter._polling_network_error_count == 3 + assert adapter._polling_conflict_count == 1 + + generation = adapter._polling_generation + verifier = adapter._polling_progress_verifier_task + assert generation > 0 + assert verifier is not None and not verifier.done() + assert adapter._send_path_degraded is True + + request = adapter._instrument_polling_request( + _ControlledRequest(result=(200, b'{"ok":true,"result":[]}')) + ) + await _request_for_generation(generation, request, "getUpdates") + await asyncio.wait_for(verifier, timeout=1) + assert adapter._polling_network_error_count == 0 + assert adapter._polling_conflict_count == 0 + assert adapter._send_path_degraded is False + + +@pytest.mark.asyncio +async def test_repeated_starts_replace_verifier_and_stale_verifier_cannot_heal(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + + await adapter._start_polling_resilient( + drop_pending_updates=False, error_callback=MagicMock() + ) + generation_1 = adapter._polling_generation + progress_1 = adapter._polling_progress_event + verifier_1 = adapter._polling_progress_verifier_task + + await adapter._start_polling_resilient( + drop_pending_updates=False, error_callback=MagicMock() + ) + verifier_2 = adapter._polling_progress_verifier_task + await asyncio.sleep(0) + + assert adapter._polling_generation == generation_1 + 1 + assert verifier_1.cancelled() + assert verifier_2 is not verifier_1 and not verifier_2.done() + assert [task for task in adapter._background_tasks if not task.done()] == [verifier_2] + + progress_1.set() + await asyncio.sleep(0) + assert adapter._send_path_degraded is True + await _cancel_task(verifier_2) + + +@pytest.mark.asyncio +async def test_disconnect_fences_verifier_and_late_progress_completion(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + await adapter._start_polling_resilient( + drop_pending_updates=False, error_callback=MagicMock() + ) + generation = adapter._polling_generation + verifier = adapter._polling_progress_verifier_task + + entered = asyncio.Event() + release = asyncio.Event() + request = adapter._instrument_polling_request( + _ControlledRequest( + result=(200, b'{"ok":true,"result":[]}'), + entered=entered, + release=release, + ) + ) + completion = asyncio.create_task( + _request_for_generation(generation, request, "getUpdates") + ) + await entered.wait() + + await adapter.disconnect() + + assert verifier.done() + assert adapter._polling_progress_verifier_task is None + assert adapter._polling_progress_accepting is False + assert adapter._polling_generation > generation + assert adapter._send_path_degraded is True + + release.set() + assert await completion == (200, b'{"ok":true,"result":[]}') + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_disconnect_during_polling_start_returns_false_without_recovery(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + entered = asyncio.Event() + release = asyncio.Event() + recovery = MagicMock() + adapter._schedule_polling_recovery = recovery + + async def blocked_start(**_kwargs): + entered.set() + await release.wait() + + adapter._app.updater.start_polling = blocked_start + start = asyncio.create_task( + adapter._start_polling_resilient( + drop_pending_updates=False, + error_callback=MagicMock(), + ) + ) + await entered.wait() + + await adapter.disconnect() + release.set() + result = await start + + assert result is False + recovery.assert_not_called() + assert adapter._polling_error_task is None + assert not adapter.has_fatal_error + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_caller_cancellation_during_polling_start_still_propagates(): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + entered = asyncio.Event() + release = asyncio.Event() + recovery = MagicMock() + adapter._schedule_polling_recovery = recovery + + async def blocked_start(**_kwargs): + entered.set() + await release.wait() + + adapter._app.updater.start_polling = blocked_start + start = asyncio.create_task( + adapter._start_polling_resilient( + drop_pending_updates=False, + error_callback=MagicMock(), + ) + ) + await entered.wait() + + start.cancel() + with pytest.raises(asyncio.CancelledError): + await start + + assert start.cancelled() + recovery.assert_not_called() + assert not adapter.has_fatal_error + assert adapter._send_path_degraded is True + await adapter.disconnect() + + +@pytest.mark.asyncio +async def test_disconnect_cancels_recovery_before_it_can_rearm_progress(monkeypatch): + adapter = _make_adapter() + adapter._app = _mock_polling_app() + adapter._app.updater.running = False + adapter._polling_error_callback_ref = MagicMock() + + drain_entered = asyncio.Event() + release_drain = asyncio.Event() + start_entered = asyncio.Event() + release_start = asyncio.Event() + teardown_paused = asyncio.Event() + release_teardown = asyncio.Event() + + async def immediate_backoff(_delay): + return None + + async def blocked_drain(): + drain_entered.set() + await release_drain.wait() + + async def blocked_start_polling(**_kwargs): + start_entered.set() + await release_start.wait() + + async def blocked_status_indicator(*, online): + assert online is False + teardown_paused.set() + await release_teardown.wait() + + monkeypatch.setattr(tg_adapter.asyncio, "sleep", immediate_backoff) + monkeypatch.setattr(adapter, "_drain_polling_connections", blocked_drain) + monkeypatch.setattr( + adapter._app.updater, "start_polling", blocked_start_polling + ) + monkeypatch.setattr(adapter, "_set_status_indicator", blocked_status_indicator) + + recovery = asyncio.create_task( + adapter._handle_polling_network_error(ConnectionError("offline")) + ) + adapter._polling_error_task = recovery + await drain_entered.wait() + + disconnect = asyncio.create_task(adapter.disconnect()) + await teardown_paused.wait() + + try: + # Before the fix, disconnect pauses here before cancelling recovery. + # Releasing the recovery lets it begin a fresh generation after the + # teardown fence, and matching progress can then heal the adapter. + if not recovery.done(): + release_drain.set() + await start_entered.wait() + + rearmed_after_fence = adapter._polling_progress_accepting + adapter._record_polling_progress(adapter._polling_generation) + + assert rearmed_after_fence is False + assert getattr(adapter, "_polling_teardown_started", False) is True + assert adapter._polling_progress_accepting is False + assert adapter._send_path_degraded is True + assert recovery.done() + finally: + release_drain.set() + release_start.set() + release_teardown.set() + for task in (recovery, disconnect): + if not task.done(): + task.cancel() + await asyncio.gather(recovery, disconnect, return_exceptions=True) diff --git a/tests/gateway/test_telegram_send_path_health.py b/tests/gateway/test_telegram_send_path_health.py index 37c76152c913..b533b3492dd4 100644 --- a/tests/gateway/test_telegram_send_path_health.py +++ b/tests/gateway/test_telegram_send_path_health.py @@ -65,49 +65,32 @@ async def test_send_short_circuits_when_path_degraded(): @pytest.mark.asyncio -async def test_reconnect_storm_sets_and_heartbeat_clears_flag(monkeypatch): - """_handle_polling_network_error sets the flag while reconnecting; if the - reconnect attempt itself raises (polling not yet healthy), the flag stays - True until a later successful heartbeat probe in - _verify_polling_after_reconnect clears it.""" +async def test_get_me_success_without_polling_progress_does_not_heal(monkeypatch): + """A responsive general Bot API path is not proof that getUpdates works.""" adapter = _make_adapter() adapter._app = MagicMock() adapter._app.updater = MagicMock() adapter._app.updater.running = True - adapter._app.updater.stop = AsyncMock() - # First start_polling attempt fails — the reconnect handler must leave the - # flag set (path still unhealthy) and not clear it prematurely. - adapter._app.updater.start_polling = AsyncMock(side_effect=OSError("still down")) adapter._app.bot = MagicMock() adapter._app.bot.get_me = AsyncMock(return_value=MagicMock()) - adapter._polling_error_callback_ref = AsyncMock() - monkeypatch.setattr( - "plugins.platforms.telegram.adapter.Update", MagicMock(ALL_TYPES=[]) - ) - # Suppress the self-rescheduled retry so the test doesn't recurse. - monkeypatch.setattr( - "plugins.platforms.telegram.adapter.asyncio.ensure_future", MagicMock() - ) - with patch("plugins.platforms.telegram.adapter.asyncio.sleep", new_callable=AsyncMock): - await adapter._handle_polling_network_error(OSError("Bad Gateway")) - # start_polling failed → path still degraded. + generation, progress = adapter._begin_polling_generation() + recovery = MagicMock() + monkeypatch.setattr(adapter, "_schedule_polling_recovery", recovery) + monkeypatch.setattr( + "plugins.platforms.telegram.adapter._POLLING_PROGRESS_TIMEOUT", 0, + raising=False, + ) + await adapter._verify_polling_after_reconnect(generation, progress) + + adapter._app.bot.get_me.assert_awaited_once() + recovery.assert_called_once() assert adapter._send_path_degraded is True - # Now the deferred probe runs against a recovered (running) updater and - # a responsive bot — it clears the flag. - adapter._app.updater.running = True - with patch("plugins.platforms.telegram.adapter.asyncio.sleep", new_callable=AsyncMock): - await adapter._verify_polling_after_reconnect() - assert adapter._send_path_degraded is False - @pytest.mark.asyncio -async def test_successful_reconnect_clears_flag_without_probe(monkeypatch): - """Regression for #35205: a successful start_polling() clears - _send_path_degraded immediately, so outbound sends are not blocked for - the full HEARTBEAT_PROBE_DELAY window (and never get stuck True if the - deferred probe is never scheduled / never runs).""" +async def test_successful_reconnect_waits_for_get_updates_progress(monkeypatch): + """start_polling() return alone cannot heal; matching progress can.""" adapter = _make_adapter() adapter._app = MagicMock() adapter._app.updater = MagicMock() @@ -120,17 +103,18 @@ async def test_successful_reconnect_clears_flag_without_probe(monkeypatch): monkeypatch.setattr( "plugins.platforms.telegram.adapter.Update", MagicMock(ALL_TYPES=[]) ) - # Don't let the deferred probe run — prove the clear happens in the - # reconnect handler itself, not in _verify_polling_after_reconnect. - monkeypatch.setattr( - adapter, "_verify_polling_after_reconnect", AsyncMock() - ) - with patch("plugins.platforms.telegram.adapter.asyncio.sleep", new_callable=AsyncMock): await adapter._handle_polling_network_error(OSError("Bad Gateway")) + verifier = adapter._polling_progress_verifier_task + assert adapter._send_path_degraded is True + assert adapter._polling_network_error_count == 1 + blocked = await adapter.send("123", "hello") + assert blocked.success is False + + adapter._record_polling_progress(adapter._polling_generation) + await verifier assert adapter._send_path_degraded is False assert adapter._polling_network_error_count == 0 - # And send() works again right away. result = await adapter.send("123", "hello") assert result.success is True diff --git a/tests/test_telegram_polling_progress_ptb.py b/tests/test_telegram_polling_progress_ptb.py new file mode 100644 index 000000000000..ad4ed2caad84 --- /dev/null +++ b/tests/test_telegram_polling_progress_ptb.py @@ -0,0 +1,292 @@ +"""Integration coverage for polling progress against the installed PTB runtime.""" + +import asyncio + +import pytest +from telegram.error import Conflict, TelegramError +from telegram.request import BaseRequest + +from gateway.config import PlatformConfig +from plugins.platforms.telegram import adapter as tg_adapter +from plugins.platforms.telegram.adapter import TelegramAdapter + + +class _GeneralRequest(BaseRequest): + @property + def read_timeout(self): + return 10 + + async def initialize(self): + return None + + async def shutdown(self): + return None + + async def do_request(self, url, method, request_data=None, **_kwargs): + if url.endswith("/getMe"): + return ( + 200, + b'{"ok":true,"result":{"id":1,"is_bot":true,' + b'"first_name":"Test","username":"test_bot"}}', + ) + return 200, b'{"ok":true,"result":true}' + + +class _GetUpdatesRequest(BaseRequest): + def __init__(self): + self.initial_conflict_sent = False + self.replacement_enabled = False + self.replacement_progress_sent = False + self.cleanup_calls = 0 + self.block = asyncio.Event() + + @property + def read_timeout(self): + return 10 + + async def initialize(self): + return None + + async def shutdown(self): + return None + + async def do_request(self, url, method, request_data=None, **_kwargs): + parameters = request_data.parameters if request_data is not None else {} + timeout = parameters.get("timeout") + timeout_seconds = ( + timeout.total_seconds() if hasattr(timeout, "total_seconds") else timeout + ) + if timeout_seconds == 0: + self.cleanup_calls += 1 + return 200, b'{"ok":true,"result":[]}' + if not self.initial_conflict_sent: + self.initial_conflict_sent = True + return ( + 409, + b'{"ok":false,"error_code":409,' + b'"description":"Conflict: another getUpdates request"}', + ) + if self.replacement_enabled and not self.replacement_progress_sent: + self.replacement_progress_sent = True + return 200, b'{"ok":true,"result":[]}' + await self.block.wait() + return 200, b'{"ok":true,"result":[]}' + + +class _EnvelopeRequest(BaseRequest): + def __init__(self, payload): + self.payload = payload + + @property + def read_timeout(self): + return 10 + + async def initialize(self): + return None + + async def shutdown(self): + return None + + async def do_request(self, url, method, request_data=None, **_kwargs): + return 200, self.payload + + +async def _cancel_task(task): + if task is None or task.done(): + return + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_real_base_request_invalid_200_body_cannot_record_progress(): + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + request = adapter._instrument_polling_request(_EnvelopeRequest(b"not-json")) + context_token = tg_adapter._POLLING_GENERATION_CONTEXT.set(generation) + + try: + with pytest.raises(TelegramError, match="Invalid server response"): + await request.post("https://api.telegram.org/bot-token/getUpdates") + finally: + tg_adapter._POLLING_GENERATION_CONTEXT.reset(context_token) + + assert not progress.is_set() + assert adapter._polling_network_error_count == 4 + assert adapter._polling_conflict_count == 3 + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_real_base_request_bom_rejected_by_ptb_cannot_record_progress(): + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + request = adapter._instrument_polling_request( + _EnvelopeRequest(b'\xef\xbb\xbf{"ok":true,"result":[]}') + ) + context_token = tg_adapter._POLLING_GENERATION_CONTEXT.set(generation) + + try: + with pytest.raises(TelegramError, match="Invalid server response"): + await request.post("https://api.telegram.org/bot-token/getUpdates") + finally: + tg_adapter._POLLING_GENERATION_CONTEXT.reset(context_token) + + assert not progress.is_set() + assert adapter._polling_network_error_count == 4 + assert adapter._polling_conflict_count == 3 + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_real_base_request_ptb_replacement_decode_records_progress(): + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + request = adapter._instrument_polling_request( + _EnvelopeRequest(b'{"ok":true,"result":[],"note":"\xff"}') + ) + context_token = tg_adapter._POLLING_GENERATION_CONTEXT.set(generation) + + try: + result = await request.post("https://api.telegram.org/bot-token/getUpdates") + finally: + tg_adapter._POLLING_GENERATION_CONTEXT.reset(context_token) + + assert result == [] + assert progress.is_set() + assert adapter._polling_network_error_count == 0 + assert adapter._polling_conflict_count == 0 + assert adapter._send_path_degraded is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("payload", "missing_result"), + [ + (b'{"ok":false,"result":[]}', False), + (b'{"ok":true}', True), + ], +) +async def test_real_base_request_unsuccessful_200_envelope_cannot_record_progress( + payload, missing_result +): + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + request = adapter._instrument_polling_request(_EnvelopeRequest(payload)) + context_token = tg_adapter._POLLING_GENERATION_CONTEXT.set(generation) + + try: + if missing_result: + with pytest.raises(KeyError, match="result"): + await request.post("https://api.telegram.org/bot-token/getUpdates") + else: + assert await request.post( + "https://api.telegram.org/bot-token/getUpdates" + ) == [] + finally: + tg_adapter._POLLING_GENERATION_CONTEXT.reset(context_token) + + assert not progress.is_set() + assert adapter._polling_network_error_count == 4 + assert adapter._polling_conflict_count == 3 + assert adapter._send_path_degraded is True + + +@pytest.mark.asyncio +async def test_real_base_request_valid_success_envelope_records_progress(): + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + generation, progress = adapter._begin_polling_generation() + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + request = adapter._instrument_polling_request( + _EnvelopeRequest(b'{"ok":true,"result":[]}') + ) + context_token = tg_adapter._POLLING_GENERATION_CONTEXT.set(generation) + + try: + result = await request.post( + "https://api.telegram.org/bot-token/getUpdates" + ) + finally: + tg_adapter._POLLING_GENERATION_CONTEXT.reset(context_token) + + assert result == [] + assert progress.is_set() + assert adapter._polling_network_error_count == 0 + assert adapter._polling_conflict_count == 0 + assert adapter._send_path_degraded is False + + +@pytest.mark.asyncio +async def test_real_ptb_stop_cleanup_cannot_heal_recovery_generation(): + assert tg_adapter.TELEGRAM_AVAILABLE is True + adapter = TelegramAdapter(PlatformConfig(enabled=True, token="123456:test-token")) + polling_request = _GetUpdatesRequest() + app = ( + tg_adapter.Application.builder() + .token("123456:test-token") + .request(_GeneralRequest()) + .get_updates_request(adapter._instrument_polling_request(polling_request)) + .build() + ) + adapter._app = app + adapter._polling_network_error_count = 4 + adapter._polling_conflict_count = 3 + callback_called = asyncio.Event() + recovery_task = None + + async def stop_for_recovery(): + await app.updater.stop() + + def schedule_recovery(error): + nonlocal recovery_task + assert isinstance(error, Conflict) + recovery_task = asyncio.create_task(stop_for_recovery()) + callback_called.set() + + await app.initialize() + try: + await adapter._start_polling_once( + app, + drop_pending_updates=False, + error_callback=schedule_recovery, + ) + generation = adapter._polling_generation + progress = adapter._polling_progress_event + await asyncio.wait_for(callback_called.wait(), timeout=2) + await asyncio.wait_for(recovery_task, timeout=3) + + assert polling_request.cleanup_calls == 1 + assert not progress.is_set() + assert adapter._polling_network_error_count == 4 + assert adapter._polling_conflict_count == 3 + assert adapter._send_path_degraded is True + + polling_request.replacement_enabled = True + await adapter._start_polling_once( + app, + drop_pending_updates=False, + error_callback=schedule_recovery, + ) + replacement_generation = adapter._polling_generation + replacement_progress = adapter._polling_progress_event + await asyncio.wait_for(replacement_progress.wait(), timeout=2) + + assert replacement_generation == generation + 1 + assert adapter._polling_network_error_count == 0 + assert adapter._polling_conflict_count == 0 + assert adapter._send_path_degraded is False + finally: + polling_request.block.set() + if app.updater.running: + await app.updater.stop() + await _cancel_task(adapter._polling_progress_verifier_task) + await app.shutdown() From c5aaf7646809f19452281f627df17d6c6a6fe52b Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:52:06 +0530 Subject: [PATCH 068/149] chore(release): map @Roseyco-management in AUTHOR_MAP For PR #63581 salvage (telegram: require getUpdates progress before polling is healthy). SilentKnight87 uses a noreply GitHub email which auto-skips. --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 2c528e15516a..d6568f505e87 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -79,6 +79,7 @@ AUTHOR_MAP = { "poowis2011@hotmail.com": "Umi4Life", # PR #47377 salvage (agent: emit one-shot fallback switch notice on successful fallback so gateway users see model/provider change; #35419) "austin@openvm067.space": "austinlaw076", # PR #57563 partial salvage (auth: lazy per-profile Anthropic OAuth file; gateway: whatsapp_cloud/line added to port-binding platform set) "sunsky.lau@gmail.com": "liuhao1024", # PR #56993 salvage (gateway: process-level HERMES_HOME for pid/lock/status identity files; #56986) + "roseycomanagement@roseyco.co.uk": "Roseyco-management", # PR #63581 salvage (telegram: require getUpdates progress before polling is healthy; #63243, #63766) "gauravsaxena.jaipur@gmail.com": "gauravsaxena1997", # PR #59868 partial salvage (agent: guard response.text against httpx.ResponseNotRead in _summarize_api_error; #59769) "blueirobin02@gmail.com": "irresi", # PR #59048 salvage (gateway: scope reset banners' session info to the serving profile; #59003) "jashlee+microsoft@microsoft.com": "s905060", # PR #57943 salvage (photon: auto-reinstall stale sidecar node_modules when lockfile is newer than npm's install marker; #59169) From adf62065a73fa9682917053dd55197f63c4a43e4 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:05:06 +0530 Subject: [PATCH 069/149] test(telegram): guard PTB integration tests with importorskip CI test slices don't install python-telegram-bot (optional dep), causing a ModuleNotFoundError on collection. Add pytest.importorskip('telegram') before the PTB imports. --- tests/test_telegram_polling_progress_ptb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_telegram_polling_progress_ptb.py b/tests/test_telegram_polling_progress_ptb.py index ad4ed2caad84..11f3858670f4 100644 --- a/tests/test_telegram_polling_progress_ptb.py +++ b/tests/test_telegram_polling_progress_ptb.py @@ -3,6 +3,8 @@ import asyncio import pytest + +pytest.importorskip("telegram") # PTB is an optional dependency; skip if absent. from telegram.error import Conflict, TelegramError from telegram.request import BaseRequest From 444b5e96fa2829c29cfd7ecdc84d89f83a1441da Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:15:52 +0530 Subject: [PATCH 070/149] chore(release): map arnispiekus in AUTHOR_MAP For PR #63581 salvage (telegram: require getUpdates progress before polling is healthy). --- scripts/release.py | 1 + tests/test_telegram_polling_progress_ptb.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release.py b/scripts/release.py index d6568f505e87..d13802599769 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -84,6 +84,7 @@ AUTHOR_MAP = { "blueirobin02@gmail.com": "irresi", # PR #59048 salvage (gateway: scope reset banners' session info to the serving profile; #59003) "jashlee+microsoft@microsoft.com": "s905060", # PR #57943 salvage (photon: auto-reinstall stale sidecar node_modules when lockfile is newer than npm's install marker; #59169) "lohinth25@proton.me": "l0h1nth", # PR #32210 salvage (mattermost: accept leading-space slash commands from mobile clients; #25184) + "roseycomanagement@roseyco.co.uk": "arnispiekus", # PR #63581 salvage (telegram: require getUpdates progress before polling is healthy; #63243) "perkintahmaz50@gmail.com": "devatnull", # PR #58704 salvage (whatsapp: native Baileys polls, clarify-as-poll, location pins, structured quoted replies, PTT/audio split, bridge_helpers extraction) "tim@iteachyouai.com": "tjp2021", # PR #4097 salvage (copilot: per-turn x-initiator header so user prompts bill as premium requests; #3040) "39274208+falkoro@users.noreply.github.com": "falkoro", # PRs #58519/#58520 salvage (config: env-ref-aware load_config cache invalidation; auxiliary: honor auxiliary..base_url/api_key with explicit provider arg) diff --git a/tests/test_telegram_polling_progress_ptb.py b/tests/test_telegram_polling_progress_ptb.py index 11f3858670f4..c25c50335820 100644 --- a/tests/test_telegram_polling_progress_ptb.py +++ b/tests/test_telegram_polling_progress_ptb.py @@ -3,8 +3,7 @@ import asyncio import pytest - -pytest.importorskip("telegram") # PTB is an optional dependency; skip if absent. +pytest.importorskip("telegram", reason="python-telegram-bot not installed") from telegram.error import Conflict, TelegramError from telegram.request import BaseRequest From 2d0f2185cf3bbf996128dfd5341eea1395b3aca7 Mon Sep 17 00:00:00 2001 From: Gille <4317663+helix4u@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:48:48 -0600 Subject: [PATCH 071/149] fix(desktop): clear stale compaction status across session switches (#64127) * fix(desktop): clear stale compaction status Clear the compaction phase when a turn resumes with model or tool activity, and key response timers by session and turn so switching chats preserves elapsed time.\n\nSupersedes #48115 by porting its resumed-content approach to the current split stream hook and covering tool-first resumptions.\n\nCo-authored-by: liuhao1024 * fix(desktop): resume after thinking activity * fix(desktop): clear turn timer on stop --- .../compaction-event.test.tsx | 82 +++++++++++++++++++ .../hooks/use-message-stream/gateway-event.ts | 25 ++++++ .../hooks/use-prompt-actions/index.test.tsx | 29 ++++++- .../session/hooks/use-prompt-actions/index.ts | 14 +++- .../assistant-ui/thread/status.test.tsx | 56 +++++++++++++ .../components/assistant-ui/thread/status.tsx | 14 +++- 6 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx create mode 100644 apps/desktop/src/components/assistant-ui/thread/status.test.tsx diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx b/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx new file mode 100644 index 000000000000..a403bde23800 --- /dev/null +++ b/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx @@ -0,0 +1,82 @@ +import { QueryClient } from '@tanstack/react-query' +import { act, cleanup, render, waitFor } from '@testing-library/react' +import { useEffect, useRef } from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ClientSessionState } from '@/app/types' +import { createClientSessionState } from '@/lib/chat-runtime' +import { $compactingSessions, setSessionCompacting } from '@/store/compaction' +import type { RpcEvent } from '@/types/hermes' + +import { useMessageStream } from './index' + +const SID = 'session-1' +const OTHER_SID = 'session-2' +let handleEvent: ((event: RpcEvent) => void) | null = null + +function Harness() { + const activeSessionIdRef = useRef(SID) + const sessionStateByRuntimeIdRef = useRef(new Map()) + const queryClientRef = useRef(new QueryClient()) + + const stream = useMessageStream({ + activeSessionIdRef, + hydrateFromStoredSession: vi.fn(async () => undefined), + queryClient: queryClientRef.current, + refreshHermesConfig: vi.fn(async () => undefined), + refreshSessions: vi.fn(async () => undefined), + sessionStateByRuntimeIdRef, + updateSessionState: (sessionId, updater) => { + const current = sessionStateByRuntimeIdRef.current.get(sessionId) ?? createClientSessionState() + const next = updater(current) + sessionStateByRuntimeIdRef.current.set(sessionId, next) + + return next + } + }) + + useEffect(() => { + handleEvent = stream.handleGatewayEvent + }, [stream.handleGatewayEvent]) + + return null +} + +async function mountStream() { + render() + await waitFor(() => expect(handleEvent).not.toBeNull()) +} + +function emit(type: RpcEvent['type'], payload: RpcEvent['payload'] = {}) { + act(() => handleEvent!({ payload, session_id: SID, type })) +} + +describe('useMessageStream compaction lifecycle', () => { + beforeEach(() => { + handleEvent = null + $compactingSessions.set({}) + }) + + afterEach(() => { + cleanup() + $compactingSessions.set({}) + vi.restoreAllMocks() + }) + + it.each([ + ['message.delta', { text: 'resumed' }], + ['thinking.delta', { text: 'still working' }], + ['reasoning.delta', { text: 'thinking again' }], + ['tool.start', { name: 'terminal', tool_id: 'tool-1' }] + ] as const)('clears the stale compaction phase when %s resumes the turn', async (type, payload) => { + await mountStream() + setSessionCompacting(OTHER_SID, true) + + emit('status.update', { kind: 'compacting' }) + expect($compactingSessions.get()).toEqual({ [OTHER_SID]: true, [SID]: true }) + + emit(type, payload) + + expect($compactingSessions.get()).toEqual({ [OTHER_SID]: true }) + }) +}) diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts index 77104ba6295d..c50db6ef1b1d 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts @@ -50,6 +50,19 @@ import type { ClientSessionState } from '../../../types' import { hasSessionInfoStatePatch, sessionInfoStatePatch, SUBAGENT_EVENT_TYPES, toTodoPayload } from './utils' +const COMPACTION_RESUME_EVENT_TYPES = new Set([ + 'message.delta', + 'thinking.delta', + 'reasoning.delta', + 'reasoning.available', + 'moa.reference', + 'moa.aggregating', + 'tool.start', + 'tool.progress', + 'tool.generating', + 'tool.complete' +]) + interface GatewayEventDeps { activeSessionIdRef: MutableRefObject compactedTurnRef: MutableRefObject> @@ -118,6 +131,18 @@ export function useGatewayEventHandler(deps: GatewayEventDeps) { const sessionId = route.sessionId const isActiveEvent = !!sessionId && sessionId === activeSessionIdRef.current + // Mid-turn compaction does not emit another message.start. The first + // model output or tool event proves summarization has finished and the + // turn has resumed, so retire the phase label without waiting for the + // whole turn to complete. + if ( + sessionId && + COMPACTION_RESUME_EVENT_TYPES.has(event.type) && + compactedTurnRef.current.has(sessionId) + ) { + setSessionCompacting(sessionId, false) + } + if (event.type === 'gateway.ready') { return } else if (event.type === 'session.info') { diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx index 3e725b1480d3..9c5c269eb27b 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { textPart } from '@/lib/chat-messages' import { $composerAttachments, $composerDraft, type ComposerAttachment, setComposerDraft } from '@/store/composer' -import { $busy, $connection, $messages, $sessions, setSessions } from '@/store/session' +import { $busy, $connection, $messages, $sessions, $turnStartedAt, setSessions } from '@/store/session' import type { SessionInfo } from '@/types/hermes' import { uploadComposerAttachment, usePromptActions } from '.' @@ -1075,6 +1075,7 @@ describe('usePromptActions sleep/wake session recovery', () => { afterEach(() => { cleanup() + $turnStartedAt.set(null) vi.restoreAllMocks() }) @@ -1168,6 +1169,32 @@ describe('usePromptActions sleep/wake session recovery', () => { expect(calls[2]?.params).toEqual({ session_id: RECOVERED_SESSION_ID }) }) + it('clears the active and cached turn clocks when stopping a turn', async () => { + const states: Record[] = [] + const requestGateway = vi.fn(async () => ({}) as never) + $turnStartedAt.set(1_700_000_000_000) + + let handle: HarnessHandle | null = null + await actRender( + (handle = h)} + onSeedState={state => states.push(state)} + refreshSessions={async () => undefined} + requestGateway={requestGateway} + /> + ) + + await handle!.cancelRun() + + expect($turnStartedAt.get()).toBeNull() + expect(states.at(-1)).toMatchObject({ + awaitingResponse: false, + busy: false, + interrupted: true, + turnStartedAt: null + }) + }) + it('surfaces the original error (no resume) when the failure is not "session not found"', async () => { const calls: string[] = [] const states: Record[] = [] diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts index 51a3689ae4e1..89f978cd69f8 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts @@ -21,7 +21,15 @@ import { resetSessionBackground } from '@/store/composer-status' import { clearNotifications, notify, notifyError } from '@/store/notifications' import { clearPreviewArtifacts } from '@/store/preview-status' import { clearAllPrompts } from '@/store/prompts' -import { $busy, $connection, $messages, setAwaitingResponse, setBusy, setMessages } from '@/store/session' +import { + $busy, + $connection, + $messages, + setAwaitingResponse, + setBusy, + setMessages, + setTurnStartedAt +} from '@/store/session' import { clearSessionSubagents } from '@/store/subagents' import { clearSessionTodos } from '@/store/todos' @@ -501,6 +509,7 @@ export function usePromptActions({ } setAwaitingResponse(false) + setTurnStartedAt(null) const finalizeMessages = (messages: ChatMessage[], streamId?: string | null) => messages @@ -526,7 +535,8 @@ export function usePromptActions({ streamId: null, pendingBranchGroup: null, needsInput: false, - interrupted: true + interrupted: true, + turnStartedAt: null } }) diff --git a/apps/desktop/src/components/assistant-ui/thread/status.test.tsx b/apps/desktop/src/components/assistant-ui/thread/status.test.tsx new file mode 100644 index 000000000000..b4920e1ec483 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/thread/status.test.tsx @@ -0,0 +1,56 @@ +import { act, cleanup, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { __resetElapsedTimerRegistryForTests } from '@/components/chat/activity-timer' +import { I18nProvider } from '@/i18n' +import { $activeSessionId, $turnStartedAt } from '@/store/session' + +import { ResponseLoadingIndicator } from './status' + +function renderIndicator() { + return render( + + + + ) +} + +describe('ResponseLoadingIndicator timer', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')) + __resetElapsedTimerRegistryForTests() + }) + + afterEach(() => { + cleanup() + $activeSessionId.set(null) + $turnStartedAt.set(null) + __resetElapsedTimerRegistryForTests() + vi.useRealTimers() + }) + + it('preserves each running session timer while switching between sessions', () => { + $activeSessionId.set('session-a') + $turnStartedAt.set(Date.now()) + const sessionA = renderIndicator() + + act(() => vi.advanceTimersByTime(5_000)) + expect(screen.getByText('5s')).toBeTruthy() + sessionA.unmount() + + $activeSessionId.set('session-b') + $turnStartedAt.set(Date.now()) + const sessionB = renderIndicator() + + act(() => vi.advanceTimersByTime(3_000)) + expect(screen.getByText('3s')).toBeTruthy() + sessionB.unmount() + + $activeSessionId.set('session-a') + $turnStartedAt.set(new Date('2026-01-01T00:00:00.000Z').getTime()) + renderIndicator() + + expect(screen.getByText('8s')).toBeTruthy() + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/thread/status.tsx b/apps/desktop/src/components/assistant-ui/thread/status.tsx index 53c6f415a85f..f65793d18ca8 100644 --- a/apps/desktop/src/components/assistant-ui/thread/status.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/status.tsx @@ -11,6 +11,7 @@ import { cn } from '@/lib/utils' import { $backgroundResume } from '@/store/background-delegation' import { $compactionActive } from '@/store/compaction' import { $activeSessionAwaitingInput } from '@/store/prompts' +import { $activeSessionId, $turnStartedAt } from '@/store/session' const StatusRow: FC<{ children: ReactNode; label: string } & React.ComponentPropsWithoutRef<'div'>> = ({ children, @@ -36,6 +37,13 @@ const CompactionHint: FC = () => ( {COMPACTION_LABEL} ) +function useActiveTurnTimerKey(): string | undefined { + const activeSessionId = useStore($activeSessionId) + const turnStartedAt = useStore($turnStartedAt) + + return activeSessionId && turnStartedAt ? `turn:${activeSessionId}:${turnStartedAt}` : undefined +} + export const CenteredThreadSpinner: FC = () => { const { t } = useI18n() @@ -59,7 +67,8 @@ export const CenteredThreadSpinner: FC = () => { export const ResponseLoadingIndicator: FC = () => { const { t } = useI18n() - const elapsed = useElapsedSeconds() + const timerKey = useActiveTurnTimerKey() + const elapsed = useElapsedSeconds(true, timerKey) const compacting = useStore($compactionActive) return ( @@ -134,6 +143,7 @@ export const StreamStallIndicator: FC = () => { const [stalled, setStalled] = useState(false) const compacting = useStore($compactionActive) + const turnTimerKey = useActiveTurnTimerKey() // A pending clarify / approval / sudo / secret means the turn is paused on the // user, not working — so don't resurrect the "thinking" timer while they // decide (matches the pet's awaitingInput pose taking priority over busy). @@ -147,7 +157,7 @@ export const StreamStallIndicator: FC = () => { }, [activity]) const active = (stalled || compacting) && !awaitingInput - const elapsed = useElapsedSeconds(active) + const elapsed = useElapsedSeconds(active, compacting ? turnTimerKey : undefined) if (!active) { return null From c084085a3e520178fb3aa27c6ba7a411df8c79a0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:24:29 -0700 Subject: [PATCH 072/149] test: remove flaky test_crashed_runner_produces_error_completion (#64431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flaked 3 times today across 3 unrelated PRs (#64321, #64319, #64409), on two different CI shards (slice 1 and slice 8), while passing deterministically on local runs of the same SHAs. The test polls process_registry.completion_queue for 5s waiting for a daemon-thread completion event; since the durable completion delivery work (67f4e1b4a, d0e9a42ce) the crashed-runner path also writes through the sqlite-backed persistence layer, and on slow CI runners the in-memory enqueue can lose the 5s race. Coverage note: the durable-delivery suite in this file covers the completed-runner and submit-failure paths through persistence, but not a runner that raises mid-flight — that specific path loses its direct test with this removal. A deterministic (non-racing) replacement can follow separately if wanted. --- tests/tools/test_async_delegation.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index 77732960bcaf..cc4f7651d364 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -167,24 +167,6 @@ def test_dispatch_rejected_at_capacity(): ev.set() -def test_crashed_runner_produces_error_completion(): - def boom(): - raise RuntimeError("subagent exploded") - - r = ad.dispatch_async_delegation( - goal="risky", context=None, toolsets=None, role="leaf", model="m", - session_key="", runner=boom, max_async_children=3, - ) - assert r["status"] == "dispatched" - evt = _drain_one() - assert evt is not None - assert evt["status"] == "error" - text = format_process_notification(evt) - assert text is not None - assert "did not complete successfully" in text - assert "subagent exploded" in text - - def test_interrupt_all_signals_running_children(): ev = threading.Event() interrupted = {"count": 0} From 34d07732ddaadf25cf3c093972ec04391fe61c39 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Sun, 28 Jun 2026 04:49:48 +0800 Subject: [PATCH 073/149] fix(telegram): respect rich_messages config for pipe table routing Remove the pipe-table bypass from _rich_delivery_enabled() so that rich_messages: false is fully honoured. Previously, pipe tables were auto-routed to sendRichMessage regardless of the config flag, breaking delivery on clients without Bot API 10.1 support (AyuGram, Telegram Web, some desktop clients). Fixes #53824 --- plugins/platforms/telegram/adapter.py | 12 +++---- tests/gateway/test_telegram_rich_messages.py | 35 +++++++++----------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index 3a986283239c..8ab596076365 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -1482,11 +1482,12 @@ class TelegramAdapter(BasePlatformAdapter): def _content_is_pipe_table_primary(self, content: str) -> bool: """True when pipe tables are the only rich construct in *content*. - Tables are auto-routed to ``sendRichMessage`` even when the full - ``rich_messages`` opt-in is off — MarkdownV2 has no table syntax and - the legacy path rewrites them into bullet lists, which reads like a - regression when users enable Telegram Topics and expect native tables. - Task lists, ``
``, and block math still require the full opt-in. + Used downstream to decide rendering strategy *after* the + ``rich_messages`` config gate has already approved rich delivery. + MarkdownV2 has no table syntax — the legacy path rewrites them into + bullet lists — so callers may choose native table rendering when the + opt-in is active. Task lists, ``
``, and block math indicate + composite rich content and return ``False``. """ if not content or not any( _TABLE_SEPARATOR_RE.match(line) for line in content.splitlines() @@ -1504,7 +1505,6 @@ class TelegramAdapter(BasePlatformAdapter): """Whether rich delivery is allowed for this payload.""" return bool( getattr(self, "_rich_messages_enabled", True) - or self._content_is_pipe_table_primary(content) ) def _rich_eligible(self, content: str) -> bool: diff --git a/tests/gateway/test_telegram_rich_messages.py b/tests/gateway/test_telegram_rich_messages.py index eb4a5b3802d8..3588442d64aa 100644 --- a/tests/gateway/test_telegram_rich_messages.py +++ b/tests/gateway/test_telegram_rich_messages.py @@ -582,21 +582,20 @@ async def test_notification_opt_in_drops_disable_flag(): @pytest.mark.asyncio -async def test_table_only_uses_rich_when_rich_messages_opt_out(): - """Pipe tables auto-route to sendRichMessage even without the full opt-in.""" +async def test_table_only_uses_legacy_when_rich_messages_opt_out(): + """Pipe tables respect the rich_messages: false config and stay on legacy.""" adapter = _make_adapter(extra={"rich_messages": False}) result = await adapter.send("12345", TABLE_ONLY_CONTENT) assert result.success is True - api_kwargs = _rich_api_kwargs(adapter) - assert api_kwargs["rich_message"]["markdown"] == TABLE_ONLY_CONTENT - adapter._bot.send_message.assert_not_called() + adapter._bot.do_api_request.assert_not_called() + adapter._bot.send_message.assert_awaited() @pytest.mark.asyncio -async def test_table_only_uses_rich_with_default_config(): - """Default config keeps task lists on legacy but upgrades bare tables.""" +async def test_table_only_uses_legacy_with_default_config(): + """Default config (rich_messages unset → False) keeps tables on legacy path.""" config = PlatformConfig(enabled=True, token="fake-token") adapter = TelegramAdapter(config) bot = MagicMock() @@ -608,13 +607,13 @@ async def test_table_only_uses_rich_with_default_config(): result = await adapter.send("12345", TABLE_ONLY_CONTENT) assert result.success is True - bot.do_api_request.assert_awaited_once() - bot.send_message.assert_not_called() + bot.do_api_request.assert_not_called() + bot.send_message.assert_awaited() @pytest.mark.asyncio -async def test_dm_topic_resumed_send_uses_rich_for_table_without_reply_anchor(): - """Resumed/synthetic DM-topic sends route tables via direct_messages_topic_id.""" +async def test_dm_topic_resumed_send_uses_legacy_for_table_when_opt_out(): + """Resumed DM-topic sends respect rich_messages: false for tables.""" adapter = _make_adapter(extra={"rich_messages": False}) result = await adapter.send( @@ -628,14 +627,13 @@ async def test_dm_topic_resumed_send_uses_rich_for_table_without_reply_anchor(): ) assert result.success is True - api_kwargs = _rich_api_kwargs(adapter) - assert api_kwargs["direct_messages_topic_id"] == 20189 - assert "reply_parameters" not in api_kwargs - assert api_kwargs["rich_message"]["markdown"] == TABLE_ONLY_CONTENT + adapter._bot.do_api_request.assert_not_called() + adapter._bot.send_message.assert_awaited() @pytest.mark.asyncio -async def test_finalize_edit_rich_includes_forum_topic_routing(): +async def test_finalize_edit_legacy_includes_forum_topic_routing(): + """With rich_messages: false, table edits use legacy path with topic routing.""" adapter = _make_adapter(extra={"rich_messages": False}) result = await adapter.edit_message( @@ -647,9 +645,8 @@ async def test_finalize_edit_rich_includes_forum_topic_routing(): ) assert result.success is True - api_kwargs = _rich_edit_kwargs(adapter) - assert api_kwargs["message_thread_id"] == 5 - assert api_kwargs["rich_message"]["markdown"] == TABLE_ONLY_CONTENT + adapter._bot.do_api_request.assert_not_called() + adapter._bot.edit_message_text.assert_awaited() @pytest.mark.asyncio From b45a217e0a5b0e4fee19fa830c0322024309d4d8 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Thu, 2 Jul 2026 22:42:06 +0800 Subject: [PATCH 074/149] fix(agent): gate Telegram rich-Markdown hint on rich_messages config The platform hint in PLATFORM_HINTS['telegram'] always encouraged rich Markdown constructs (tables, task lists, math, collapsible details) even when rich_messages: false (the default). This caused the agent to produce formatting that MarkdownV2 cannot render, especially broken on Telegram Web. Split the hint into a base hint (MarkdownV2-compatible) and a TELEGRAM_RICH_MESSAGES_HINT extension. The extension is conditionally appended in system_prompt.py only when platforms.telegram.extra.rich_messages is true. Fixes #57122 --- agent/prompt_builder.py | 35 +++++++++++++++--------- agent/system_prompt.py | 15 +++++++++++ tests/agent/test_prompt_builder.py | 21 ++++++++++----- tests/agent/test_system_prompt.py | 43 ++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index b5b2b58c3621..957b3f5fa1ce 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -659,19 +659,7 @@ PLATFORM_HINTS = { "Standard Markdown is automatically converted to Telegram formatting. " "Supported: **bold**, *italic*, ~~strikethrough~~, ||spoiler||, " "`inline code`, ```code blocks```, [links](url), and ## headers. " - "Telegram now supports rich Markdown, so lean into it: whenever it " - "makes the answer clearer or easier to scan, actively reach for real " - "Markdown tables (pipe `| col | col |` syntax), bullet and numbered " - "lists, task lists (`- [ ]` / `- [x]`), headings, nested blockquotes, " - "collapsible details, footnotes/references, math/formulas (`$...$`, " - "`$$...$$`), underline, subscript/superscript, marked (highlighted) " - "text, and anchors. Default to structured formatting over dense " - "paragraphs for any comparison, set of steps, key/value summary, or " - "tabular data. Prefer real Markdown tables and task lists over " - "hand-built bullet substitutes when presenting structured data; these " - "degrade gracefully (tables become readable bullet groups) when rich " - "rendering is unavailable, but advanced constructs like math and " - "collapsible details may render as plain source text in that case. " + "Prefer bullet lists and labeled key:value pairs for structured data. " "You can send media files natively: to deliver a file to the user, " "include MEDIA:/absolute/path/to/file in your response. Images " "(.png, .jpg, .webp) appear as photos, audio (.ogg) sends as voice " @@ -865,6 +853,27 @@ PLATFORM_HINTS = { ), } +# Telegram rich-messages extension — only injected when the user has opted in +# to ``platforms.telegram.extra.rich_messages: true``. The base +# PLATFORM_HINTS["telegram"] covers MarkdownV2-compatible constructs; this +# extension adds the Bot API 10.1 rich-Markdown guidance (tables, task lists, +# collapsible details, math, etc.). +TELEGRAM_RICH_MESSAGES_HINT = ( + "Telegram now supports rich Markdown, so lean into it: whenever it " + "makes the answer clearer or easier to scan, actively reach for real " + "Markdown tables (pipe `| col | col |` syntax), bullet and numbered " + "lists, task lists (`- [ ]` / `- [x]`), headings, nested blockquotes, " + "collapsible details, footnotes/references, math/formulas (`$...$`, " + "`$$...$$`), underline, subscript/superscript, marked (highlighted) " + "text, and anchors. Default to structured formatting over dense " + "paragraphs for any comparison, set of steps, key/value summary, or " + "tabular data. Prefer real Markdown tables and task lists over " + "hand-built bullet substitutes when presenting structured data; these " + "degrade gracefully (tables become readable bullet groups) when rich " + "rendering is unavailable, but advanced constructs like math and " + "collapsible details may render as plain source text in that case. " +) + # --------------------------------------------------------------------------- # Environment hints — execution-environment awareness for the agent. # Unlike PLATFORM_HINTS (which describe the messaging channel), these describe diff --git a/agent/system_prompt.py b/agent/system_prompt.py index ea33df54a0d0..17959af3c510 100644 --- a/agent/system_prompt.py +++ b/agent/system_prompt.py @@ -40,6 +40,7 @@ from agent.prompt_builder import ( SKILLS_GUIDANCE, STEER_CHANNEL_NOTE, TASK_COMPLETION_GUIDANCE, + TELEGRAM_RICH_MESSAGES_HINT, TOOL_USE_ENFORCEMENT_GUIDANCE, TOOL_USE_ENFORCEMENT_MODELS, drain_truncation_warnings, @@ -429,6 +430,20 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) except Exception: pass + # For Telegram: append the rich-messages extension only when the user has + # opted in to ``platforms.telegram.extra.rich_messages: true``. The base + # hint covers MarkdownV2-compatible constructs; the extension adds Bot API + # 10.1 guidance (tables, task lists, math, collapsible details, etc.). + if platform_key == "telegram" and _default_hint: + try: + from hermes_cli.config import load_config_readonly + _cfg = load_config_readonly() + _tg_extra = ((_cfg.get("platforms") or {}).get("telegram") or {}).get("extra") or {} + if _tg_extra.get("rich_messages"): + _default_hint = _default_hint.rstrip() + " " + TELEGRAM_RICH_MESSAGES_HINT + except Exception: + pass # Config read failure — fall back to base hint only + _effective_hint = _resolve_platform_hint(agent, platform_key, _default_hint) if platform_key == "tui" and _effective_hint: _effective_hint = _tui_embedded_pane_clarifier(_effective_hint) diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index 858c880ec8fb..481f68eaed3d 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -1092,15 +1092,22 @@ class TestPromptBuilderConstants: hint = PLATFORM_HINTS["telegram"] lowered = hint.lower() assert "Telegram has NO table syntax" not in hint - assert "rich markdown" in lowered - assert "table" in lowered - assert "task list" in lowered - assert "math" in lowered + # Base hint covers MarkdownV2-compatible constructs. + assert "MEDIA:" in hint + # Rich-messages extension (TELEGRAM_RICH_MESSAGES_HINT) covers the + # Bot API 10.1 guidance; it is injected conditionally in + # system_prompt.py when rich_messages: true. + from agent.prompt_builder import TELEGRAM_RICH_MESSAGES_HINT + rich_lowered = TELEGRAM_RICH_MESSAGES_HINT.lower() + assert "rich markdown" in rich_lowered + assert "table" in rich_lowered + assert "task list" in rich_lowered + assert "math" in rich_lowered # Hint should proactively steer toward structured formatting, not just # permit it: bullet + numbered lists for scannable, structured output. - assert "bullet" in lowered - assert "numbered" in lowered - # Local media delivery guidance must remain intact. + assert "bullet" in rich_lowered + assert "numbered" in rich_lowered + # Local media delivery guidance must remain intact in the base hint. assert "include MEDIA:" in hint def test_platform_hints_mattermost(self): diff --git a/tests/agent/test_system_prompt.py b/tests/agent/test_system_prompt.py index 6ebf2a61960a..0536de1dccd0 100644 --- a/tests/agent/test_system_prompt.py +++ b/tests/agent/test_system_prompt.py @@ -99,3 +99,46 @@ class TestCodingContextBlock: monkeypatch.setenv("TERMINAL_CWD", str(tmp_path)) agent = _make_agent(valid_tool_names=[], platform="cli") assert "coding agent" not in _stable_prompt(agent) + + +class TestTelegramRichMessagesHint: + """Verify that TELEGRAM_RICH_MESSAGES_HINT is conditionally included.""" + + def test_base_hint_without_rich_messages(self, monkeypatch): + """When rich_messages is False (default), only the base hint is used.""" + agent = _make_agent(platform="telegram") + # Mock config to return rich_messages: false (default) + with patch("hermes_cli.config.load_config_readonly") as mock_cfg: + mock_cfg.return_value = { + "platforms": {"telegram": {"extra": {"rich_messages": False}}} + } + stable = _stable_prompt(agent) + # Base hint should be present + assert "Standard Markdown is automatically converted" in stable + # Rich-messages extension should NOT be present + assert "lean into it" not in stable + assert "task lists" not in stable + + def test_rich_hint_with_rich_messages_enabled(self, monkeypatch): + """When rich_messages is True, the rich-messages extension is appended.""" + agent = _make_agent(platform="telegram") + with patch("hermes_cli.config.load_config_readonly") as mock_cfg: + mock_cfg.return_value = { + "platforms": {"telegram": {"extra": {"rich_messages": True}}} + } + stable = _stable_prompt(agent) + # Base hint should be present + assert "Standard Markdown is automatically converted" in stable + # Rich-messages extension should be present + assert "lean into it" in stable + assert "task lists" in stable + assert "math/formulas" in stable + + def test_base_hint_without_config(self, monkeypatch): + """When config has no telegram section, only base hint is used.""" + agent = _make_agent(platform="telegram") + with patch("hermes_cli.config.load_config_readonly") as mock_cfg: + mock_cfg.return_value = {} + stable = _stable_prompt(agent) + assert "Standard Markdown is automatically converted" in stable + assert "lean into it" not in stable From 4e6e5181c643b26dc8189b1b0f0196ed48916726 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:48:39 -0700 Subject: [PATCH 075/149] refactor(telegram): drop dead _content_is_pipe_table_primary helper After #53825's fix removed the auto-rich table bypass from _rich_delivery_enabled(), _content_is_pipe_table_primary() had zero callers. Remove it and simplify _rich_delivery_enabled() to the bare rich_messages opt-in check (content param no longer used). --- plugins/platforms/telegram/adapter.py | 32 ++++----------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index 8ab596076365..32aa2481d390 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -1479,33 +1479,9 @@ class TelegramAdapter(BasePlatformAdapter): return True return False - def _content_is_pipe_table_primary(self, content: str) -> bool: - """True when pipe tables are the only rich construct in *content*. - - Used downstream to decide rendering strategy *after* the - ``rich_messages`` config gate has already approved rich delivery. - MarkdownV2 has no table syntax — the legacy path rewrites them into - bullet lists — so callers may choose native table rendering when the - opt-in is active. Task lists, ``
``, and block math indicate - composite rich content and return ``False``. - """ - if not content or not any( - _TABLE_SEPARATOR_RE.match(line) for line in content.splitlines() - ): - return False - if re.search(r"(?m)^\s*[-*]\s+\[[ xX]\]\s+", content): - return False - if re.search(r"(?m)^|^", content): - return False - if "$$" in content: - return False - return True - - def _rich_delivery_enabled(self, content: str) -> bool: - """Whether rich delivery is allowed for this payload.""" - return bool( - getattr(self, "_rich_messages_enabled", True) - ) + def _rich_delivery_enabled(self) -> bool: + """Whether rich delivery is allowed (``rich_messages`` opt-in).""" + return bool(getattr(self, "_rich_messages_enabled", True)) def _rich_eligible(self, content: str) -> bool: """Capability/content eligibility for rich, ignoring ``expect_edits``. @@ -1517,7 +1493,7 @@ class TelegramAdapter(BasePlatformAdapter): FINAL edit should still upgrade to rich when the content warrants it. """ return bool( - self._rich_delivery_enabled(content) + self._rich_delivery_enabled() and not getattr(self, "_rich_send_disabled", False) and content and content.strip() From b4c2c4f922cb46fc35186d568348bfd1602a8756 Mon Sep 17 00:00:00 2001 From: neo Date: Sun, 5 Jul 2026 03:21:15 +0800 Subject: [PATCH 076/149] fix: drop empty user turns from MoA advisory view (strict-provider 400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MoA's _reference_messages() unconditionally appended every user-role message to the advisory view sent to reference models, even when the message content was an empty string or a non-string/multimodal payload that the text-extraction step flattens to "". Strict providers (Kimi/Moonshot, and others that enforce non-empty user content) reject such a message with: 400 Invalid request: the message at position N with role 'user' must not be empty Lenient providers (DeepSeek) accept it, so an identical rendered view passes on one reference and 400s on another within the same fan-out — the user sees "kimi doesn't support MoA" when the real cause is an empty user turn leaking into the advisory transcript. Skip empty user turns, mirroring the existing behavior for empty assistant turns (which are already dropped when they carry no parts). The end-on-user invariant is preserved: the synthetic advisory-request user turn is still appended when the view would otherwise end on an assistant turn. Adds a regression test asserting the advisory view contains no empty user turn and still ends on a user turn. --- agent/moa_loop.py | 16 +++++++++-- tests/run_agent/test_moa_loop_mode.py | 38 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 267f33dd7e5a..d8461e921dc2 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -501,8 +501,20 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: # a placeholder so the reference knows a non-text turn # happened. text = "[user sent non-text content (e.g. an image attachment)]" - if text.strip(): - last_user_content = text + if not text.strip(): + # Genuinely empty user turn (content="" / None). It carries + # nothing advisory, and strict providers (Kimi/Moonshot, ZAI, + # and others that enforce non-empty user content) reject it + # with 400 "message ... with role 'user' must not be empty" — + # the same way the assistant branch below drops turns with no + # parts. Lenient providers (DeepSeek) accept the empty turn, + # which is why a MoA fan-out would fail on one reference and + # pass on another for the identical rendered view. The + # advisory view is already not strictly alternating (adjacent + # assistant turns occur in every tool loop), so dropping a + # contentless turn is safe. + continue + last_user_content = text rendered.append({"role": "user", "content": text}) elif role == "assistant": parts: list[str] = [] diff --git a/tests/run_agent/test_moa_loop_mode.py b/tests/run_agent/test_moa_loop_mode.py index 094278541762..659f64da7a94 100644 --- a/tests/run_agent/test_moa_loop_mode.py +++ b/tests/run_agent/test_moa_loop_mode.py @@ -392,6 +392,44 @@ def test_reference_messages_fresh_user_turn_ends_on_that_user(): assert view[-1] == {"role": "user", "content": "q2 current"} +def test_reference_messages_drops_empty_user_turns(): + """Empty user turns must not leak into the advisory view. + + A user message whose content is "" or a non-string/multimodal payload + (flattened to "" by the text-extraction step) carries nothing advisory. + Strict providers (Kimi/Moonshot and others that enforce non-empty user + content) reject such a message with + 400 "message ... with role 'user' must not be empty", while lenient + providers (DeepSeek) accept it — so a fan-out over the identical rendered + view fails on one reference and passes on another. The renderer must emit + NO empty user turn, mirroring how empty assistant turns are dropped. + """ + from agent.moa_loop import _reference_messages + + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "real question"}, + {"role": "assistant", "content": "", "tool_calls": [ + {"function": {"name": "read_file", "arguments": '{"path":"c.yaml"}'}} + ]}, + {"role": "tool", "content": "some result"}, + {"role": "user", "content": ""}, # empty string user turn + {"role": "user", "content": [{"type": "text", "text": "multimodal"}]}, # non-string -> "" + ] + + view = _reference_messages(messages) + + # No user turn in the view may be empty/whitespace-only. + empty_users = [ + m for m in view + if m.get("role") == "user" and not str(m.get("content", "")).strip() + ] + assert empty_users == [], f"empty user turn leaked into advisory view: {empty_users}" + # The real user prompt survives and the view still ends on a user turn. + assert view[0] == {"role": "user", "content": "real question"} + assert view[-1]["role"] == "user" + + def test_run_reference_prepends_advisory_system_prompt(monkeypatch): """Each reference call gets the advisory-role system prompt first. From b013ed03e539e94466a0aaafc4d67e40cdeb888d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:31:07 -0700 Subject: [PATCH 077/149] fix(moa): scope the non-text placeholder to structured content only Follow-up to the cherry-picked empty-user-turn drop: the placeholder introduced in 8582f35d9 fired for whitespace-only STRING turns too (content=' ' flattens to non-stripping text but isn't in the (None, '', []) exclusion set), fabricating an attachment note for a turn that carried nothing. Gate the placeholder on isinstance(content, list) so only genuinely structured (e.g. image-only) turns get it; empty and whitespace-only string turns now fall through to the drop path. Edge cases verified: trailing empty user turn still ends the view on the synthetic advisory marker; an all-empty transcript degenerates to []. --- agent/moa_loop.py | 6 ++++-- tests/run_agent/test_moa_loop_mode.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index d8461e921dc2..28707b2a4f14 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -491,7 +491,7 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: if role == "system": continue if role == "user": - if not text.strip() and content not in (None, "", []): + if not text.strip() and isinstance(content, list) and content: # Structured content with no extractable text (e.g. an # image-only turn). Emitting an empty user message would be # dropped/rejected by strict providers (Anthropic 400s on @@ -499,7 +499,9 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: # mode), and silently skipping the turn would break # user/assistant alternation in the advisory view. Substitute # a placeholder so the reference knows a non-text turn - # happened. + # happened. Only structured content qualifies — an empty or + # whitespace-only STRING turn carries nothing and is dropped + # below instead. text = "[user sent non-text content (e.g. an image attachment)]" if not text.strip(): # Genuinely empty user turn (content="" / None). It carries diff --git a/tests/run_agent/test_moa_loop_mode.py b/tests/run_agent/test_moa_loop_mode.py index 659f64da7a94..1c1ab8f71eb7 100644 --- a/tests/run_agent/test_moa_loop_mode.py +++ b/tests/run_agent/test_moa_loop_mode.py @@ -1241,3 +1241,27 @@ def test_reference_guidance_appends_text_part_to_decorated_trailing_user(): assert content[0] == marked_part # The guidance rides as a trailing text part outside the cached span. assert content[1] == {"type": "text", "text": "\n\nREFERENCE BLOCK"} + + +def test_reference_messages_drops_whitespace_only_string_user_turn(): + """A whitespace-only STRING user turn is dropped, not placeholdered. + + The non-text placeholder exists for structured content (image-only turns) + where a real turn happened that the reference should know about. A bare + whitespace string carries nothing — emitting it would 400 strict + providers (Kimi/Moonshot 'role user must not be empty'), and + placeholdering it would fabricate an attachment that never existed. + """ + from agent.moa_loop import _reference_messages + + messages = [ + {"role": "user", "content": " "}, + {"role": "assistant", "content": "a"}, + {"role": "user", "content": "real"}, + ] + + view = _reference_messages(messages) + + assert view[0] == {"role": "assistant", "content": "a"} + assert view[-1] == {"role": "user", "content": "real"} + assert all(str(m["content"]).strip() for m in view) From 72ac0d6af05346823faccf31221633fa3bf5a6e4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:40:31 -0700 Subject: [PATCH 078/149] chore: add neo-claw-bot to AUTHOR_MAP (PR #58465 salvage) --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index d13802599769..39e891d0f096 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json" # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "neo@neodeMac-mini.local": "neo-claw-bot", # PR #58465 salvage (moa: drop empty user turns from advisory view) "m.guttmann@journaway.com": "mguttmann", # PR #63738 salvage (Anthropic setup-token pool auth normalization) "VrtxOmega@pm.me": "VrtxOmega", # PR #43809 salvage (desktop: WSL folder-picker path bridge) "jake.long.vu@vucar.net": "jakelongvu-bot", # PR #36683 partial salvage (approval: honor canonical approvals.timeout in gateway waits) From f9e35e6e9490b2219c9cee599c80fa9e17fcc398 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 9 Jul 2026 22:33:26 -0400 Subject: [PATCH 079/149] fix(auth): route session refresh with provider hint cookie --- hermes_cli/dashboard_auth/cookies.py | 38 ++ hermes_cli/dashboard_auth/middleware.py | 31 +- hermes_cli/dashboard_auth/routes.py | 2 + .../test_dashboard_auth_401_reauth.py | 56 ++ .../hermes_cli/test_dashboard_auth_cookies.py | 485 +++++++++--------- 5 files changed, 374 insertions(+), 238 deletions(-) diff --git a/hermes_cli/dashboard_auth/cookies.py b/hermes_cli/dashboard_auth/cookies.py index ef7f79b27d3f..8bcd9db78eb6 100644 --- a/hermes_cli/dashboard_auth/cookies.py +++ b/hermes_cli/dashboard_auth/cookies.py @@ -66,6 +66,10 @@ from fastapi.responses import Response # request's HTTPS + prefix combination. SESSION_AT_COOKIE = "hermes_session_at" SESSION_RT_COOKIE = "hermes_session_rt" +# Provider that minted the session. This non-secret routing hint prevents a +# refresh token from being handed to the wrong provider when several dashboard +# auth plugins are enabled (for example Basic + Nous OAuth). +SESSION_PROVIDER_COOKIE = "hermes_session_provider" PKCE_COOKIE = "hermes_session_pkce" # One-shot loop-guard marker for the auto-SSO redirect (Phase 1, # cloud-auto-discovery). Set when the gate auto-initiates the portal OAuth @@ -141,6 +145,24 @@ def _common_attrs(*, use_https: bool, prefix: str) -> dict: return attrs +def set_session_provider_cookie( + response: Response, + *, + provider: str, + use_https: bool, + prefix: str = "", +) -> None: + """Persist the non-secret provider routing hint for token refresh.""" + if not provider: + return + response.set_cookie( + _resolved_name(SESSION_PROVIDER_COOKIE, use_https=use_https, prefix=prefix), + provider, + max_age=_RT_MAX_AGE, + **_common_attrs(use_https=use_https, prefix=prefix), + ) + + def set_session_cookies( response: Response, *, @@ -149,6 +171,7 @@ def set_session_cookies( access_token_expires_in: int, use_https: bool, prefix: str = "", + provider: str = "", ) -> None: """Set the session cookies on the response. @@ -181,6 +204,12 @@ def set_session_cookies( max_age=_RT_MAX_AGE, **_common_attrs(use_https=use_https, prefix=prefix), ) + set_session_provider_cookie( + response, + provider=provider, + use_https=use_https, + prefix=prefix, + ) def clear_session_cookies(response: Response, *, prefix: str = "") -> None: @@ -202,6 +231,10 @@ def clear_session_cookies(response: Response, *, prefix: str = "") -> None: f"{variant}{SESSION_RT_COOKIE}", "", max_age=0, path=path, httponly=True, samesite="lax", ) + response.set_cookie( + f"{variant}{SESSION_PROVIDER_COOKIE}", "", max_age=0, + path=path, httponly=True, samesite="lax", + ) def set_pkce_cookie( @@ -248,6 +281,11 @@ def read_session_cookies(request: Request) -> Tuple[Optional[str], Optional[str] return at, rt +def read_session_provider(request: Request) -> Optional[str]: + """Return the provider routing hint associated with the session cookies.""" + return _read_with_fallback(request, SESSION_PROVIDER_COOKIE) + + def read_pkce_cookie(request: Request) -> Optional[str]: return _read_with_fallback(request, PKCE_COOKIE) diff --git a/hermes_cli/dashboard_auth/middleware.py b/hermes_cli/dashboard_auth/middleware.py index 362ed729d65b..da5f9f1ed23b 100644 --- a/hermes_cli/dashboard_auth/middleware.py +++ b/hermes_cli/dashboard_auth/middleware.py @@ -28,7 +28,9 @@ from hermes_cli.dashboard_auth.base import ProviderError, RefreshExpiredError from hermes_cli.dashboard_auth.cookies import ( clear_sso_attempt_cookie, read_session_cookies, + read_session_provider, read_sso_attempt_cookie, + set_session_provider_cookie, set_sso_attempt_cookie, ) from hermes_cli.dashboard_auth.public_paths import PUBLIC_API_PATHS @@ -276,6 +278,7 @@ async def gated_auth_middleware( return await call_next(request) at, _rt = read_session_cookies(request) + provider_hint = read_session_provider(request) if not at and not _rt: # Neither token present — no session at all. Nothing to verify or # refresh. Before falling back to the /login interstitial, try to @@ -321,7 +324,10 @@ async def gated_auth_middleware( # 503 — distinguishing "transient IDP outage" (don't force re-login) # from "token genuinely invalid" (fall through to refresh/relogin). unreachable_provider: str | None = None - for provider in list_session_providers(): + providers = list_session_providers() + if provider_hint: + providers = [provider for provider in providers if provider.name == provider_hint] + for provider in providers: try: session = provider.verify_session(access_token=at) except ProviderError as e: @@ -355,7 +361,7 @@ async def gated_auth_middleware( # one). On success we re-set the rotated cookies on the response and # serve the request transparently; on RefreshExpiredError (RT dead / # revoked / reuse-detected) we fall through to clear-and-relogin. - refreshed = _attempt_refresh(request, refresh_token=_rt) + refreshed = _attempt_refresh(request, refresh_token=_rt, provider_hint=provider_hint) if refreshed is not None: new_session, refreshing_provider = refreshed request.state.session = new_session @@ -378,6 +384,7 @@ async def gated_auth_middleware( access_token_expires_in=_expires_in_seconds(new_session), use_https=detect_https(request), prefix=prefix_from_request(request), + provider=refreshing_provider, ) audit_log( AuditEvent.REFRESH_SUCCESS, @@ -405,7 +412,18 @@ async def gated_auth_middleware( return response request.state.session = session - return await call_next(request) + response = await call_next(request) + if not provider_hint and session.provider: + from hermes_cli.dashboard_auth.cookies import detect_https + from hermes_cli.dashboard_auth.prefix import prefix_from_request + + set_session_provider_cookie( + response, + provider=session.provider, + use_https=detect_https(request), + prefix=prefix_from_request(request), + ) + return response def _expires_in_seconds(session) -> int: @@ -421,7 +439,7 @@ def _expires_in_seconds(session) -> int: return max(60, int(session.expires_at) - int(time.time())) -def _attempt_refresh(request: Request, *, refresh_token): +def _attempt_refresh(request: Request, *, refresh_token, provider_hint: str | None = None): """Try to rotate an expired session via the refresh token. Returns ``(new_session, provider_name)`` on success, or ``None`` if @@ -435,7 +453,10 @@ def _attempt_refresh(request: Request, *, refresh_token): """ if not refresh_token: return None - for provider in list_session_providers(): + providers = list_session_providers() + if provider_hint: + providers = [provider for provider in providers if provider.name == provider_hint] + for provider in providers: try: new_session = provider.refresh_session(refresh_token=refresh_token) except RefreshExpiredError: diff --git a/hermes_cli/dashboard_auth/routes.py b/hermes_cli/dashboard_auth/routes.py index ee595be7d821..5b833e5df79a 100644 --- a/hermes_cli/dashboard_auth/routes.py +++ b/hermes_cli/dashboard_auth/routes.py @@ -365,6 +365,7 @@ async def auth_callback( access_token_expires_in=expires_in, use_https=detect_https(request), prefix=_prefix(request), + provider=session.provider, ) clear_pkce_cookie(resp, prefix=_prefix(request)) # Clear the one-shot auto-SSO loop-guard marker now that login succeeded, @@ -549,6 +550,7 @@ async def auth_password_login(request: Request, body: _PasswordLoginBody): access_token_expires_in=expires_in, use_https=detect_https(request), prefix=_prefix(request), + provider=session.provider, ) return resp diff --git a/tests/hermes_cli/test_dashboard_auth_401_reauth.py b/tests/hermes_cli/test_dashboard_auth_401_reauth.py index 458be58c7942..769b5d1ccf69 100644 --- a/tests/hermes_cli/test_dashboard_auth_401_reauth.py +++ b/tests/hermes_cli/test_dashboard_auth_401_reauth.py @@ -35,6 +35,7 @@ from hermes_cli import web_server from hermes_cli.dashboard_auth import clear_providers, register_provider from hermes_cli.dashboard_auth.cookies import ( SESSION_AT_COOKIE, + SESSION_PROVIDER_COOKIE, SESSION_RT_COOKIE, clear_session_cookies, set_session_cookies, @@ -252,6 +253,61 @@ class TestTransparentRefreshOnAccessTokenEviction: for c in set_cookies ), f"no rotated RT cookie in {set_cookies!r}" + def test_provider_hint_routes_refresh_to_token_owner(self, gated_app): + """A Nous-style RT must not be rejected by Basic just because Basic + was registered first. The non-secret provider hint routes directly to + the provider that minted the session.""" + class WrongProvider(StubAuthProvider): + name = "basic" + + def __init__(self): + super().__init__() + self.refresh_calls = 0 + + def refresh_session(self, *, refresh_token: str): + self.refresh_calls += 1 + raise AssertionError("foreign refresh token reached Basic provider") + + wrong = WrongProvider() + _provider, valid_rt = self._build_rt_only_app() + clear_providers() + register_provider(wrong) + register_provider(StubAuthProvider(default_ttl=900)) + gated_app.cookies.clear() + gated_app.cookies.set(SESSION_RT_COOKIE, valid_rt) + gated_app.cookies.set(SESSION_PROVIDER_COOKIE, "stub") + + response = gated_app.get("/api/sessions", follow_redirects=False) + + assert response.status_code == 200 + assert wrong.refresh_calls == 0 + assert any( + SESSION_PROVIDER_COOKIE in cookie and "stub" in cookie + for cookie in response.headers.get_list("set-cookie") + ) + + def test_valid_legacy_session_is_migrated_with_provider_hint(self, gated_app): + import time as _t + from tests.hermes_cli.conftest_dashboard_auth import _sign + + valid_at = _sign({ + "sub": "stub-user-1", + "email": "stub@example.test", + "name": "Stub User", + "org_id": "stub-org-1", + "exp": int(_t.time()) + 900, + }) + gated_app.cookies.clear() + gated_app.cookies.set(SESSION_AT_COOKIE, valid_at) + + response = gated_app.get("/api/sessions") + + assert response.status_code == 200 + assert any( + SESSION_PROVIDER_COOKIE in cookie and "stub" in cookie + for cookie in response.headers.get_list("set-cookie") + ) + def test_no_cookies_at_all_still_bounces(self, gated_app): """Guard the fix didn't over-reach: a request with NEITHER cookie must still 401 to login (nothing to verify or refresh).""" diff --git a/tests/hermes_cli/test_dashboard_auth_cookies.py b/tests/hermes_cli/test_dashboard_auth_cookies.py index 7109b7b70993..25ae2b8ff4ae 100644 --- a/tests/hermes_cli/test_dashboard_auth_cookies.py +++ b/tests/hermes_cli/test_dashboard_auth_cookies.py @@ -1,233 +1,252 @@ -"""Tests for the dashboard-auth cookie helpers.""" -from __future__ import annotations - -from fastapi import FastAPI -from fastapi.responses import Response -from fastapi.testclient import TestClient -from starlette.requests import Request - -from hermes_cli.dashboard_auth.cookies import ( - PKCE_COOKIE, - SESSION_AT_COOKIE, - SESSION_RT_COOKIE, - clear_pkce_cookie, - clear_session_cookies, - read_pkce_cookie, - read_session_cookies, - set_pkce_cookie, - set_session_cookies, -) - - -def _build_app(use_https: bool = True, prefix: str = ""): - app = FastAPI() - - @app.get("/set") - def set_endpoint(): - r = Response("ok") - set_session_cookies( - r, access_token="AT", refresh_token="RT", - access_token_expires_in=3600, use_https=use_https, - prefix=prefix, - ) - return r - - @app.get("/set-pkce") - def set_pkce(): - r = Response("ok") - set_pkce_cookie(r, payload="provider=stub;state=s;verifier=v", - use_https=use_https, prefix=prefix) - return r - - @app.get("/clear") - def clear(): - r = Response("ok") - clear_session_cookies(r, prefix=prefix) - clear_pkce_cookie(r, prefix=prefix) - return r - - return app - - -# Cookie name resolution helpers used throughout — the bare name resolves -# to a request-shape-dependent variant (__Host- / __Secure- / bare). -# Tests pin a specific shape so a regression in the name-resolution -# logic fails loudly rather than silently breaking sessions. - - -def test_session_cookies_use_host_prefix_on_https_direct(): - """HTTPS + no proxy prefix → __Host- prefix (strongest spec - hardening: bound to exact origin, requires Path=/, requires Secure).""" - client = TestClient(_build_app(use_https=True, prefix="")) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) - rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) - for c in (at, rt): - assert "HttpOnly" in c - assert "samesite=lax" in c.lower() - assert "Secure" in c - assert "Path=/" in c - - -def test_session_cookies_use_secure_prefix_when_proxied(): - """HTTPS + /hermes prefix → __Secure- prefix (__Host- forbids - Path != "/"; __Secure- keeps the Secure-required hardening).""" - client = TestClient(_build_app(use_https=True, prefix="/hermes")) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Secure-{SESSION_AT_COOKIE}=")) - assert "Path=/hermes" in at - assert "Secure" in at - # __Host- variant must NOT be emitted on the prefix path. - assert not any( - c.startswith(f"__Host-{SESSION_AT_COOKIE}=") for c in cookies - ) - - -def test_session_cookies_use_bare_name_on_http(): - """Loopback HTTP dev: __Host- / __Secure- both require Secure, which - we can't set on HTTP. Use bare cookie names.""" - client = TestClient(_build_app(use_https=False)) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - # Bare name present; no __Host- / __Secure- variant emitted. - assert any(c.startswith(f"{SESSION_AT_COOKIE}=") for c in cookies) - assert not any( - c.startswith(f"__Host-{SESSION_AT_COOKIE}=") - or c.startswith(f"__Secure-{SESSION_AT_COOKIE}=") - for c in cookies - ) - # No Secure flag (HTTP). - at = next(c for c in cookies if c.startswith(f"{SESSION_AT_COOKIE}=")) - assert "Secure" not in at - - -def test_session_cookies_have_30day_rt_and_token_ttl_at(): - client = TestClient(_build_app(use_https=True)) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) - rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) - assert "Max-Age=3600" in at - assert "Max-Age=2592000" in rt # 30 days = 30 * 86400 - - -def test_clear_session_cookies_emits_expired_at_and_rt(): - """``clear_session_cookies`` emits Max-Age=0 deletions for every - plausible cookie-name variant under the active prefix so we flush - stale cookies that an older deploy may have set under a different - prefix.""" - client = TestClient(_build_app()) - r = client.get("/clear") - cookies = r.headers.get_list("set-cookie") - # At least one variant of each session cookie should be deleted. - assert any( - SESSION_AT_COOKIE in c and "Max-Age=0" in c for c in cookies - ) - assert any( - SESSION_RT_COOKIE in c and "Max-Age=0" in c for c in cookies - ) - - -def test_pkce_cookie_short_ttl_and_path_root(): - client = TestClient(_build_app(use_https=True)) - r = client.get("/set-pkce") - pkce = next( - c for c in r.headers.get_list("set-cookie") - if PKCE_COOKIE in c - ) - assert "HttpOnly" in pkce - assert "Max-Age=600" in pkce # 10 minutes - assert "Path=/" in pkce - assert "Secure" in pkce - - -def test_read_session_cookies_from_request_bare_name(): - """Reader accepts the bare name (loopback) by default.""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"{SESSION_AT_COOKIE}=at_value; {SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_cookies_from_request_host_prefix(): - """Reader also finds cookies set with the __Host- variant - (HTTPS direct deploy).""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"__Host-{SESSION_AT_COOKIE}=at_value; " - f"__Host-{SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_cookies_from_request_secure_prefix(): - """Reader also finds cookies set with the __Secure- variant - (HTTPS behind a proxy prefix).""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"__Secure-{SESSION_AT_COOKIE}=at_value; " - f"__Secure-{SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_cookies_missing_returns_none(): - req = Request({"type": "http", "method": "GET", "path": "/", "headers": []}) - assert read_session_cookies(req) == (None, None) - - -def test_read_pkce_cookie_round_trip(): - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [(b"cookie", f"{PKCE_COOKIE}=state=s;verifier=v".encode())], - } - req = Request(scope) - assert read_pkce_cookie(req) == "state=s" # NB: cookie value stops at ';' - - -def test_detect_https_via_scheme(): - """``detect_https`` reads from request.url.scheme. - - Under uvicorn proxy_headers=True the scheme is rewritten from - ``X-Forwarded-Proto``; that's an integration concern, not unit. - """ - from hermes_cli.dashboard_auth.cookies import detect_https - http_req = Request({ - "type": "http", "method": "GET", "path": "/", "scheme": "http", - "headers": [], "server": ("x", 80), - }) - https_req = Request({ - "type": "http", "method": "GET", "path": "/", "scheme": "https", - "headers": [], "server": ("x", 443), - }) - assert detect_https(http_req) is False - assert detect_https(https_req) is True +"""Tests for the dashboard-auth cookie helpers.""" +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.responses import Response +from fastapi.testclient import TestClient +from starlette.requests import Request + +from hermes_cli.dashboard_auth.cookies import ( + PKCE_COOKIE, + SESSION_AT_COOKIE, + SESSION_PROVIDER_COOKIE, + SESSION_RT_COOKIE, + clear_pkce_cookie, + clear_session_cookies, + read_pkce_cookie, + read_session_cookies, + read_session_provider, + set_pkce_cookie, + set_session_cookies, +) + + +def _build_app(use_https: bool = True, prefix: str = ""): + app = FastAPI() + + @app.get("/set") + def set_endpoint(): + r = Response("ok") + set_session_cookies( + r, access_token="AT", refresh_token="RT", + access_token_expires_in=3600, use_https=use_https, + prefix=prefix, provider="nous", + ) + return r + + @app.get("/set-pkce") + def set_pkce(): + r = Response("ok") + set_pkce_cookie(r, payload="provider=stub;state=s;verifier=v", + use_https=use_https, prefix=prefix) + return r + + @app.get("/clear") + def clear(): + r = Response("ok") + clear_session_cookies(r, prefix=prefix) + clear_pkce_cookie(r, prefix=prefix) + return r + + return app + + +# Cookie name resolution helpers used throughout — the bare name resolves +# to a request-shape-dependent variant (__Host- / __Secure- / bare). +# Tests pin a specific shape so a regression in the name-resolution +# logic fails loudly rather than silently breaking sessions. + + +def test_session_cookies_use_host_prefix_on_https_direct(): + """HTTPS + no proxy prefix → __Host- prefix (strongest spec + hardening: bound to exact origin, requires Path=/, requires Secure).""" + client = TestClient(_build_app(use_https=True, prefix="")) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) + rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) + provider = next(c for c in cookies if c.startswith(f"__Host-{SESSION_PROVIDER_COOKIE}=nous")) + for c in (at, rt, provider): + assert "HttpOnly" in c + assert "samesite=lax" in c.lower() + assert "Secure" in c + assert "Path=/" in c + + +def test_session_cookies_use_secure_prefix_when_proxied(): + """HTTPS + /hermes prefix → __Secure- prefix (__Host- forbids + Path != "/"; __Secure- keeps the Secure-required hardening).""" + client = TestClient(_build_app(use_https=True, prefix="/hermes")) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Secure-{SESSION_AT_COOKIE}=")) + assert "Path=/hermes" in at + assert "Secure" in at + # __Host- variant must NOT be emitted on the prefix path. + assert not any( + c.startswith(f"__Host-{SESSION_AT_COOKIE}=") for c in cookies + ) + + +def test_session_cookies_use_bare_name_on_http(): + """Loopback HTTP dev: __Host- / __Secure- both require Secure, which + we can't set on HTTP. Use bare cookie names.""" + client = TestClient(_build_app(use_https=False)) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + # Bare name present; no __Host- / __Secure- variant emitted. + assert any(c.startswith(f"{SESSION_AT_COOKIE}=") for c in cookies) + assert not any( + c.startswith(f"__Host-{SESSION_AT_COOKIE}=") + or c.startswith(f"__Secure-{SESSION_AT_COOKIE}=") + for c in cookies + ) + # No Secure flag (HTTP). + at = next(c for c in cookies if c.startswith(f"{SESSION_AT_COOKIE}=")) + assert "Secure" not in at + + +def test_session_cookies_have_30day_rt_and_token_ttl_at(): + client = TestClient(_build_app(use_https=True)) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) + rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) + assert "Max-Age=3600" in at + assert "Max-Age=2592000" in rt # 30 days = 30 * 86400 + + +def test_clear_session_cookies_emits_expired_at_and_rt(): + """``clear_session_cookies`` emits Max-Age=0 deletions for every + plausible cookie-name variant under the active prefix so we flush + stale cookies that an older deploy may have set under a different + prefix.""" + client = TestClient(_build_app()) + r = client.get("/clear") + cookies = r.headers.get_list("set-cookie") + # At least one variant of each session cookie should be deleted. + assert any( + SESSION_AT_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + assert any( + SESSION_RT_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + assert any( + SESSION_PROVIDER_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + + +def test_pkce_cookie_short_ttl_and_path_root(): + client = TestClient(_build_app(use_https=True)) + r = client.get("/set-pkce") + pkce = next( + c for c in r.headers.get_list("set-cookie") + if PKCE_COOKIE in c + ) + assert "HttpOnly" in pkce + assert "Max-Age=600" in pkce # 10 minutes + assert "Path=/" in pkce + assert "Secure" in pkce + + +def test_read_session_cookies_from_request_bare_name(): + """Reader accepts the bare name (loopback) by default.""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"{SESSION_AT_COOKIE}=at_value; {SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_provider_from_request(): + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Host-{SESSION_PROVIDER_COOKIE}=nous".encode(), + )], + } + assert read_session_provider(Request(scope)) == "nous" + + +def test_read_session_cookies_from_request_host_prefix(): + """Reader also finds cookies set with the __Host- variant + (HTTPS direct deploy).""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Host-{SESSION_AT_COOKIE}=at_value; " + f"__Host-{SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_cookies_from_request_secure_prefix(): + """Reader also finds cookies set with the __Secure- variant + (HTTPS behind a proxy prefix).""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Secure-{SESSION_AT_COOKIE}=at_value; " + f"__Secure-{SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_cookies_missing_returns_none(): + req = Request({"type": "http", "method": "GET", "path": "/", "headers": []}) + assert read_session_cookies(req) == (None, None) + + +def test_read_pkce_cookie_round_trip(): + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [(b"cookie", f"{PKCE_COOKIE}=state=s;verifier=v".encode())], + } + req = Request(scope) + assert read_pkce_cookie(req) == "state=s" # NB: cookie value stops at ';' + + +def test_detect_https_via_scheme(): + """``detect_https`` reads from request.url.scheme. + + Under uvicorn proxy_headers=True the scheme is rewritten from + ``X-Forwarded-Proto``; that's an integration concern, not unit. + """ + from hermes_cli.dashboard_auth.cookies import detect_https + http_req = Request({ + "type": "http", "method": "GET", "path": "/", "scheme": "http", + "headers": [], "server": ("x", 80), + }) + https_req = Request({ + "type": "http", "method": "GET", "path": "/", "scheme": "https", + "headers": [], "server": ("x", 443), + }) + assert detect_https(http_req) is False + assert detect_https(https_req) is True From 96a070844898b4fba9a77dda844a875e23e675d7 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 10 Jul 2026 18:31:27 -0400 Subject: [PATCH 080/149] fix(auth): preserve provider fallback during refresh --- hermes_cli/dashboard_auth/base.py | 16 +- hermes_cli/dashboard_auth/middleware.py | 78 ++- .../test_dashboard_auth_401_reauth.py | 96 ++++ .../hermes_cli/test_dashboard_auth_cookies.py | 504 +++++++++--------- 4 files changed, 413 insertions(+), 281 deletions(-) diff --git a/hermes_cli/dashboard_auth/base.py b/hermes_cli/dashboard_auth/base.py index 8f376f352108..e8b8a7730b1d 100644 --- a/hermes_cli/dashboard_auth/base.py +++ b/hermes_cli/dashboard_auth/base.py @@ -94,9 +94,11 @@ class InvalidCredentialsError(Exception): class RefreshExpiredError(Exception): - """The refresh token is dead. + """This provider rejects the refresh token as dead or invalid. - Middleware clears cookies and forces re-login (302 → ``/login``). + In a multi-provider deployment this does not prove token ownership, so + middleware may try remaining providers. It clears cookies and forces + re-login only after every reachable provider rejects the token. """ @@ -125,9 +127,13 @@ class DashboardAuthProvider(ABC): raises ``ProviderError`` if the IDP is unreachable. Middleware treats expiry and unreachable differently (expiry → refresh; unreachable → 503). - * ``refresh_session`` raises ``RefreshExpiredError`` when the - refresh token is also invalid; middleware then forces re-login. - Raises ``ProviderError`` on network failure. + * ``refresh_session`` raises ``RefreshExpiredError`` when the refresh + token is invalid for that provider. Middleware tries the remaining + providers because an opaque foreign token can be indistinguishable + from an expired one; it forces re-login only after every reachable + provider rejects the token. Raises ``ProviderError`` on network + failure; middleware still tries remaining providers, but returns 503 + without clearing cookies if none succeeds and any was unavailable. * ``revoke_session`` is best-effort and must not raise. Subclasses MUST set ``name`` (lowercase identifier, stable forever) diff --git a/hermes_cli/dashboard_auth/middleware.py b/hermes_cli/dashboard_auth/middleware.py index da5f9f1ed23b..caa1b3a6e5dd 100644 --- a/hermes_cli/dashboard_auth/middleware.py +++ b/hermes_cli/dashboard_auth/middleware.py @@ -24,7 +24,11 @@ from fastapi.responses import JSONResponse, RedirectResponse, Response from hermes_cli.dashboard_auth import list_session_providers from hermes_cli.dashboard_auth.audit import AuditEvent, audit_log -from hermes_cli.dashboard_auth.base import ProviderError, RefreshExpiredError +from hermes_cli.dashboard_auth.base import ( + DashboardAuthProvider, + ProviderError, + RefreshExpiredError, +) from hermes_cli.dashboard_auth.cookies import ( clear_sso_attempt_cookie, read_session_cookies, @@ -85,6 +89,22 @@ def _client_ip(request: Request) -> str: return request.client.host if request.client else "" +def _ordered_session_providers( + provider_hint: str | None, +) -> list[DashboardAuthProvider]: + """Prefer the hinted provider without making the hint authoritative. + + The cookie can outlive a provider rename/removal or become stale after a + deployment change. A stable sort moves a matching provider to the front + while preserving registration order for every remaining candidate; an + unknown hint therefore leaves the normal scan unchanged. + """ + providers = list_session_providers() + if provider_hint: + providers.sort(key=lambda provider: provider.name != provider_hint) + return providers + + def _unauth_response(request: Request, *, reason: str) -> Response: """API routes → 401 JSON with ``login_url``; HTML routes → 302 → /login. @@ -324,10 +344,7 @@ async def gated_auth_middleware( # 503 — distinguishing "transient IDP outage" (don't force re-login) # from "token genuinely invalid" (fall through to refresh/relogin). unreachable_provider: str | None = None - providers = list_session_providers() - if provider_hint: - providers = [provider for provider in providers if provider.name == provider_hint] - for provider in providers: + for provider in _ordered_session_providers(provider_hint): try: session = provider.verify_session(access_token=at) except ProviderError as e: @@ -359,9 +376,22 @@ async def gated_auth_middleware( # Access token is expired/invalid. Before forcing re-login, try to # rotate it using the refresh token (if the session cookie carries # one). On success we re-set the rotated cookies on the response and - # serve the request transparently; on RefreshExpiredError (RT dead / - # revoked / reuse-detected) we fall through to clear-and-relogin. - refreshed = _attempt_refresh(request, refresh_token=_rt, provider_hint=provider_hint) + # serve the request transparently; only after every provider rejects + # the RT do we fall through to clear-and-relogin. + try: + refreshed = _attempt_refresh( + request, + refresh_token=_rt, + provider_hint=provider_hint, + ) + except ProviderError as e: + # At least one provider could not confirm or reject the RT, and no + # other provider refreshed it. Preserve the cookies and surface a + # transient outage instead of turning uncertainty into a logout. + return JSONResponse( + {"detail": f"Auth provider {str(e)!r} unreachable"}, + status_code=503, + ) if refreshed is not None: new_session, refreshing_provider = refreshed request.state.session = new_session @@ -442,33 +472,29 @@ def _expires_in_seconds(session) -> int: def _attempt_refresh(request: Request, *, refresh_token, provider_hint: str | None = None): """Try to rotate an expired session via the refresh token. - Returns ``(new_session, provider_name)`` on success, or ``None`` if - there's no RT or every provider's ``refresh_session`` failed with - ``RefreshExpiredError`` (dead/revoked/reuse-detected RT → force re-login). - - A ``ProviderError`` (Portal unreachable) is NOT swallowed into a re-login - here — re-raising would 500 the request; instead we log and return None so - the caller forces a clean re-login, which is the safer UX than a hard - error on a transient network blip during the narrow refresh window. + The provider hint only changes candidate order. ``RefreshExpiredError`` + rejects the token for that candidate, but cannot prove ownership because + providers such as Basic raise it for foreign opaque tokens too. Likewise, + ``ProviderError`` only makes that candidate unavailable. Both are audited + and the remaining providers are tried. Returns ``None`` only when there is + no RT or every reachable provider rejects it. If no provider succeeds and + at least one raised ``ProviderError``, re-raises with that provider's name + so the caller can return 503 without clearing potentially valid cookies. """ if not refresh_token: return None - providers = list_session_providers() - if provider_hint: - providers = [provider for provider in providers if provider.name == provider_hint] - for provider in providers: + unavailable_provider: str | None = None + for provider in _ordered_session_providers(provider_hint): try: new_session = provider.refresh_session(refresh_token=refresh_token) except RefreshExpiredError: - # This provider owns the RT but it's dead — stop trying others - # (an RT belongs to exactly one provider) and force re-login. audit_log( AuditEvent.REFRESH_FAILURE, provider=provider.name, reason="refresh_expired", ip=_client_ip(request), ) - return None + continue except ProviderError as e: _log.warning( "dashboard-auth: provider %r unreachable during refresh: %s", @@ -480,7 +506,11 @@ def _attempt_refresh(request: Request, *, refresh_token, provider_hint: str | No reason="provider_unreachable", ip=_client_ip(request), ) - return None + if unavailable_provider is None: + unavailable_provider = provider.name + continue if new_session is not None: return new_session, provider.name + if unavailable_provider is not None: + raise ProviderError(unavailable_provider) return None diff --git a/tests/hermes_cli/test_dashboard_auth_401_reauth.py b/tests/hermes_cli/test_dashboard_auth_401_reauth.py index 769b5d1ccf69..63ab76a9a833 100644 --- a/tests/hermes_cli/test_dashboard_auth_401_reauth.py +++ b/tests/hermes_cli/test_dashboard_auth_401_reauth.py @@ -33,6 +33,7 @@ from fastapi.testclient import TestClient from hermes_cli import web_server from hermes_cli.dashboard_auth import clear_providers, register_provider +from hermes_cli.dashboard_auth.base import ProviderError, RefreshExpiredError from hermes_cli.dashboard_auth.cookies import ( SESSION_AT_COOKIE, SESSION_PROVIDER_COOKIE, @@ -286,6 +287,101 @@ class TestTransparentRefreshOnAccessTokenEviction: for cookie in response.headers.get_list("set-cookie") ) + def test_unknown_provider_hint_retains_verify_fallback(self, gated_app): + """A hint for a removed provider must not suppress the normal scan.""" + import time as _t + from tests.hermes_cli.conftest_dashboard_auth import _sign + + valid_at = _sign({ + "sub": "stub-user-1", + "email": "stub@example.test", + "name": "Stub User", + "org_id": "stub-org-1", + "exp": int(_t.time()) + 900, + }) + gated_app.cookies.clear() + gated_app.cookies.set(SESSION_AT_COOKIE, valid_at) + gated_app.cookies.set(SESSION_PROVIDER_COOKIE, "removed-provider") + + response = gated_app.get("/api/auth/me") + + assert response.status_code == 200 + assert response.json()["provider"] == "stub" + + @pytest.mark.parametrize( + "error_type", + [RefreshExpiredError, ProviderError], + ids=["token-rejected", "provider-unreachable"], + ) + def test_stale_provider_hint_refresh_error_falls_back( + self, + gated_app, + error_type, + ): + """A stale known hint may reject a foreign RT or be unavailable. + + Either failure applies only to that provider candidate; remaining + providers still get a chance to claim the token. + """ + class StaleHintProvider(StubAuthProvider): + name = "basic" + + def __init__(self): + super().__init__() + self.refresh_calls = 0 + + def refresh_session(self, *, refresh_token: str): + self.refresh_calls += 1 + raise error_type("foreign refresh token") + + stale = StaleHintProvider() + _provider, valid_rt = self._build_rt_only_app() + clear_providers() + register_provider(stale) + register_provider(StubAuthProvider(default_ttl=900)) + gated_app.cookies.clear() + gated_app.cookies.set(SESSION_RT_COOKIE, valid_rt) + gated_app.cookies.set(SESSION_PROVIDER_COOKIE, "basic") + + response = gated_app.get("/api/sessions", follow_redirects=False) + + assert response.status_code == 200 + assert stale.refresh_calls == 1 + assert any( + SESSION_PROVIDER_COOKIE in cookie and "stub" in cookie + for cookie in response.headers.get_list("set-cookie") + ) + + def test_refresh_outage_returns_503_without_clearing_cookies(self, gated_app): + """Uncertain ownership during an outage must not log the user out.""" + class UnreachableProvider(StubAuthProvider): + name = "unreachable" + + def refresh_session(self, *, refresh_token: str): + raise ProviderError("simulated provider outage") + + class RejectingProvider(StubAuthProvider): + name = "rejecting" + + def refresh_session(self, *, refresh_token: str): + raise RefreshExpiredError("foreign refresh token") + + clear_providers() + register_provider(UnreachableProvider()) + register_provider(RejectingProvider()) + gated_app.cookies.clear() + gated_app.cookies.set(SESSION_RT_COOKIE, "opaque-refresh-token") + gated_app.cookies.set(SESSION_PROVIDER_COOKIE, "unreachable") + + response = gated_app.get("/api/sessions", follow_redirects=False) + + assert response.status_code == 503 + assert gated_app.cookies.get(SESSION_RT_COOKIE) == "opaque-refresh-token" + assert not any( + SESSION_RT_COOKIE in cookie and "Max-Age=0" in cookie + for cookie in response.headers.get_list("set-cookie") + ) + def test_valid_legacy_session_is_migrated_with_provider_hint(self, gated_app): import time as _t from tests.hermes_cli.conftest_dashboard_auth import _sign diff --git a/tests/hermes_cli/test_dashboard_auth_cookies.py b/tests/hermes_cli/test_dashboard_auth_cookies.py index 25ae2b8ff4ae..3f0baaa4dbb2 100644 --- a/tests/hermes_cli/test_dashboard_auth_cookies.py +++ b/tests/hermes_cli/test_dashboard_auth_cookies.py @@ -1,252 +1,252 @@ -"""Tests for the dashboard-auth cookie helpers.""" -from __future__ import annotations - -from fastapi import FastAPI -from fastapi.responses import Response -from fastapi.testclient import TestClient -from starlette.requests import Request - -from hermes_cli.dashboard_auth.cookies import ( - PKCE_COOKIE, - SESSION_AT_COOKIE, - SESSION_PROVIDER_COOKIE, - SESSION_RT_COOKIE, - clear_pkce_cookie, - clear_session_cookies, - read_pkce_cookie, - read_session_cookies, - read_session_provider, - set_pkce_cookie, - set_session_cookies, -) - - -def _build_app(use_https: bool = True, prefix: str = ""): - app = FastAPI() - - @app.get("/set") - def set_endpoint(): - r = Response("ok") - set_session_cookies( - r, access_token="AT", refresh_token="RT", - access_token_expires_in=3600, use_https=use_https, - prefix=prefix, provider="nous", - ) - return r - - @app.get("/set-pkce") - def set_pkce(): - r = Response("ok") - set_pkce_cookie(r, payload="provider=stub;state=s;verifier=v", - use_https=use_https, prefix=prefix) - return r - - @app.get("/clear") - def clear(): - r = Response("ok") - clear_session_cookies(r, prefix=prefix) - clear_pkce_cookie(r, prefix=prefix) - return r - - return app - - -# Cookie name resolution helpers used throughout — the bare name resolves -# to a request-shape-dependent variant (__Host- / __Secure- / bare). -# Tests pin a specific shape so a regression in the name-resolution -# logic fails loudly rather than silently breaking sessions. - - -def test_session_cookies_use_host_prefix_on_https_direct(): - """HTTPS + no proxy prefix → __Host- prefix (strongest spec - hardening: bound to exact origin, requires Path=/, requires Secure).""" - client = TestClient(_build_app(use_https=True, prefix="")) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) - rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) - provider = next(c for c in cookies if c.startswith(f"__Host-{SESSION_PROVIDER_COOKIE}=nous")) - for c in (at, rt, provider): - assert "HttpOnly" in c - assert "samesite=lax" in c.lower() - assert "Secure" in c - assert "Path=/" in c - - -def test_session_cookies_use_secure_prefix_when_proxied(): - """HTTPS + /hermes prefix → __Secure- prefix (__Host- forbids - Path != "/"; __Secure- keeps the Secure-required hardening).""" - client = TestClient(_build_app(use_https=True, prefix="/hermes")) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Secure-{SESSION_AT_COOKIE}=")) - assert "Path=/hermes" in at - assert "Secure" in at - # __Host- variant must NOT be emitted on the prefix path. - assert not any( - c.startswith(f"__Host-{SESSION_AT_COOKIE}=") for c in cookies - ) - - -def test_session_cookies_use_bare_name_on_http(): - """Loopback HTTP dev: __Host- / __Secure- both require Secure, which - we can't set on HTTP. Use bare cookie names.""" - client = TestClient(_build_app(use_https=False)) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - # Bare name present; no __Host- / __Secure- variant emitted. - assert any(c.startswith(f"{SESSION_AT_COOKIE}=") for c in cookies) - assert not any( - c.startswith(f"__Host-{SESSION_AT_COOKIE}=") - or c.startswith(f"__Secure-{SESSION_AT_COOKIE}=") - for c in cookies - ) - # No Secure flag (HTTP). - at = next(c for c in cookies if c.startswith(f"{SESSION_AT_COOKIE}=")) - assert "Secure" not in at - - -def test_session_cookies_have_30day_rt_and_token_ttl_at(): - client = TestClient(_build_app(use_https=True)) - r = client.get("/set") - cookies = r.headers.get_list("set-cookie") - at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) - rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) - assert "Max-Age=3600" in at - assert "Max-Age=2592000" in rt # 30 days = 30 * 86400 - - -def test_clear_session_cookies_emits_expired_at_and_rt(): - """``clear_session_cookies`` emits Max-Age=0 deletions for every - plausible cookie-name variant under the active prefix so we flush - stale cookies that an older deploy may have set under a different - prefix.""" - client = TestClient(_build_app()) - r = client.get("/clear") - cookies = r.headers.get_list("set-cookie") - # At least one variant of each session cookie should be deleted. - assert any( - SESSION_AT_COOKIE in c and "Max-Age=0" in c for c in cookies - ) - assert any( - SESSION_RT_COOKIE in c and "Max-Age=0" in c for c in cookies - ) - assert any( - SESSION_PROVIDER_COOKIE in c and "Max-Age=0" in c for c in cookies - ) - - -def test_pkce_cookie_short_ttl_and_path_root(): - client = TestClient(_build_app(use_https=True)) - r = client.get("/set-pkce") - pkce = next( - c for c in r.headers.get_list("set-cookie") - if PKCE_COOKIE in c - ) - assert "HttpOnly" in pkce - assert "Max-Age=600" in pkce # 10 minutes - assert "Path=/" in pkce - assert "Secure" in pkce - - -def test_read_session_cookies_from_request_bare_name(): - """Reader accepts the bare name (loopback) by default.""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"{SESSION_AT_COOKIE}=at_value; {SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_provider_from_request(): - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"__Host-{SESSION_PROVIDER_COOKIE}=nous".encode(), - )], - } - assert read_session_provider(Request(scope)) == "nous" - - -def test_read_session_cookies_from_request_host_prefix(): - """Reader also finds cookies set with the __Host- variant - (HTTPS direct deploy).""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"__Host-{SESSION_AT_COOKIE}=at_value; " - f"__Host-{SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_cookies_from_request_secure_prefix(): - """Reader also finds cookies set with the __Secure- variant - (HTTPS behind a proxy prefix).""" - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [( - b"cookie", - f"__Secure-{SESSION_AT_COOKIE}=at_value; " - f"__Secure-{SESSION_RT_COOKIE}=rt_value".encode(), - )], - } - req = Request(scope) - at, rt = read_session_cookies(req) - assert at == "at_value" - assert rt == "rt_value" - - -def test_read_session_cookies_missing_returns_none(): - req = Request({"type": "http", "method": "GET", "path": "/", "headers": []}) - assert read_session_cookies(req) == (None, None) - - -def test_read_pkce_cookie_round_trip(): - scope = { - "type": "http", - "method": "GET", - "path": "/", - "headers": [(b"cookie", f"{PKCE_COOKIE}=state=s;verifier=v".encode())], - } - req = Request(scope) - assert read_pkce_cookie(req) == "state=s" # NB: cookie value stops at ';' - - -def test_detect_https_via_scheme(): - """``detect_https`` reads from request.url.scheme. - - Under uvicorn proxy_headers=True the scheme is rewritten from - ``X-Forwarded-Proto``; that's an integration concern, not unit. - """ - from hermes_cli.dashboard_auth.cookies import detect_https - http_req = Request({ - "type": "http", "method": "GET", "path": "/", "scheme": "http", - "headers": [], "server": ("x", 80), - }) - https_req = Request({ - "type": "http", "method": "GET", "path": "/", "scheme": "https", - "headers": [], "server": ("x", 443), - }) - assert detect_https(http_req) is False - assert detect_https(https_req) is True +"""Tests for the dashboard-auth cookie helpers.""" +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.responses import Response +from fastapi.testclient import TestClient +from starlette.requests import Request + +from hermes_cli.dashboard_auth.cookies import ( + PKCE_COOKIE, + SESSION_AT_COOKIE, + SESSION_PROVIDER_COOKIE, + SESSION_RT_COOKIE, + clear_pkce_cookie, + clear_session_cookies, + read_pkce_cookie, + read_session_cookies, + read_session_provider, + set_pkce_cookie, + set_session_cookies, +) + + +def _build_app(use_https: bool = True, prefix: str = ""): + app = FastAPI() + + @app.get("/set") + def set_endpoint(): + r = Response("ok") + set_session_cookies( + r, access_token="AT", refresh_token="RT", + access_token_expires_in=3600, use_https=use_https, + prefix=prefix, provider="nous", + ) + return r + + @app.get("/set-pkce") + def set_pkce(): + r = Response("ok") + set_pkce_cookie(r, payload="provider=stub;state=s;verifier=v", + use_https=use_https, prefix=prefix) + return r + + @app.get("/clear") + def clear(): + r = Response("ok") + clear_session_cookies(r, prefix=prefix) + clear_pkce_cookie(r, prefix=prefix) + return r + + return app + + +# Cookie name resolution helpers used throughout — the bare name resolves +# to a request-shape-dependent variant (__Host- / __Secure- / bare). +# Tests pin a specific shape so a regression in the name-resolution +# logic fails loudly rather than silently breaking sessions. + + +def test_session_cookies_use_host_prefix_on_https_direct(): + """HTTPS + no proxy prefix → __Host- prefix (strongest spec + hardening: bound to exact origin, requires Path=/, requires Secure).""" + client = TestClient(_build_app(use_https=True, prefix="")) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) + rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) + provider = next(c for c in cookies if c.startswith(f"__Host-{SESSION_PROVIDER_COOKIE}=nous")) + for c in (at, rt, provider): + assert "HttpOnly" in c + assert "samesite=lax" in c.lower() + assert "Secure" in c + assert "Path=/" in c + + +def test_session_cookies_use_secure_prefix_when_proxied(): + """HTTPS + /hermes prefix → __Secure- prefix (__Host- forbids + Path != "/"; __Secure- keeps the Secure-required hardening).""" + client = TestClient(_build_app(use_https=True, prefix="/hermes")) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Secure-{SESSION_AT_COOKIE}=")) + assert "Path=/hermes" in at + assert "Secure" in at + # __Host- variant must NOT be emitted on the prefix path. + assert not any( + c.startswith(f"__Host-{SESSION_AT_COOKIE}=") for c in cookies + ) + + +def test_session_cookies_use_bare_name_on_http(): + """Loopback HTTP dev: __Host- / __Secure- both require Secure, which + we can't set on HTTP. Use bare cookie names.""" + client = TestClient(_build_app(use_https=False)) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + # Bare name present; no __Host- / __Secure- variant emitted. + assert any(c.startswith(f"{SESSION_AT_COOKIE}=") for c in cookies) + assert not any( + c.startswith(f"__Host-{SESSION_AT_COOKIE}=") + or c.startswith(f"__Secure-{SESSION_AT_COOKIE}=") + for c in cookies + ) + # No Secure flag (HTTP). + at = next(c for c in cookies if c.startswith(f"{SESSION_AT_COOKIE}=")) + assert "Secure" not in at + + +def test_session_cookies_have_30day_rt_and_token_ttl_at(): + client = TestClient(_build_app(use_https=True)) + r = client.get("/set") + cookies = r.headers.get_list("set-cookie") + at = next(c for c in cookies if c.startswith(f"__Host-{SESSION_AT_COOKIE}=")) + rt = next(c for c in cookies if c.startswith(f"__Host-{SESSION_RT_COOKIE}=")) + assert "Max-Age=3600" in at + assert "Max-Age=2592000" in rt # 30 days = 30 * 86400 + + +def test_clear_session_cookies_emits_expired_at_and_rt(): + """``clear_session_cookies`` emits Max-Age=0 deletions for every + plausible cookie-name variant under the active prefix so we flush + stale cookies that an older deploy may have set under a different + prefix.""" + client = TestClient(_build_app()) + r = client.get("/clear") + cookies = r.headers.get_list("set-cookie") + # At least one variant of each session cookie should be deleted. + assert any( + SESSION_AT_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + assert any( + SESSION_RT_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + assert any( + SESSION_PROVIDER_COOKIE in c and "Max-Age=0" in c for c in cookies + ) + + +def test_pkce_cookie_short_ttl_and_path_root(): + client = TestClient(_build_app(use_https=True)) + r = client.get("/set-pkce") + pkce = next( + c for c in r.headers.get_list("set-cookie") + if PKCE_COOKIE in c + ) + assert "HttpOnly" in pkce + assert "Max-Age=600" in pkce # 10 minutes + assert "Path=/" in pkce + assert "Secure" in pkce + + +def test_read_session_cookies_from_request_bare_name(): + """Reader accepts the bare name (loopback) by default.""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"{SESSION_AT_COOKIE}=at_value; {SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_provider_from_request(): + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Host-{SESSION_PROVIDER_COOKIE}=nous".encode(), + )], + } + assert read_session_provider(Request(scope)) == "nous" + + +def test_read_session_cookies_from_request_host_prefix(): + """Reader also finds cookies set with the __Host- variant + (HTTPS direct deploy).""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Host-{SESSION_AT_COOKIE}=at_value; " + f"__Host-{SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_cookies_from_request_secure_prefix(): + """Reader also finds cookies set with the __Secure- variant + (HTTPS behind a proxy prefix).""" + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [( + b"cookie", + f"__Secure-{SESSION_AT_COOKIE}=at_value; " + f"__Secure-{SESSION_RT_COOKIE}=rt_value".encode(), + )], + } + req = Request(scope) + at, rt = read_session_cookies(req) + assert at == "at_value" + assert rt == "rt_value" + + +def test_read_session_cookies_missing_returns_none(): + req = Request({"type": "http", "method": "GET", "path": "/", "headers": []}) + assert read_session_cookies(req) == (None, None) + + +def test_read_pkce_cookie_round_trip(): + scope = { + "type": "http", + "method": "GET", + "path": "/", + "headers": [(b"cookie", f"{PKCE_COOKIE}=state=s;verifier=v".encode())], + } + req = Request(scope) + assert read_pkce_cookie(req) == "state=s" # NB: cookie value stops at ';' + + +def test_detect_https_via_scheme(): + """``detect_https`` reads from request.url.scheme. + + Under uvicorn proxy_headers=True the scheme is rewritten from + ``X-Forwarded-Proto``; that's an integration concern, not unit. + """ + from hermes_cli.dashboard_auth.cookies import detect_https + http_req = Request({ + "type": "http", "method": "GET", "path": "/", "scheme": "http", + "headers": [], "server": ("x", 80), + }) + https_req = Request({ + "type": "http", "method": "GET", "path": "/", "scheme": "https", + "headers": [], "server": ("x", 443), + }) + assert detect_https(http_req) is False + assert detect_https(https_req) is True From 46e87b14fd6c943ef0d6671fb0d74c5dde5d4c6b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:48:26 -0700 Subject: [PATCH 081/149] chore(release): map unsupportedpastels in AUTHOR_MAP --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 39e891d0f096..ce71d417a916 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -328,6 +328,7 @@ AUTHOR_MAP = { "290859878+synapsesx@users.noreply.github.com": "synapsesx", "157689911+itsflownium@users.noreply.github.com": "itsflownium", "dirtyren@users.noreply.github.com": "dirtyren", + "theoldwizard123@pm.me": "unsupportedpastels", "johnmlussier@gmail.com": "John-Lussier", "chenkun_lws@126.com": "bytesnail", # PR #60360 salvage (--yolo startup ordering; #60328) "iamgexin@qq.com": "nullptr0807", # PR #60956 salvage (gateway hygiene in-place compaction; #60947) From 20502b407c8d80e14808442d58ebbe90cb8b543b Mon Sep 17 00:00:00 2001 From: Changhyun Min Date: Mon, 8 Jun 2026 11:27:55 +0900 Subject: [PATCH 082/149] feat(agent): add Upstage Solar as a model provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Upstage Solar as a bundled model-provider plugin. Solar exposes an OpenAI-compatible chat-completions endpoint at https://api.upstage.ai/v1, so the generic chat_completions transport handles request/response/streaming/tool calls — the profile is the core integration. Provider registration (Upstage isn't in models.dev, so each registry that does not auto-wire from the plugin layer needs an explicit entry — same pattern as nvidia/gmi): - plugins/model-providers/upstage/: UpstageProfile + plugin.yaml. Picker default and offline catalog list only the agentic Solar Pro models, led by `solar-pro` (rolling alias for the latest Pro). default_aux_model empty so aux tasks use the main model. `solar` alias. UPSTAGE_BASE_URL overrides the host. - hermes_cli/providers.py: HERMES_OVERLAYS + label + `solar` alias, so resolve_provider_full('upstage') resolves (without this, an explicit `provider: upstage` in config was dropped and fell through to auto-detect). - hermes_cli/auth.py: PROVIDER_REGISTRY entry + `solar` alias, so `hermes doctor` / resolve_provider recognise upstage (the static-registry path the lazy profile-extension doesn't reliably cover at validation time). - hermes_cli/models.py: CANONICAL_PROVIDERS entry places Upstage Solar in the curated picker order (above the auto-appended `custom`). - agent/model_metadata.py: context-window fallbacks (/v1/models omits context_length); `solar-pro` carries the 128K Pro context as the catch-all. Reasoning: UpstageProfile.build_api_kwargs_extras wires Solar's top-level `reasoning_effort` (low|medium|high; xhigh/max→high). Reasoning-capable families are solar-pro* and solar-open*; solar-mini/syn-pro never receive it. Defaults ON at medium when unset (matches the /reasoning "medium (default)" label); `/reasoning none` disables; explicit/saved settings are honored. No reasoning_content echo handling needed (unlike DeepSeek/Kimi). Web dashboard: - web/src/pages/EnvPage.tsx: add an "Upstage Solar" provider group so UPSTAGE_API_KEY / UPSTAGE_BASE_URL appear under LLM Providers (not "Other"). Docs/tests: - .env.example: documents UPSTAGE_API_KEY / UPSTAGE_BASE_URL. - tests: profile wiring, reasoning_effort mapping (pro/open/mini, efforts, disabled, default-on), provider-resolver regression (resolve_provider_full / get_provider / solar alias / overlay), `solar-pro` default. Testing: pytest tests/providers tests/plugins/model_providers tests/hermes_cli/test_upstage_provider.py tests/run_agent/test_provider_parity.py tests/hermes_cli/test_api_key_providers.py; ruff clean. Verified end-to-end: `hermes doctor` shows "Upstage Solar", and live chat works via both `--provider upstage` and `--provider solar`. Reasoning wire format per https://console.upstage.ai/api/docs/for-agents/raw. Platforms tested: macOS. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 9 ++ agent/model_metadata.py | 13 ++ hermes_cli/auth.py | 9 ++ hermes_cli/models.py | 1 + hermes_cli/providers.py | 10 ++ plugins/model-providers/upstage/__init__.py | 106 ++++++++++++ plugins/model-providers/upstage/plugin.yaml | 5 + tests/hermes_cli/test_upstage_provider.py | 95 +++++++++++ .../model_providers/test_upstage_profile.py | 153 ++++++++++++++++++ web/src/pages/EnvPage.tsx | 1 + 10 files changed, 402 insertions(+) create mode 100644 plugins/model-providers/upstage/__init__.py create mode 100644 plugins/model-providers/upstage/plugin.yaml create mode 100644 tests/hermes_cli/test_upstage_provider.py create mode 100644 tests/plugins/model_providers/test_upstage_profile.py diff --git a/.env.example b/.env.example index 6eac3487e9a6..893bda62110a 100644 --- a/.env.example +++ b/.env.example @@ -136,6 +136,15 @@ # Optional base URL override: # XIAOMI_BASE_URL=https://api.xiaomimimo.com/v1 +# ============================================================================= +# LLM PROVIDER (Upstage Solar) +# ============================================================================= +# Upstage provides access to Upstage Solar models. +# Get your key at: https://console.upstage.ai/api-keys +# UPSTAGE_API_KEY=your_key_here +# Optional base URL override: +# UPSTAGE_BASE_URL=https://api.upstage.ai/v1 + # ============================================================================= # TOOL API KEYS # ============================================================================= diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 74b6c2f13943..00e6e4b3d53e 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -318,6 +318,19 @@ DEFAULT_CONTEXT_LENGTHS = { "grok": 131072, # catch-all (grok-beta, unknown grok-*) # Kimi "kimi": 262144, + # Upstage Solar — api.upstage.ai/v1/models does not return context_length, + # so these fallbacks keep token budgeting / compression from probing down + # to the 128k default. Substring matching is longest-first, so the versioned + # ids win over the "solar-pro" rolling-alias entry, which in turn covers + # future solar-pro* releases at the Pro context size. + # Sources: Solar Pro 3 = 128K, Solar Pro 2 = 64K, Solar Mini = 32K, + # Solar Open 2 = 256K. + "solar-open2-preview": 262144, # 256K (longest-first: wins over solar-open2) + "solar-open2": 262144, # 256K + "solar-pro3": 131072, + "solar-pro": 131072, # rolling alias → latest Solar Pro (currently pro3) + "solar-pro2": 65536, + "solar-mini": 32768, # Tencent — Hy3 Preview (Hunyuan) with 256K context window. # OpenRouter live metadata reports 262144 (256 × 1024); align the # static fallback so cache and offline both agree (issue #22268). diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 8804bcbeca15..9d6db629ed20 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -290,6 +290,14 @@ PROVIDER_REGISTRY: Dict[str, ProviderConfig] = { api_key_env_vars=("GMI_API_KEY",), base_url_env_var="GMI_BASE_URL", ), + "upstage": ProviderConfig( + id="upstage", + name="Upstage Solar", + auth_type="api_key", + inference_base_url="https://api.upstage.ai/v1", + api_key_env_vars=("UPSTAGE_API_KEY",), + base_url_env_var="UPSTAGE_BASE_URL", + ), "minimax": ProviderConfig( id="minimax", name="MiniMax", @@ -1677,6 +1685,7 @@ def resolve_provider( "step": "stepfun", "stepfun-coding-plan": "stepfun", "arcee-ai": "arcee", "arceeai": "arcee", "gmi-cloud": "gmi", "gmicloud": "gmi", + "solar": "upstage", "minimax-china": "minimax-cn", "minimax_cn": "minimax-cn", "minimax-portal": "minimax-oauth", "minimax-global": "minimax-oauth", "minimax_oauth": "minimax-oauth", "alibaba_coding": "alibaba-coding-plan", "alibaba-coding": "alibaba-coding-plan", diff --git a/hermes_cli/models.py b/hermes_cli/models.py index d81c409814c7..58852e551882 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -1088,6 +1088,7 @@ CANONICAL_PROVIDERS: list[ProviderEntry] = [ ProviderEntry("bedrock", "AWS Bedrock", "AWS Bedrock (Claude, Nova, Llama, DeepSeek; IAM or API key)"), ProviderEntry("azure-foundry", "Azure Foundry", "Azure Foundry (OpenAI-style or Anthropic-style endpoint, your Azure AI deployment)"), ProviderEntry("qwen-oauth", "Qwen OAuth (Portal)", "Qwen OAuth (Reuses local Qwen CLI login)"), + ProviderEntry("upstage", "Upstage Solar", "Upstage (Solar API)"), ] # Auto-extend CANONICAL_PROVIDERS with any provider registered in providers/ diff --git a/hermes_cli/providers.py b/hermes_cli/providers.py index 53d0e23e06b5..2880b33fc4d0 100644 --- a/hermes_cli/providers.py +++ b/hermes_cli/providers.py @@ -201,6 +201,12 @@ HERMES_OVERLAYS: Dict[str, HermesOverlay] = { extra_env_vars=("FIREWORKS_API_KEY",), base_url_override="https://api.fireworks.ai/inference/v1", ), + "upstage": HermesOverlay( + transport="openai_chat", + extra_env_vars=("UPSTAGE_API_KEY",), + base_url_override="https://api.upstage.ai/v1", + base_url_env_var="UPSTAGE_BASE_URL", + ), "ollama-cloud": HermesOverlay( transport="openai_chat", base_url_override="https://ollama.com/v1", @@ -352,6 +358,9 @@ ALIASES: Dict[str, str] = { "fireworks-ai": "fireworks", "fw": "fireworks", + # upstage + "solar": "upstage", + # Local server aliases → virtual "local" concept (resolved via user config) "lmstudio": "lmstudio", "lm-studio": "lmstudio", @@ -376,6 +385,7 @@ _LABEL_OVERRIDES: Dict[str, str] = { "stepfun": "StepFun Step Plan", "xiaomi": "Xiaomi MiMo", "gmi": "GMI Cloud", + "upstage": "Upstage Solar", "tencent-tokenhub": "Tencent TokenHub", "lmstudio": "LM Studio", "local": "Local endpoint", diff --git a/plugins/model-providers/upstage/__init__.py b/plugins/model-providers/upstage/__init__.py new file mode 100644 index 000000000000..03e954e3e4ea --- /dev/null +++ b/plugins/model-providers/upstage/__init__.py @@ -0,0 +1,106 @@ +"""Upstage Solar provider profile.""" + +from typing import Any + +from providers import register_provider +from providers.base import ProviderProfile + + +# Model-name markers for Solar families that accept ``reasoning_effort``. +# Substring match (not startswith) so dated variants like +# ``solar-pro3-250127`` and ``solar-open-...`` are covered too. +_REASONING_MODEL_MARKERS = ("solar-pro", "solar-open") + +# When the user hasn't picked a reasoning effort, Hermes passes +# reasoning_config=None. Solar's own server default is "minimal" (reasoning +# off), which is the wrong default for an agentic workload. We default reasoning +# ON at this effort — matching the "medium (default)" that Hermes' /reasoning +# panel shows for an unset config, so the displayed default and the real wire +# value agree. An explicit saved setting or a `/reasoning ` change is +# always honored over this default; `/reasoning none` disables it. +_DEFAULT_REASONING_EFFORT = "medium" + + +def _model_supports_reasoning(model: str | None) -> bool: + """Solar reasoning-capable models. + + The Solar Pro family (``solar-pro``, ``solar-pro2``, ``solar-pro3`` and + dated variants like ``solar-pro3-250127``) and the Solar Open family + (``solar-open*``) accept ``reasoning_effort``. ``solar-mini`` / ``syn-pro`` + ignore the parameter, so we don't send it. + """ + m = (model or "").strip().lower() + return any(marker in m for marker in _REASONING_MODEL_MARKERS) + + +class UpstageProfile(ProviderProfile): + """Upstage Solar — top-level ``reasoning_effort`` control. + + Solar Pro/Open expose reasoning through a top-level ``reasoning_effort`` + field (``minimal`` | ``low`` | ``medium`` | ``high``), mirroring OpenAI's + shape. Unlike DeepSeek/Kimi it does NOT require echoing ``reasoning_content`` + back on later turns, so only the request field needs wiring. We emit at most + ``low`` | ``medium`` | ``high`` — the explicit values both Solar Pro 2 and + Pro 3 accept. + + Default-on: Solar's own server default is ``minimal`` (off), but for an + agentic workload we default reasoning ON (``_DEFAULT_REASONING_EFFORT``) + when the user hasn't picked an effort. The user can still set any level or + turn it off with ``/reasoning none``. + """ + + def build_api_kwargs_extras( + self, *, reasoning_config: dict | None = None, model: str | None = None, **context + ) -> tuple[dict[str, Any], dict[str, Any]]: + top_level: dict[str, Any] = {} + + # solar-mini / syn-pro ignore reasoning_effort — send nothing. + if not _model_supports_reasoning(model): + return {}, top_level + + # Unset (reasoning_config is None) → default reasoning ON for agents. + if not reasoning_config or not isinstance(reasoning_config, dict): + return {}, {"reasoning_effort": _DEFAULT_REASONING_EFFORT} + + # Explicitly disabled (`/reasoning none`) → omit the field so Solar + # applies its own default (minimal = off). + if reasoning_config.get("enabled") is False: + return {}, top_level + + # Map Hermes' effort vocabulary onto Solar's accepted set. xhigh/max + # collapse to high (Solar's strongest). minimal → off (omit). An + # enabled request with no recognised effort uses the default effort. + effort = (reasoning_config.get("effort") or "").strip().lower() + mapped = { + "minimal": None, + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "high", + "max": "high", + }.get(effort, _DEFAULT_REASONING_EFFORT) + + if mapped: + top_level["reasoning_effort"] = mapped + return {}, top_level + + +upstage = UpstageProfile( + name="upstage", + aliases=("solar",), + display_name="Upstage Solar", + description="Upstage (Solar API)", + signup_url="https://console.upstage.ai/api-keys", + env_vars=("UPSTAGE_API_KEY", "UPSTAGE_BASE_URL"), + base_url="https://api.upstage.ai/v1", + auth_type="api_key", + # default_aux_model left empty → auxiliary side tasks use the main model. + # entry [0] is the setup default. solar-pro is a rolling alias for the + # latest Solar Pro, so the default tracks the current flagship. + fallback_models=( + "solar-pro", + "solar-pro3", + ), +) + +register_provider(upstage) diff --git a/plugins/model-providers/upstage/plugin.yaml b/plugins/model-providers/upstage/plugin.yaml new file mode 100644 index 000000000000..4de2e3d4ba5d --- /dev/null +++ b/plugins/model-providers/upstage/plugin.yaml @@ -0,0 +1,5 @@ +name: upstage-provider +kind: model-provider +version: 1.0.0 +description: Upstage (Solar API) +author: Upstage AI diff --git a/tests/hermes_cli/test_upstage_provider.py b/tests/hermes_cli/test_upstage_provider.py new file mode 100644 index 000000000000..621b86f8d413 --- /dev/null +++ b/tests/hermes_cli/test_upstage_provider.py @@ -0,0 +1,95 @@ +"""Focused tests for Upstage Solar first-class provider wiring. + +Regression guard for the bug where `hermes model` saved `provider: upstage` +correctly but, on re-entry, showed a different provider as active. Root cause: +`hermes_cli/providers.py` (the resolver behind `resolve_provider_full`) had no +`upstage` overlay, so `resolve_provider_full("upstage")` returned None, the +config provider was discarded, and resolution fell through to env auto-detect. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + +if "dotenv" not in sys.modules: + fake_dotenv = types.ModuleType("dotenv") + fake_dotenv.load_dotenv = lambda *args, **kwargs: None + sys.modules["dotenv"] = fake_dotenv + + +class TestUpstageResolver: + """The providers.py resolver must recognise upstage (the actual bug).""" + + def test_resolve_provider_full_recognizes_upstage(self): + from hermes_cli.providers import resolve_provider_full + + pdef = resolve_provider_full("upstage", {}, []) + assert pdef is not None, ( + "resolve_provider_full('upstage') returned None — config " + "`provider: upstage` would be discarded and auto-detect would win" + ) + assert pdef.id == "upstage" + assert pdef.base_url == "https://api.upstage.ai/v1" + assert "UPSTAGE_API_KEY" in pdef.api_key_env_vars + + def test_get_provider_returns_upstage_def(self): + from hermes_cli.providers import get_provider + + pdef = get_provider("upstage") + assert pdef is not None and pdef.id == "upstage" + assert pdef.transport == "openai_chat" + + def test_solar_alias_normalizes_to_upstage(self): + from hermes_cli.providers import normalize_provider, resolve_provider_full + + assert normalize_provider("solar") == "upstage" + pdef = resolve_provider_full("solar", {}, []) + assert pdef is not None and pdef.id == "upstage" + + +class TestUpstageOverlay: + def test_overlay_exists(self): + from hermes_cli.providers import HERMES_OVERLAYS + + assert "upstage" in HERMES_OVERLAYS + overlay = HERMES_OVERLAYS["upstage"] + assert overlay.transport == "openai_chat" + assert overlay.extra_env_vars == ("UPSTAGE_API_KEY",) + assert overlay.base_url_override == "https://api.upstage.ai/v1" + assert overlay.base_url_env_var == "UPSTAGE_BASE_URL" + assert not overlay.is_aggregator + + def test_provider_label(self): + from hermes_cli.providers import get_label + + assert get_label("upstage") == "Upstage Solar" + + +class TestUpstageConfigProviderWins: + """End-to-end: an explicit config provider must beat env auto-detect. + + Mirrors the display logic in `hermes_cli/main.py` (cmd_model): read + `model.provider`, resolve it, and only fall back to auto-detect when that + resolution fails. With a stray DEEPSEEK_API_KEY present (the user's case), + upstage must still win because it is configured explicitly. + """ + + def test_explicit_upstage_beats_stray_deepseek_key(self, monkeypatch): + from hermes_cli.providers import resolve_provider_full + + monkeypatch.setenv("DEEPSEEK_API_KEY", "junk") + monkeypatch.setenv("UPSTAGE_API_KEY", "up-test-key") + + config_provider = "upstage" # from config model.provider + active = "" + if config_provider and config_provider != "auto": + adef = resolve_provider_full(config_provider, {}, []) + active = adef.id if adef is not None else "" + + assert active == "upstage", ( + "explicit config provider should resolve to upstage, not fall " + "through to deepseek auto-detect" + ) diff --git a/tests/plugins/model_providers/test_upstage_profile.py b/tests/plugins/model_providers/test_upstage_profile.py new file mode 100644 index 000000000000..591408d71b76 --- /dev/null +++ b/tests/plugins/model_providers/test_upstage_profile.py @@ -0,0 +1,153 @@ +"""Unit tests for the Upstage Solar provider profile. + +Upstage Solar is a plain OpenAI-compatible api-key provider, so this verifies +the profile is registered correctly and wires the expected identity, endpoint, +auth, and catalog fields — the contract every downstream layer (auth, models, +doctor, runtime_provider, transport) reads from. +""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def upstage_profile(): + """Resolve the registered Upstage profile via the provider registry. + + Importing ``model_tools`` triggers plugin discovery, which registers the + Upstage profile. Going through ``get_provider_profile`` keeps the test + honest about the actual registration path (name + alias resolution). + """ + import model_tools # noqa: F401 + import providers + + profile = providers.get_provider_profile("upstage") + assert profile is not None, "upstage provider profile must be registered" + return profile + + +class TestUpstageProfile: + def test_identity_and_endpoint(self, upstage_profile): + assert upstage_profile.name == "upstage" + assert upstage_profile.api_mode == "chat_completions" + assert upstage_profile.auth_type == "api_key" + assert upstage_profile.base_url == "https://api.upstage.ai/v1" + assert upstage_profile.get_hostname() == "api.upstage.ai" + + def test_solar_alias_resolves(self): + import model_tools # noqa: F401 + import providers + + assert providers.get_provider_profile("solar") is upstage_profile_singleton() + + def test_env_vars(self, upstage_profile): + # API key first, optional base-url override second (priority order). + assert upstage_profile.env_vars == ("UPSTAGE_API_KEY", "UPSTAGE_BASE_URL") + + def test_fallback_models_are_agentic_pro_only(self, upstage_profile): + # Only the agentic, tool-calling Solar Pro models belong in the offline + # catalog — Mini is capable but not agentic, so it's never promoted as a + # default. Live /v1/models still surfaces everything when a key is set. + assert upstage_profile.fallback_models == ( + "solar-pro", + "solar-pro3", + ) + + def test_default_model_is_solar_pro(self, upstage_profile): + # Entry [0] is the setup default (get_default_model_for_provider). + assert upstage_profile.fallback_models[0] == "solar-pro" + + def test_aux_model_left_empty(self, upstage_profile): + # Unset → auxiliary side tasks fall back to the user's main model. + assert upstage_profile.default_aux_model == "" + + +class TestUpstageReasoning: + """``build_api_kwargs_extras`` wires Solar's top-level ``reasoning_effort``. + + Solar Pro accepts ``reasoning_effort`` (minimal|low|medium|high, default + minimal=off) and never requires echoing ``reasoning_content`` back, so only + the request field is emitted — always top-level, never in extra_body. + """ + + @pytest.mark.parametrize("effort", ["low", "medium", "high"]) + def test_pro_explicit_effort_passes_through(self, upstage_profile, effort): + extra_body, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": effort}, model="solar-pro3" + ) + assert extra_body == {} + assert top_level == {"reasoning_effort": effort} + + @pytest.mark.parametrize("effort", ["xhigh", "max"]) + def test_pro_strong_efforts_collapse_to_high(self, upstage_profile, effort): + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": effort}, model="solar-pro2" + ) + assert top_level == {"reasoning_effort": "high"} + + def test_pro_enabled_without_effort_defaults_on(self, upstage_profile): + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True}, model="solar-pro3" + ) + assert top_level == {"reasoning_effort": "medium"} + + def test_pro_minimal_effort_is_omitted(self, upstage_profile): + # Explicit minimal == reasoning off → omit so Solar applies its default. + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "minimal"}, model="solar-pro3" + ) + assert top_level == {} + + def test_disabled_omits_field(self, upstage_profile): + # `/reasoning none` → enabled False → explicitly off. + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": False, "effort": "high"}, model="solar-pro3" + ) + assert top_level == {} + + @pytest.mark.parametrize("model", ["solar-pro3", "solar-pro", "solar-open2"]) + def test_no_config_defaults_reasoning_on(self, upstage_profile, model): + # Unset reasoning_config → default ON at medium (matches the /reasoning + # "medium (default)" label), not Solar's server default of minimal/off. + _, top_level = upstage_profile.build_api_kwargs_extras(model=model) + assert top_level == {"reasoning_effort": "medium"} + + @pytest.mark.parametrize("model", ["solar-mini", "syn-pro"]) + def test_no_config_non_pro_still_omits(self, upstage_profile, model): + # Default-on must not leak to non-reasoning models. + _, top_level = upstage_profile.build_api_kwargs_extras(model=model) + assert top_level == {} + + @pytest.mark.parametrize( + "model", + [ + "solar-pro3-250127", + "solar-open", + "solar-open-250127", + "solar-open2", + "solar-open2-260528", + ], + ) + def test_pro_and_open_variants_support_reasoning(self, upstage_profile, model): + # Both the Solar Pro and Solar Open families (incl. dated variants) + # accept reasoning_effort. + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "high"}, model=model + ) + assert top_level == {"reasoning_effort": "high"} + + @pytest.mark.parametrize("model", ["solar-mini", "syn-pro"]) + def test_non_pro_models_never_send_reasoning(self, upstage_profile, model): + # solar-mini / syn-pro ignore reasoning_effort, so never send it. + extra_body, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "high"}, model=model + ) + assert extra_body == {} + assert top_level == {} + + +def upstage_profile_singleton(): + import providers + + return providers.get_provider_profile("upstage") diff --git a/web/src/pages/EnvPage.tsx b/web/src/pages/EnvPage.tsx index 4262b6c8f399..edcc45844cb2 100644 --- a/web/src/pages/EnvPage.tsx +++ b/web/src/pages/EnvPage.tsx @@ -65,6 +65,7 @@ const PROVIDER_GROUPS: { prefix: string; name: string; priority: number }[] = [ { prefix: "OPENCODE_ZEN_", name: "OpenCode Zen", priority: 11 }, { prefix: "OPENROUTER_", name: "OpenRouter", priority: 12 }, { prefix: "XIAOMI_", name: "Xiaomi MiMo", priority: 13 }, + { prefix: "UPSTAGE_", name: "Upstage Solar", priority: 14 }, ]; function getProviderGroup(key: string): string { From 0031c5c3715cca9650683c89cc5aa1dccf8a5ce4 Mon Sep 17 00:00:00 2001 From: Changhyun Min Date: Thu, 11 Jun 2026 17:40:46 +0900 Subject: [PATCH 083/149] refactor(agent): treat unknown Solar models as reasoning-capable Invert the reasoning-support check from an allow-list (solar-pro, solar-open) to a deny-list of the known non-reasoning families (solar-mini, syn-pro). Newly released Solar models now get reasoning_effort by default instead of having it silently dropped. Co-Authored-By: Claude Fable 5 --- plugins/model-providers/upstage/__init__.py | 26 ++++++++++----- .../model_providers/test_upstage_profile.py | 33 +++++++++++++++---- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/plugins/model-providers/upstage/__init__.py b/plugins/model-providers/upstage/__init__.py index 03e954e3e4ea..982e741f9e86 100644 --- a/plugins/model-providers/upstage/__init__.py +++ b/plugins/model-providers/upstage/__init__.py @@ -6,10 +6,12 @@ from providers import register_provider from providers.base import ProviderProfile -# Model-name markers for Solar families that accept ``reasoning_effort``. -# Substring match (not startswith) so dated variants like -# ``solar-pro3-250127`` and ``solar-open-...`` are covered too. -_REASONING_MODEL_MARKERS = ("solar-pro", "solar-open") +# Model-name markers for Solar families that do NOT accept ``reasoning_effort``. +# Deny-list on purpose: newly released Solar models are assumed +# reasoning-capable by default, so only the known non-reasoning families are +# listed here. Substring match (not startswith) so dated variants like +# ``solar-mini-250127`` are covered too. +_NON_REASONING_MODEL_MARKERS = ("solar-mini", "syn-pro") # When the user hasn't picked a reasoning effort, Hermes passes # reasoning_config=None. Solar's own server default is "minimal" (reasoning @@ -22,15 +24,20 @@ _DEFAULT_REASONING_EFFORT = "medium" def _model_supports_reasoning(model: str | None) -> bool: - """Solar reasoning-capable models. + """Solar reasoning-capable models — True unless the model is deny-listed. The Solar Pro family (``solar-pro``, ``solar-pro2``, ``solar-pro3`` and dated variants like ``solar-pro3-250127``) and the Solar Open family - (``solar-open*``) accept ``reasoning_effort``. ``solar-mini`` / ``syn-pro`` - ignore the parameter, so we don't send it. + (``solar-open*``) accept ``reasoning_effort``; only ``solar-mini`` / + ``syn-pro`` ignore the parameter, so we deny-list those and treat every + other (incl. future) Solar model as reasoning-capable. + + ``None``/empty model → True: the provider default (``fallback_models[0]``, + the ``solar-pro`` rolling alias) is reasoning-capable, so an unset model + gets the same default-on behaviour. """ m = (model or "").strip().lower() - return any(marker in m for marker in _REASONING_MODEL_MARKERS) + return not any(marker in m for marker in _NON_REASONING_MODEL_MARKERS) class UpstageProfile(ProviderProfile): @@ -54,7 +61,8 @@ class UpstageProfile(ProviderProfile): ) -> tuple[dict[str, Any], dict[str, Any]]: top_level: dict[str, Any] = {} - # solar-mini / syn-pro ignore reasoning_effort — send nothing. + # solar-mini / syn-pro (the deny-list) ignore reasoning_effort — send + # nothing. Everything else, including future Solar models, gets it. if not _model_supports_reasoning(model): return {}, top_level diff --git a/tests/plugins/model_providers/test_upstage_profile.py b/tests/plugins/model_providers/test_upstage_profile.py index 591408d71b76..e78c2326ee11 100644 --- a/tests/plugins/model_providers/test_upstage_profile.py +++ b/tests/plugins/model_providers/test_upstage_profile.py @@ -113,9 +113,9 @@ class TestUpstageReasoning: _, top_level = upstage_profile.build_api_kwargs_extras(model=model) assert top_level == {"reasoning_effort": "medium"} - @pytest.mark.parametrize("model", ["solar-mini", "syn-pro"]) - def test_no_config_non_pro_still_omits(self, upstage_profile, model): - # Default-on must not leak to non-reasoning models. + @pytest.mark.parametrize("model", ["solar-mini", "solar-mini-202610", "syn-pro"]) + def test_no_config_deny_listed_still_omits(self, upstage_profile, model): + # Default-on must not leak to the deny-listed non-reasoning models. _, top_level = upstage_profile.build_api_kwargs_extras(model=model) assert top_level == {} @@ -137,15 +137,36 @@ class TestUpstageReasoning: ) assert top_level == {"reasoning_effort": "high"} - @pytest.mark.parametrize("model", ["solar-mini", "syn-pro"]) - def test_non_pro_models_never_send_reasoning(self, upstage_profile, model): - # solar-mini / syn-pro ignore reasoning_effort, so never send it. + @pytest.mark.parametrize("model", ["solar-mini", "solar-mini-202610", "syn-pro"]) + def test_deny_listed_models_never_send_reasoning(self, upstage_profile, model): + # solar-mini / syn-pro ignore reasoning_effort, so never send it — + # even when the user explicitly enables reasoning. extra_body, top_level = upstage_profile.build_api_kwargs_extras( reasoning_config={"enabled": True, "effort": "high"}, model=model ) assert extra_body == {} assert top_level == {} + @pytest.mark.parametrize("model", ["solar-future", "solar-future-260601"]) + def test_unknown_future_models_default_to_reasoning(self, upstage_profile, model): + # Deny-list semantics: a future Solar model we've never heard of is + # assumed reasoning-capable, so reasoning_effort is sent instead of + # being silently dropped (the old allow-list failure mode). + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "high"}, model=model + ) + assert top_level == {"reasoning_effort": "high"} + + # And the unset-config default-on path applies to it too. + _, top_level = upstage_profile.build_api_kwargs_extras(model=model) + assert top_level == {"reasoning_effort": "medium"} + + def test_none_model_defaults_to_reasoning(self, upstage_profile): + # No model in context → treated as reasoning-capable, consistent with + # the provider default (fallback_models[0] == "solar-pro"). + _, top_level = upstage_profile.build_api_kwargs_extras(model=None) + assert top_level == {"reasoning_effort": "medium"} + def upstage_profile_singleton(): import providers From c1e36f4329d78990cd457ed3cfbdd2bd4e60ee63 Mon Sep 17 00:00:00 2001 From: Changhyun Min Date: Thu, 2 Jul 2026 13:15:42 +0900 Subject: [PATCH 084/149] fix(agent): register Upstage keys in the env-var catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `UPSTAGE_API_KEY` / `UPSTAGE_BASE_URL` were wired through the provider resolver, auth registry, and the EnvPage grouping, but never added to `OPTIONAL_ENV_VARS` in hermes_cli/config.py. The dashboard/desktop Providers page builds its list from that catalog (`/api/env` iterates `OPTIONAL_ENV_VARS`), so with no entry the keys were never emitted and "Upstage Solar" never rendered — the EnvPage prefix group stayed empty. Add both keys under `category: "provider"` (matching gmi/minimax) so they show up in `hermes dashboard` / `hermes desktop` under "Upstage Solar". Adds a regression test asserting the catalog contains them, mirroring the existing GMI coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- hermes_cli/config.py | 15 +++++++++++++++ tests/hermes_cli/test_upstage_provider.py | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/hermes_cli/config.py b/hermes_cli/config.py index b95b1c666a89..0754933d6082 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3707,6 +3707,21 @@ OPTIONAL_ENV_VARS = { "category": "provider", "advanced": True, }, + "UPSTAGE_API_KEY": { + "description": "Upstage API key for Solar LLM models", + "prompt": "Upstage API Key", + "url": "https://console.upstage.ai/api-keys", + "password": True, + "category": "provider", + }, + "UPSTAGE_BASE_URL": { + "description": "Upstage base URL override (default: https://api.upstage.ai/v1)", + "prompt": "Upstage base URL (leave empty for default)", + "url": None, + "password": False, + "category": "provider", + "advanced": True, + }, "AWS_REGION": { "description": "AWS region for Bedrock API calls (e.g. us-east-1, eu-central-1)", "prompt": "AWS Region", diff --git a/tests/hermes_cli/test_upstage_provider.py b/tests/hermes_cli/test_upstage_provider.py index 621b86f8d413..4e2829f61419 100644 --- a/tests/hermes_cli/test_upstage_provider.py +++ b/tests/hermes_cli/test_upstage_provider.py @@ -68,6 +68,26 @@ class TestUpstageOverlay: assert get_label("upstage") == "Upstage Solar" +class TestUpstageEnvCatalog: + """The dashboard/desktop Providers page lists only OPTIONAL_ENV_VARS keys + whose category is "provider". Without these entries UPSTAGE_API_KEY / + UPSTAGE_BASE_URL never reach the frontend and Upstage stays invisible even + though EnvPage.tsx has a matching PROVIDER_GROUPS prefix. + """ + + def test_optional_env_vars_include_upstage(self): + from hermes_cli.config import OPTIONAL_ENV_VARS + + assert "UPSTAGE_API_KEY" in OPTIONAL_ENV_VARS + assert OPTIONAL_ENV_VARS["UPSTAGE_API_KEY"]["category"] == "provider" + assert OPTIONAL_ENV_VARS["UPSTAGE_API_KEY"]["password"] is True + assert OPTIONAL_ENV_VARS["UPSTAGE_API_KEY"]["url"] + + assert "UPSTAGE_BASE_URL" in OPTIONAL_ENV_VARS + assert OPTIONAL_ENV_VARS["UPSTAGE_BASE_URL"]["category"] == "provider" + assert OPTIONAL_ENV_VARS["UPSTAGE_BASE_URL"]["password"] is False + + class TestUpstageConfigProviderWins: """End-to-end: an explicit config provider must beat env auto-detect. From 5f3d57400b46d1e0d2056c49fbdee9872ad9081f Mon Sep 17 00:00:00 2001 From: Changhyun Min Date: Thu, 2 Jul 2026 13:15:42 +0900 Subject: [PATCH 085/149] refactor(agent): drop solar-open2-preview from Solar context fallbacks Remove the `solar-open2-preview` context-window entry; `solar-open2` covers the Open 2 family at the same 256K window. Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/model_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 00e6e4b3d53e..d3223002eed0 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -325,7 +325,6 @@ DEFAULT_CONTEXT_LENGTHS = { # future solar-pro* releases at the Pro context size. # Sources: Solar Pro 3 = 128K, Solar Pro 2 = 64K, Solar Mini = 32K, # Solar Open 2 = 256K. - "solar-open2-preview": 262144, # 256K (longest-first: wins over solar-open2) "solar-open2": 262144, # 256K "solar-pro3": 131072, "solar-pro": 131072, # rolling alias → latest Solar Pro (currently pro3) From 35d3fc3b09142f4ce70b7d6d9d0396caecbf95e0 Mon Sep 17 00:00:00 2001 From: Changhyun Min Date: Thu, 2 Jul 2026 13:26:05 +0900 Subject: [PATCH 086/149] refactor(agent): drop the solar-pro rolling alias, default to solar-pro3 Pin the Upstage default to the concrete solar-pro3 instead of the solar-pro rolling alias: - plugin fallback_models is now ("solar-pro3",); entry [0] is the setup default - drop the "solar-pro" context-window fallback entry (solar-pro3 covers it) - update the reasoning default-on docstring and profile tests accordingly Co-Authored-By: Claude Opus 4.8 (1M context) --- agent/model_metadata.py | 6 ++---- plugins/model-providers/upstage/__init__.py | 8 +++----- tests/plugins/model_providers/test_upstage_profile.py | 7 +++---- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index d3223002eed0..03c05177b1f6 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -320,14 +320,12 @@ DEFAULT_CONTEXT_LENGTHS = { "kimi": 262144, # Upstage Solar — api.upstage.ai/v1/models does not return context_length, # so these fallbacks keep token budgeting / compression from probing down - # to the 128k default. Substring matching is longest-first, so the versioned - # ids win over the "solar-pro" rolling-alias entry, which in turn covers - # future solar-pro* releases at the Pro context size. + # to the 128k default. Ids are matched longest-first, so dated variants + # (e.g. solar-pro3-250127) resolve via their family prefix. # Sources: Solar Pro 3 = 128K, Solar Pro 2 = 64K, Solar Mini = 32K, # Solar Open 2 = 256K. "solar-open2": 262144, # 256K "solar-pro3": 131072, - "solar-pro": 131072, # rolling alias → latest Solar Pro (currently pro3) "solar-pro2": 65536, "solar-mini": 32768, # Tencent — Hy3 Preview (Hunyuan) with 256K context window. diff --git a/plugins/model-providers/upstage/__init__.py b/plugins/model-providers/upstage/__init__.py index 982e741f9e86..7bc202d4bf6a 100644 --- a/plugins/model-providers/upstage/__init__.py +++ b/plugins/model-providers/upstage/__init__.py @@ -33,8 +33,8 @@ def _model_supports_reasoning(model: str | None) -> bool: other (incl. future) Solar model as reasoning-capable. ``None``/empty model → True: the provider default (``fallback_models[0]``, - the ``solar-pro`` rolling alias) is reasoning-capable, so an unset model - gets the same default-on behaviour. + ``solar-pro3``) is reasoning-capable, so an unset model gets the same + default-on behaviour. """ m = (model or "").strip().lower() return not any(marker in m for marker in _NON_REASONING_MODEL_MARKERS) @@ -103,10 +103,8 @@ upstage = UpstageProfile( base_url="https://api.upstage.ai/v1", auth_type="api_key", # default_aux_model left empty → auxiliary side tasks use the main model. - # entry [0] is the setup default. solar-pro is a rolling alias for the - # latest Solar Pro, so the default tracks the current flagship. + # entry [0] is the setup default — solar-pro3, the current Solar Pro flagship. fallback_models=( - "solar-pro", "solar-pro3", ), ) diff --git a/tests/plugins/model_providers/test_upstage_profile.py b/tests/plugins/model_providers/test_upstage_profile.py index e78c2326ee11..f37971f2f791 100644 --- a/tests/plugins/model_providers/test_upstage_profile.py +++ b/tests/plugins/model_providers/test_upstage_profile.py @@ -50,13 +50,12 @@ class TestUpstageProfile: # catalog — Mini is capable but not agentic, so it's never promoted as a # default. Live /v1/models still surfaces everything when a key is set. assert upstage_profile.fallback_models == ( - "solar-pro", "solar-pro3", ) - def test_default_model_is_solar_pro(self, upstage_profile): + def test_default_model_is_solar_pro3(self, upstage_profile): # Entry [0] is the setup default (get_default_model_for_provider). - assert upstage_profile.fallback_models[0] == "solar-pro" + assert upstage_profile.fallback_models[0] == "solar-pro3" def test_aux_model_left_empty(self, upstage_profile): # Unset → auxiliary side tasks fall back to the user's main model. @@ -163,7 +162,7 @@ class TestUpstageReasoning: def test_none_model_defaults_to_reasoning(self, upstage_profile): # No model in context → treated as reasoning-capable, consistent with - # the provider default (fallback_models[0] == "solar-pro"). + # the provider default (fallback_models[0] == "solar-pro3"). _, top_level = upstage_profile.build_api_kwargs_extras(model=None) assert top_level == {"reasoning_effort": "medium"} From 899e420ab9c121254de6db3e8513d570178f9607 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:41:15 +0530 Subject: [PATCH 087/149] refactor(upstage): drop manual auth/models registrations covered by profile auto-extend PROVIDER_REGISTRY, its alias map, and CANONICAL_PROVIDERS all auto-extend from registered ProviderProfiles since the provider-modules refactor (20a4f79ed). Verified with real imports: registry entry, 'solar' alias resolution via resolve_provider(), and the picker entry are identical with the manual entries removed. The hermes_cli/providers.py overlay stays (models.dev has a stale /v1/solar base URL and no UPSTAGE_BASE_URL var), and the manual OPTIONAL_ENV_VARS entries stay (non-advanced key + curated prompt text, matching the fireworks convention). --- hermes_cli/auth.py | 9 --------- hermes_cli/models.py | 1 - 2 files changed, 10 deletions(-) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 9d6db629ed20..8804bcbeca15 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -290,14 +290,6 @@ PROVIDER_REGISTRY: Dict[str, ProviderConfig] = { api_key_env_vars=("GMI_API_KEY",), base_url_env_var="GMI_BASE_URL", ), - "upstage": ProviderConfig( - id="upstage", - name="Upstage Solar", - auth_type="api_key", - inference_base_url="https://api.upstage.ai/v1", - api_key_env_vars=("UPSTAGE_API_KEY",), - base_url_env_var="UPSTAGE_BASE_URL", - ), "minimax": ProviderConfig( id="minimax", name="MiniMax", @@ -1685,7 +1677,6 @@ def resolve_provider( "step": "stepfun", "stepfun-coding-plan": "stepfun", "arcee-ai": "arcee", "arceeai": "arcee", "gmi-cloud": "gmi", "gmicloud": "gmi", - "solar": "upstage", "minimax-china": "minimax-cn", "minimax_cn": "minimax-cn", "minimax-portal": "minimax-oauth", "minimax-global": "minimax-oauth", "minimax_oauth": "minimax-oauth", "alibaba_coding": "alibaba-coding-plan", "alibaba-coding": "alibaba-coding-plan", diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 58852e551882..d81c409814c7 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -1088,7 +1088,6 @@ CANONICAL_PROVIDERS: list[ProviderEntry] = [ ProviderEntry("bedrock", "AWS Bedrock", "AWS Bedrock (Claude, Nova, Llama, DeepSeek; IAM or API key)"), ProviderEntry("azure-foundry", "Azure Foundry", "Azure Foundry (OpenAI-style or Anthropic-style endpoint, your Azure AI deployment)"), ProviderEntry("qwen-oauth", "Qwen OAuth (Portal)", "Qwen OAuth (Reuses local Qwen CLI login)"), - ProviderEntry("upstage", "Upstage Solar", "Upstage (Solar API)"), ] # Auto-extend CANONICAL_PROVIDERS with any provider registered in providers/ From 0d01831919c0ce32a8e34fe735ec8536303c471f Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:43:30 +0530 Subject: [PATCH 088/149] chore: add changhyun.min@gmail.com to AUTHOR_MAP (minchang, PR #42231) --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index ce71d417a916..556b0ee7bf59 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json" # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "changhyun.min@gmail.com": "minchang", # PR #42231 salvage (providers: add Upstage Solar) "neo@neodeMac-mini.local": "neo-claw-bot", # PR #58465 salvage (moa: drop empty user turns from advisory view) "m.guttmann@journaway.com": "mguttmann", # PR #63738 salvage (Anthropic setup-token pool auth normalization) "VrtxOmega@pm.me": "VrtxOmega", # PR #43809 salvage (desktop: WSL folder-picker path bridge) From f88cac71bc9576bed1d51eced6bf45ac59cb6c7c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:46:08 +0530 Subject: [PATCH 089/149] fix(upstage): map 'ultra' reasoning effort to Solar's high Main added max/ultra effort levels (#62650) after this PR branched; without the mapping 'ultra' silently fell through to the medium default. Matches the xhigh/max collapse-to-strongest convention used by other profiles. --- plugins/model-providers/upstage/__init__.py | 5 +++-- tests/plugins/model_providers/test_upstage_profile.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/model-providers/upstage/__init__.py b/plugins/model-providers/upstage/__init__.py index 7bc202d4bf6a..d36c0734a3a9 100644 --- a/plugins/model-providers/upstage/__init__.py +++ b/plugins/model-providers/upstage/__init__.py @@ -75,8 +75,8 @@ class UpstageProfile(ProviderProfile): if reasoning_config.get("enabled") is False: return {}, top_level - # Map Hermes' effort vocabulary onto Solar's accepted set. xhigh/max - # collapse to high (Solar's strongest). minimal → off (omit). An + # Map Hermes' effort vocabulary onto Solar's accepted set. xhigh/max/ + # ultra collapse to high (Solar's strongest). minimal → off (omit). An # enabled request with no recognised effort uses the default effort. effort = (reasoning_config.get("effort") or "").strip().lower() mapped = { @@ -86,6 +86,7 @@ class UpstageProfile(ProviderProfile): "high": "high", "xhigh": "high", "max": "high", + "ultra": "high", }.get(effort, _DEFAULT_REASONING_EFFORT) if mapped: diff --git a/tests/plugins/model_providers/test_upstage_profile.py b/tests/plugins/model_providers/test_upstage_profile.py index f37971f2f791..257900aed198 100644 --- a/tests/plugins/model_providers/test_upstage_profile.py +++ b/tests/plugins/model_providers/test_upstage_profile.py @@ -78,7 +78,7 @@ class TestUpstageReasoning: assert extra_body == {} assert top_level == {"reasoning_effort": effort} - @pytest.mark.parametrize("effort", ["xhigh", "max"]) + @pytest.mark.parametrize("effort", ["xhigh", "max", "ultra"]) def test_pro_strong_efforts_collapse_to_high(self, upstage_profile, effort): _, top_level = upstage_profile.build_api_kwargs_extras( reasoning_config={"enabled": True, "effort": effort}, model="solar-pro2" From 33dfb7e4ec17fb1adebab7cc63d16c4cacbcfedd Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:48:18 +0530 Subject: [PATCH 090/149] docs: add Upstage Solar to provider docs (env vars, fallback table, --provider list) --- website/docs/reference/cli-commands.md | 2 +- website/docs/reference/environment-variables.md | 2 ++ website/docs/user-guide/features/fallback-providers.md | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index 804a25aab411..ae6fe70fc21f 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -105,7 +105,7 @@ Common options: | `-q`, `--query "..."` | One-shot, non-interactive prompt. | | `-m`, `--model ` | Override the model for this run. | | `-t`, `--toolsets ` | Enable a comma-separated set of toolsets. | -| `--provider ` | Force a provider: `auto`, `openrouter`, `nous`, `openai-codex`, `copilot-acp`, `copilot`, `anthropic`, `gemini`, `huggingface`, `novita` (aliases `novita-ai`, `novitaai`), `openai-api`, `zai`, `kimi-coding`, `kimi-coding-cn`, `minimax`, `minimax-cn`, `minimax-oauth`, `kilocode`, `xiaomi`, `arcee`, `gmi`, `alibaba`, `alibaba-coding-plan` (alias `alibaba_coding`), `deepseek`, `nvidia`, `ollama-cloud`, `xai` (alias `grok`), `xai-oauth` (alias `grok-oauth`), `qwen-oauth`, `bedrock`, `opencode-zen`, `opencode-go`, `azure-foundry`, `lmstudio`, `stepfun`, `tencent-tokenhub` (alias `tencent`, `tokenhub`). | +| `--provider ` | Force a provider: `auto`, `openrouter`, `nous`, `openai-codex`, `copilot-acp`, `copilot`, `anthropic`, `gemini`, `huggingface`, `novita` (aliases `novita-ai`, `novitaai`), `openai-api`, `zai`, `kimi-coding`, `kimi-coding-cn`, `minimax`, `minimax-cn`, `minimax-oauth`, `kilocode`, `xiaomi`, `arcee`, `gmi`, `upstage` (alias `solar`), `alibaba`, `alibaba-coding-plan` (alias `alibaba_coding`), `deepseek`, `nvidia`, `ollama-cloud`, `xai` (alias `grok`), `xai-oauth` (alias `grok-oauth`), `qwen-oauth`, `bedrock`, `opencode-zen`, `opencode-go`, `azure-foundry`, `lmstudio`, `stepfun`, `tencent-tokenhub` (alias `tencent`, `tokenhub`). | | `-s`, `--skills ` | Preload one or more skills for the session (can be repeated or comma-separated). | | `-v`, `--verbose` | Verbose output. | | `-Q`, `--quiet` | Programmatic mode: suppress banner/spinner/tool previews. | diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 0a449252b634..b41cbe7e24b5 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -51,6 +51,8 @@ Hermes reads environment variables from the process environment and, for user-ma | `KILOCODE_BASE_URL` | Override Kilo Code base URL (default: `https://api.kilo.ai/api/gateway`) | | `XIAOMI_API_KEY` | Xiaomi MiMo API key ([platform.xiaomimimo.com](https://platform.xiaomimimo.com)) | | `XIAOMI_BASE_URL` | Override Xiaomi MiMo base URL (default: `https://api.xiaomimimo.com/v1`) | +| `UPSTAGE_API_KEY` | Upstage API key for Solar models ([console.upstage.ai](https://console.upstage.ai/api-keys)) | +| `UPSTAGE_BASE_URL` | Override Upstage base URL (default: `https://api.upstage.ai/v1`) | | `TOKENHUB_API_KEY` | Tencent TokenHub API key ([tokenhub.tencentmaas.com](https://tokenhub.tencentmaas.com)) | | `TOKENHUB_BASE_URL` | Override Tencent TokenHub base URL (default: `https://tokenhub.tencentmaas.com/v1`) | | `AZURE_FOUNDRY_API_KEY` | Microsoft Foundry / Azure OpenAI API key ([ai.azure.com](https://ai.azure.com/)). Not needed when `model.auth_mode: entra_id` | diff --git a/website/docs/user-guide/features/fallback-providers.md b/website/docs/user-guide/features/fallback-providers.md index ff6cbf6f9ef0..4adbc72901cb 100644 --- a/website/docs/user-guide/features/fallback-providers.md +++ b/website/docs/user-guide/features/fallback-providers.md @@ -60,6 +60,7 @@ Each entry requires both `provider` and `model`. Entries missing either field ar | DeepSeek | `deepseek` | `DEEPSEEK_API_KEY` | | NVIDIA NIM | `nvidia` | `NVIDIA_API_KEY` (optional: `NVIDIA_BASE_URL`) | | GMI Cloud | `gmi` | `GMI_API_KEY` (optional: `GMI_BASE_URL`) | +| Upstage Solar | `upstage` (alias `solar`) | `UPSTAGE_API_KEY` (optional: `UPSTAGE_BASE_URL`) | | StepFun | `stepfun` | `STEPFUN_API_KEY` (optional: `STEPFUN_BASE_URL`) | | Ollama Cloud | `ollama-cloud` | `OLLAMA_API_KEY` | | Google AI Studio | `gemini` | `GOOGLE_API_KEY` (alias: `GEMINI_API_KEY`) | From 1f41bdbecda218b33f6c918760d2600c50076d51 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:03:37 +0530 Subject: [PATCH 091/149] fix(upstage): collapse unknown future efforts to high; behavior-contract tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings from the 4-angle pass: - Unknown-but-enabled effort levels now collapse to Solar's strongest (high) instead of silently downgrading to the medium default — guards against the next #62650-style vocabulary addition. Explicit-empty effort keeps the medium default. - fallback_models test now asserts the behavior contract (non-empty, no denied families) instead of freezing the exact model tuple (change-detector, AGENTS.md reject reason). - Drop unused pytest import in test_upstage_provider.py. --- plugins/model-providers/upstage/__init__.py | 14 ++++++++------ tests/hermes_cli/test_upstage_provider.py | 1 - .../model_providers/test_upstage_profile.py | 19 ++++++++++++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/plugins/model-providers/upstage/__init__.py b/plugins/model-providers/upstage/__init__.py index d36c0734a3a9..b050051badef 100644 --- a/plugins/model-providers/upstage/__init__.py +++ b/plugins/model-providers/upstage/__init__.py @@ -76,18 +76,20 @@ class UpstageProfile(ProviderProfile): return {}, top_level # Map Hermes' effort vocabulary onto Solar's accepted set. xhigh/max/ - # ultra collapse to high (Solar's strongest). minimal → off (omit). An - # enabled request with no recognised effort uses the default effort. + # ultra collapse to high (Solar's strongest). minimal → off (omit). + # Unknown-but-enabled efforts (future vocabulary additions above + # "high", per the max/ultra precedent in #62650) also collapse to + # high rather than silently downgrading to the medium default. effort = (reasoning_config.get("effort") or "").strip().lower() + if not effort: + top_level["reasoning_effort"] = _DEFAULT_REASONING_EFFORT + return {}, top_level mapped = { "minimal": None, "low": "low", "medium": "medium", "high": "high", - "xhigh": "high", - "max": "high", - "ultra": "high", - }.get(effort, _DEFAULT_REASONING_EFFORT) + }.get(effort, "high") if mapped: top_level["reasoning_effort"] = mapped diff --git a/tests/hermes_cli/test_upstage_provider.py b/tests/hermes_cli/test_upstage_provider.py index 4e2829f61419..6f7da10761f7 100644 --- a/tests/hermes_cli/test_upstage_provider.py +++ b/tests/hermes_cli/test_upstage_provider.py @@ -12,7 +12,6 @@ from __future__ import annotations import sys import types -import pytest if "dotenv" not in sys.modules: fake_dotenv = types.ModuleType("dotenv") diff --git a/tests/plugins/model_providers/test_upstage_profile.py b/tests/plugins/model_providers/test_upstage_profile.py index 257900aed198..ce227b675d8f 100644 --- a/tests/plugins/model_providers/test_upstage_profile.py +++ b/tests/plugins/model_providers/test_upstage_profile.py @@ -49,9 +49,12 @@ class TestUpstageProfile: # Only the agentic, tool-calling Solar Pro models belong in the offline # catalog — Mini is capable but not agentic, so it's never promoted as a # default. Live /v1/models still surfaces everything when a key is set. - assert upstage_profile.fallback_models == ( - "solar-pro3", - ) + # Behavior contract (not a frozen list): non-empty, no denied families. + assert upstage_profile.fallback_models + for denied in ("solar-mini", "syn-pro"): + assert not any( + denied in m for m in upstage_profile.fallback_models + ), f"non-agentic family {denied!r} must not be a fallback default" def test_default_model_is_solar_pro3(self, upstage_profile): # Entry [0] is the setup default (get_default_model_for_provider). @@ -85,6 +88,16 @@ class TestUpstageReasoning: ) assert top_level == {"reasoning_effort": "high"} + def test_unknown_future_effort_collapses_to_high(self, upstage_profile): + # Guard against the #62650 recurrence: a future effort level Hermes + # adds above "high" must collapse to Solar's strongest, not silently + # downgrade to the medium default. + _, top_level = upstage_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "hyperthink"}, + model="solar-pro3", + ) + assert top_level == {"reasoning_effort": "high"} + def test_pro_enabled_without_effort_defaults_on(self, upstage_profile): _, top_level = upstage_profile.build_api_kwargs_extras( reasoning_config={"enabled": True}, model="solar-pro3" From d9cdb81923e4953a9a80129c3df08accb60978d7 Mon Sep 17 00:00:00 2001 From: ScotterMonk <21178861+ScotterMonk@users.noreply.github.com> Date: Tue, 23 Jun 2026 08:30:17 -0500 Subject: [PATCH 092/149] feat(config): support per-model reasoning_effort overrides Add agent.reasoning_overrides dict to config.yaml. Users can now set a reasoning_effort per model, overriding the global agent.reasoning_effort. Example: agent: reasoning_effort: "medium" # global default reasoning_overrides: "openrouter/anthropic/claude-opus-4.5": "xhigh" "openai/gpt-5": "low" "claude-sonnet-4.6": "high" # bare model name also works The helper is spelling-tolerant: override keys match regardless of provider prefix or dots-vs-dashes normalization, so users can write keys in any sensible form and they'll match. Resolution priority: 1. Session-scoped /reasoning --session override (gateway only; unchanged) 2. Per-model override from agent.reasoning_overrides (spelling-tolerant) 3. Global agent.reasoning_effort (existing) 4. Provider default (unchanged) Wired into: - CLI startup (cli.py) - Messaging gateway agent construction (gateway/run.py) - Desktop/TUI _load_reasoning_config (tui_gateway/server.py) - Cron job scheduler (cron/scheduler.py) - /model mid-session switch (agent/agent_runtime_helpers.py) + _primary_runtime now tracks reasoning_config for correct fallback recovery - Fallback activation (agent/chat_completion_helpers.py::try_activate_fallback) + Re-resolves reasoning_config for the fallback model (best-effort) Closes #21256 (per-model reasoning_effort defaults). Note: no hermes config set agent.reasoning_overrides. support; users edit the YAML directly. _set_nested splits on "." and would corrupt model keys containing version dots. --- agent/agent_runtime_helpers.py | 47 ++++ agent/chat_completion_helpers.py | 39 ++++ cli-config.yaml.example | 13 ++ cli.py | 14 +- cron/scheduler.py | 23 +- docs/PER_MODEL_REASONING.md | 101 ++++++++ gateway/run.py | 26 ++- hermes_cli/config.py | 9 +- hermes_constants.py | 125 ++++++++++ tests/cron/test_reasoning_config_per_model.py | 106 +++++++++ .../test_reasoning_config_per_model.py | 111 +++++++++ .../test_fallback_reasoning_override.py | 144 ++++++++++++ .../test_switch_model_reasoning_override.py | 220 ++++++++++++++++++ tests/test_hermes_constants.py | 200 ++++++++++++++++ .../test_reasoning_config_per_model.py | 100 ++++++++ tui_gateway/server.py | 30 ++- website/docs/user-guide/configuration.md | 31 +++ 17 files changed, 1321 insertions(+), 18 deletions(-) create mode 100644 docs/PER_MODEL_REASONING.md create mode 100644 tests/cron/test_reasoning_config_per_model.py create mode 100644 tests/gateway/test_reasoning_config_per_model.py create mode 100644 tests/run_agent/test_fallback_reasoning_override.py create mode 100644 tests/run_agent/test_switch_model_reasoning_override.py create mode 100644 tests/tui_gateway/test_reasoning_config_per_model.py diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 1cde73419a44..28f0192d8161 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1305,6 +1305,13 @@ def restore_primary_runtime(agent) -> bool: primary_provider or "?", ) + # ── Restore reasoning_config if it was saved ── + # switch_model saves reasoning_config in _primary_runtime. If the + # snapshot predates that (older sessions), keep the current value. + saved_reasoning = rt.get("reasoning_config") + if saved_reasoning is not None: + agent.reasoning_config = dict(saved_reasoning) + # ── Reset fallback chain for the new turn ── agent._fallback_activated = False agent._fallback_index = 0 @@ -2065,6 +2072,45 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo api_mode=agent.api_mode, ) + # ── Re-resolve reasoning_config from per-model override ── + # The new model may have a different reasoning_effort override. Re-read + # config so the override takes effect immediately on /model switch. + # Try both agent.model (normalized, e.g. "claude-opus-4-5") AND the raw + # config default (user's original spelling, e.g. "claude-opus-4.5") so + # override keys match regardless of how downstream consumers normalized + # the input. See plan FINDING #7 + session follow-up. + try: + from hermes_constants import ( + parse_reasoning_effort, + resolve_per_model_reasoning_effort, + ) + from hermes_cli.config import load_config as _sm_load_config + + _reasoning_cfg = _sm_load_config() or {} + _sm_overrides = (_reasoning_cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + # Try the normalized agent.model first, then the raw config default + _sm_raw_model_default = str((_reasoning_cfg.get("model") or {}).get("default", "") or "").strip() + _sm_per_model = None + for _candidate in (agent.model, _sm_raw_model_default): + if _candidate: + _sm_per_model = resolve_per_model_reasoning_effort(_candidate, _sm_overrides) + if _sm_per_model is not None: + break + if _sm_per_model is not None: + agent.reasoning_config = _sm_per_model + logger.info( + "switch_model: reasoning_config resolved to per-model override for %s: %s", + agent.model, _sm_per_model, + ) + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT str()/strip() coerce. + _sm_global = (_reasoning_cfg.get("agent") or {}).get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_sm_global) + logger.info("switch_model: reasoning_config resolved to global effort: %s", _sm_global or "(none)") + except Exception as _reasoning_err: + logger.debug("switch_model: could not re-resolve reasoning_config: %s", _reasoning_err) + # ── Invalidate cached system prompt so it rebuilds next turn ── agent._cached_system_prompt = None @@ -2087,6 +2133,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "client_kwargs": dict(agent._client_kwargs), "use_prompt_caching": agent._use_prompt_caching, "use_native_cache_layout": agent._use_native_cache_layout, + "reasoning_config": dict(agent.reasoning_config) if getattr(agent, "reasoning_config", None) else None, "compressor_model": getattr(_cc, "model", agent.model) if _cc else agent.model, "compressor_base_url": getattr(_cc, "base_url", agent.base_url) if _cc else agent.base_url, "compressor_api_key": getattr(_cc, "api_key", "") if _cc else "", diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index b93dcf661717..57cc5e76cdb8 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1635,6 +1635,45 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool api_mode=agent.api_mode, ) + # Re-resolve reasoning_config for the new fallback model (Closes #21256). + # Per-model override (if any) takes precedence, else global reasoning_effort. + # Wrapped in try/except because config load failure must not kill the swap. + try: + from hermes_cli.config import load_config + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _fb_cfg = load_config() or {} + _fb_agent_cfg = _fb_cfg.get("agent", {}) or {} + _fb_overrides = _fb_agent_cfg.get("reasoning_overrides", {}) or {} + _fb_per_model = resolve_per_model_reasoning_effort(agent.model, _fb_overrides) + if _fb_per_model is not None: + agent.reasoning_config = _fb_per_model + logger.info( + "Fallback %s: reasoning_config resolved to per-model override: %s", + agent.model, _fb_per_model, + ) + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT coerce with ``or ""``. + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_fb_global_effort) + if agent.reasoning_config: + logger.info( + "Fallback %s: reasoning_config resolved to global effort: %s", + agent.model, _fb_global_effort, + ) + else: + logger.info( + "Fallback %s: reasoning_config resolved to None (disabled or default)", + agent.model, + ) + except Exception as _reasoning_err: + logger.debug( + "Failed to resolve reasoning_config for fallback %s; keeping current: %s", + agent.model, _reasoning_err, + ) + # Keep whatever reasoning_config was active — don't break the fallback swap. + # Keep the prompt's self-identity in sync with the model actually # answering, so "what model are you?" doesn't report the primary. rewrite_prompt_model_identity(agent, fb_model, fb_provider) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 1858f451c9a4..2583bfb54f0b 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -742,6 +742,19 @@ agent: # Options: "xhigh" (max), "high", "medium", "low", "minimal", "none" (disable) reasoning_effort: "medium" + # Per-model reasoning effort overrides (optional dict) + # Key: any sensible model spelling works (exact, dots↔dashes interchangeable, + # provider prefix optional). First match wins. + # Value: reasoning effort level (same options as reasoning_effort) + # Override the global reasoning_effort for that specific model. + # NOTE: no `hermes config set` support for this key -- edit YAML directly. + # reasoning_overrides: + # "openrouter/anthropic/claude-opus-4.5": "xhigh" + # "openai/gpt-5": "low" + # "claude-opus-4.6": "high" # bare model name also works + # "deepseek/deepseek-v4-pro": "xhigh" # dots and dashes are interchangeable + reasoning_overrides: {} + # Predefined personalities (use with /personality command) personalities: helpful: "You are a helpful, friendly AI assistant." diff --git a/cli.py b/cli.py index 17d2bdd0e4e3..c836834a2169 100644 --- a/cli.py +++ b/cli.py @@ -3917,8 +3917,18 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): ) # Reasoning config (OpenRouter reasoning effort level) - self.reasoning_config = _parse_reasoning_config( - CLI_CONFIG["agent"].get("reasoning_effort", "") + # Per-model override takes precedence over global effort (Closes #21256). + _reasoning_overrides = CLI_CONFIG["agent"].get("reasoning_overrides", {}) or {} + from hermes_constants import resolve_per_model_reasoning_effort + _per_model_reasoning = resolve_per_model_reasoning_effort( + self.model, _reasoning_overrides + ) + self.reasoning_config = ( + _per_model_reasoning + if _per_model_reasoning is not None + else _parse_reasoning_config( + CLI_CONFIG["agent"].get("reasoning_effort", "") + ) ) self.service_tier = _parse_service_tier_config( CLI_CONFIG["agent"].get("service_tier", "") diff --git a/cron/scheduler.py b/cron/scheduler.py index a50704ea10be..5e80fc2d7aec 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2988,12 +2988,25 @@ def run_job( except Exception: pass - # Reasoning config from config.yaml (raw value — a YAML boolean False - # means thinking disabled, see parse_reasoning_effort) - from hermes_constants import parse_reasoning_effort - reasoning_config = parse_reasoning_effort( - _cfg.get("agent", {}).get("reasoning_effort", "") + # Reasoning config from config.yaml (per-model override > global) + from hermes_constants import ( + parse_reasoning_effort, + resolve_per_model_reasoning_effort, ) + _cron_model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {} + _cron_model = str( + _cron_model_cfg.get("default", "") or _cron_model_cfg.get("model", "") or "" + ).strip() + _cron_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + _cron_per_model = resolve_per_model_reasoning_effort(_cron_model, _cron_overrides) + if _cron_per_model is not None: + reasoning_config = _cron_per_model + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT str()/strip() coerce. + reasoning_config = parse_reasoning_effort( + _cfg.get("agent", {}).get("reasoning_effort", "") + ) # Prefill messages from env or config.yaml. The top-level # prefill_messages_file key is canonical; agent.prefill_messages_file is diff --git a/docs/PER_MODEL_REASONING.md b/docs/PER_MODEL_REASONING.md new file mode 100644 index 000000000000..14a89466d6ee --- /dev/null +++ b/docs/PER_MODEL_REASONING.md @@ -0,0 +1,101 @@ +# Hermes Agent Configuration Guide + +## Per-Model Reasoning Effort Overrides + +You can configure different reasoning effort levels for different models. This allows you to set `high` effort for complex reasoning models like `claude-opus-4.5` while keeping `medium` for faster models like `gemini-flash`. + +### Configuration + +Edit your `config.yaml` (typically at `~/.hermes/config.yaml`): + +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high + gemini-flash: medium + gpt-4.5: high +``` + +### Key Matching + +The model name matching is **spelling-tolerant**. All of these variations will match: +- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` +- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` +- With or without provider prefixes + +Exact matches take precedence over variants. + +### Resolution Order + +When determining reasoning effort for a model, Hermes checks in this order: + +1. **Session override**: `/reasoning high` (current session only) +2. **Per-model override**: `agent.reasoning_overrides.` from config.yaml +3. **Global default**: `agent.reasoning_effort` from config.yaml + +### How It Works + +The override applies automatically in these scenarios: + +- **CLI startup**: Uses the override for the configured default model +- **Gateway messaging**: Each gateway session uses the override for its model +- **Desktop/TUI**: Uses the override for the configured model +- **Model switching**: When you switch models, the reasoning effort updates to the new model's override +- **Fallback activation**: When the primary model fails and Hermes falls back to a secondary model, it uses that fallback model's override +- **Reasoning recovery**: When the primary model recovers after a fallback, the original model's override is restored + +### Examples + +#### Example 1: High effort for Opus, medium for others +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high +``` + +#### Example 2: Different efforts per model +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high + gemini-2.0-flash: low + gpt-4.5: high + o3-mini: medium +``` + +#### Example 3: With provider prefixes +```yaml +agent: + reasoning_overrides: + anthropic/claude-opus-4.5: high + google/gemini-2.0-flash: low +``` + +All of these are equivalent — the provider prefix is optional. + +### Disabling Reasoning for Specific Models + +Set the override to `none` to disable reasoning for a specific model: + +```yaml +agent: + reasoning_overrides: + gemini-flash: none +``` + +### Troubleshooting + +**Override not taking effect?** +- Check the exact model name in your config with `/model` +- Verify the override is under `agent.reasoning_overrides` (not `agent.reasoning_effort`) +- Restart the gateway or CLI session after editing config.yaml +- Check logs for parsing errors + +**Override applies but reasoning doesn't work?** +- Not all models support reasoning (e.g., `gemini-flash` has limited support) +- Check the model's documentation for reasoning capability +- Use a model that explicitly supports extended thinking + +**Session override not respecting per-model override?** +- Session overrides take precedence (by design) +- Clear the session override with `/reasoning default` to return to the per-model override diff --git a/gateway/run.py b/gateway/run.py index 8d63da9c952a..7ede7d4130df 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4852,17 +4852,33 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _load_reasoning_config() -> dict | None: - """Load reasoning effort from config.yaml. + """Load reasoning effort from config.yaml, respecting per-model overrides. Reads agent.reasoning_effort from config.yaml. Valid: "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra". Returns None to use default (medium). + + Per-model overrides (agent.reasoning_overrides) take precedence + over the global value when the current model matches a key + (spelling-tolerant). Closes #21256. """ - from hermes_constants import parse_reasoning_effort + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort cfg = _load_gateway_runtime_config() - # Keep the raw value — coercing with ``or ""`` turns a YAML boolean - # False (``reasoning_effort: false``/``off``/``no``) into "", silently - # re-enabling thinking for users who explicitly disabled it. + # Per-model override first + model_cfg = cfg.get("model") or {} + model = str( + (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") + or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") + or "" + ).strip() + overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + per_model = resolve_per_model_reasoning_effort(model, overrides) + if per_model is not None: + return per_model + # Global fallback — keep the raw value; coercing with ``or ""`` turns + # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) + # into "", silently re-enabling thinking for users who explicitly + # disabled it. effort = cfg_get(cfg, "agent", "reasoning_effort", default="") result = parse_reasoning_effort(effort) if effort and str(effort).strip() and result is None: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 0754933d6082..737833fae208 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1158,8 +1158,15 @@ DEFAULT_CONFIG = { # only controls how inbound user images are presented. "image_input_mode": "auto", "disabled_toolsets": [], + + # Per-model reasoning effort overrides (spelling-tolerant). + # Dict mapping model names (any reasonable spelling) to effort levels. + # Takes precedence over agent.reasoning_effort when the current model + # matches a key in this dict. + # Edit directly in config.yaml (no CLI support due to dots in keys). + "reasoning_overrides": {}, }, - + "terminal": { "backend": "local", "modal_mode": "auto", diff --git a/hermes_constants.py b/hermes_constants.py index 26842530c22d..5d64a619ecaa 100644 --- a/hermes_constants.py +++ b/hermes_constants.py @@ -823,6 +823,131 @@ def parse_reasoning_effort(effort) -> dict | None: return None +def _canonical_model_variants(model: str) -> list[str]: + """Generate bounded spelling variants for tolerant override matching. + + Model names mix two types of separators: + - **Word separators**: dashes between words (``claude-opus``) + - **Version separators**: dots or dashes between version digits (``4.5``, ``4-5``) + + The tricky case is that ``.`` appears in BOTH roles (word sep in some + spellings, version sep in others), so a blanket ``.replace('.', '-')`` + is lossy — it collapses version dots into dashes and no later step + recovers the canonical form (``claude-opus-4.5``). + + Strategy: generate a small set of base forms, then apply version-dot + recovery to EACH of them. This ensures symmetry: + ``claude-opus-4.5``, ``claude-opus-4-5``, and ``claude-opus.4.5`` all + produce the same variant set. + + Steps: + 1. Exact input + 2. Dots/dashes cross-substitution on the entire string + 3. Version-dot recovery applied to ALL derivatives + 4. Strip provider/aggregator prefix → bare model variants + 5. Apply version-dot recovery to bare derivatives + 6. Prepend known provider/aggregator prefixes + + Duplicates removed in insertion order (exact always wins). + """ + import re + + # Version-dot regexes — digit-separator-digit interconversion + _dash_to_dot = lambda s: re.sub(r'(\d)-(\d)', r'\1.\2', s) + _dot_to_dash = lambda s: re.sub(r'(\d)\.(\d)', r'\1-\2', s) + + seen = set() + variants = [] + + def _add(v): + if v and v not in seen: + seen.add(v) + variants.append(v) + + def _add_with_derivatives(s): + """Add s plus its dots↔dashes and version-dot derivatives.""" + _add(s) + all_dashed = s.replace('.', '-') + _add(all_dashed) + all_dotted = s.replace('-', '.') + _add(all_dotted) + # Version-dot recovery on each base form + _add(_dash_to_dot(s)) + _add(_dot_to_dash(s)) + _add(_dash_to_dot(all_dashed)) + _add(_dot_to_dash(all_dotted)) + + # 1-3. Base variants for the full string + _add_with_derivatives(model) + + # Split by / to handle provider prefix + parts = model.split('/') + + # 4. Bare model variants (strip provider/aggregator prefix) + if len(parts) >= 2: + bare = parts[-1] + _add_with_derivatives(bare) + + # Strip aggregator only (3+ parts) + # e.g. "openrouter/anthropic/claude-opus-4.5" → "anthropic/claude-opus-4.5" + if len(parts) >= 3: + _add_with_derivatives('/'.join(parts[1:])) + + # 5. Prepend known provider prefixes to bare variants + known_providers = ( + 'anthropic', 'openai', 'google', 'openrouter', 'groq', 'mistral', + 'xai', 'cohere', 'perplexity', 'together', 'fireworks', 'deepseek', + ) + bare_variants = [v for v in variants if '/' not in v] + for v in bare_variants: + for provider in known_providers: + _add(f"{provider}/{v}") + + # Prepend aggregator to single-slash variants + single_slash_variants = [v for v in variants if v.count('/') == 1] + known_aggregators = ('openrouter', 'opencode', 'fireworks', 'groq', 'together') + for v in single_slash_variants: + for agg in known_aggregators: + _add(f"{agg}/{v}") + + return variants + + +def resolve_per_model_reasoning_effort(model: str, overrides: dict | None) -> dict | None: + """Lookup a per-model reasoning_effort override with spelling-tolerance. + + Args: + model: The model string (any spelling — exact, normalized, bare, + with provider prefix, etc.) + overrides: The dict of per-model overrides from + agent.reasoning_overrides in config.yaml. Keys can be + any sensible spelling of the model name. + + Returns: + The parsed reasoning_config dict if a match is found, + None otherwise (caller should fall back to global reasoning_effort). + + Resolution order: + 1. Exact match + 2. Dots ↔ dashes variants + 3. Strip provider prefix (bare model name only) + 4. Strip aggregator prefix (middle segment only) + 5. Prepend known aggregator prefixes to bare/single-slash variants + + First non-None parse_reasoning_effort result wins. + """ + if not overrides or not isinstance(overrides, dict) or not model: + return None + + for variant in _canonical_model_variants(model): + if variant in overrides: + result = parse_reasoning_effort(overrides[variant]) + if result is not None: + return result + + return None + + def is_termux() -> bool: """Return True when running inside a Termux (Android) environment. diff --git a/tests/cron/test_reasoning_config_per_model.py b/tests/cron/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..bdd09ade118b --- /dev/null +++ b/tests/cron/test_reasoning_config_per_model.py @@ -0,0 +1,106 @@ +"""Tests for per-model reasoning_effort override in cron scheduler.""" + +import pytest + + +class TestCronPerModelReasoningConfig: + """Test cron scheduler respects per-model reasoning overrides. + + Rather than spinning up a full CronScheduler (heavy), we verify the + resolution logic by testing the helper directly against a config dict + shaped the same way the scheduler reads it. + """ + + def test_per_model_override_resolves_for_cron_model(self): + """The spelling-tolerant helper resolves the cron config's model.""" + from hermes_constants import resolve_per_model_reasoning_effort + + # Simulate cron scheduler config shape + _cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + _model_cfg = _cfg.get("model", {}) + _model = str(_model_cfg.get("default", "") or "").strip() + _overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + + result = resolve_per_model_reasoning_effort(_model, _overrides) + assert result is not None + assert result["effort"] == "xhigh" + + def test_cron_falls_back_to_global_when_no_override(self): + """When no per-model override matches, global effort is used.""" + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + _model = _cfg["model"]["default"] + _overrides = _cfg["agent"]["reasoning_overrides"] + + per_model = resolve_per_model_reasoning_effort(_model, _overrides) + assert per_model is None # no match + + # Scheduler falls back to global + effort = _cfg["agent"]["reasoning_effort"] + result = parse_reasoning_effort(effort) + assert result is not None + assert result["effort"] == "low" + + def test_cron_handles_missing_model_key(self): + """Works when config has no model.default.""" + from hermes_constants import resolve_per_model_reasoning_effort + + _cfg = { + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": {"claude-opus-4.5": "high"}, + }, + } + _model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {} + _model = str(_model_cfg.get("default", "") or _model_cfg.get("model", "") or "").strip() + _overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + + # Empty model → resolve returns None → scheduler uses global + result = resolve_per_model_reasoning_effort(_model, _overrides) + assert result is None + + def test_global_fallback_with_yaml_false(self): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + "reasoning_overrides": {"claude-opus-4.5": "xhigh"}, + }, + } + _model = _cfg["model"]["default"] + _overrides = _cfg["agent"]["reasoning_overrides"] + + per_model = resolve_per_model_reasoning_effort(_model, _overrides) + assert per_model is None # no match + + # Scheduler global fallback — raw value, no coercion + result = parse_reasoning_effort( + _cfg.get("agent", {}).get("reasoning_effort", "") + ) + assert result is not None + assert result.get("enabled") is False diff --git a/tests/gateway/test_reasoning_config_per_model.py b/tests/gateway/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..f9799de33198 --- /dev/null +++ b/tests/gateway/test_reasoning_config_per_model.py @@ -0,0 +1,111 @@ +"""Tests for per-model reasoning_effort override in gateway _load_reasoning_config.""" + +import pytest + +import gateway.run as gateway_run + + +class TestGatewayPerModelReasoningConfig: + """Test GatewayRunner._load_reasoning_config respects per-model overrides.""" + + def test_per_model_override_takes_precedence(self, monkeypatch): + """Per-model override wins over global reasoning_effort.""" + from hermes_cli.config import DEFAULT_CONFIG + + fake_cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["enabled"] is True + assert result["effort"] == "xhigh" + + def test_global_fallback_when_no_override(self, monkeypatch): + """Global reasoning_effort applies when no per-model override matches.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "high", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_spelling_tolerant_match_in_gateway(self, monkeypatch): + """Override matches even with different spelling (dots vs dashes).""" + fake_cfg = { + "model": {"default": "claude-opus-4-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", # key has dots, model has dashes + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "xhigh" + + def test_no_overrides_dict(self, monkeypatch): + """Works fine when reasoning_overrides key is absent.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "low" + + def test_empty_overrides(self, monkeypatch): + """Empty overrides dict falls back to global.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": {}, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "medium" + + def test_global_fallback_with_yaml_false(self, monkeypatch): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result.get("enabled") is False diff --git a/tests/run_agent/test_fallback_reasoning_override.py b/tests/run_agent/test_fallback_reasoning_override.py new file mode 100644 index 000000000000..1c2c3e6228a7 --- /dev/null +++ b/tests/run_agent/test_fallback_reasoning_override.py @@ -0,0 +1,144 @@ +"""Tests for per-model reasoning_effort override during fallback activation. + +Tests that try_activate_fallback re-resolves reasoning_config when +swapping to a fallback model, so per-model overrides are honored even +during error recovery. +""" + +import pytest +from unittest.mock import MagicMock, patch + + +class TestFallbackReasoningOverride: + """Test try_activate_fallback re-resolves reasoning_config.""" + + def test_fallback_re_resolves_reasoning_config(self): + """When fallback activates, reasoning_config should be re-resolved. + + We test the resolution logic directly rather than spinning up a + full try_activate_fallback (which requires extensive agent setup). + The production code calls resolve_per_model_reasoning_effort with + the fallback model string — we verify that works correctly. + """ + from hermes_constants import resolve_per_model_reasoning_effort + + # Simulate: primary was gemini-flash (medium), fallback to claude-opus-4.5 (xhigh) + overrides = { + "claude-opus-4.5": "xhigh", + "gemini-flash": "medium", + } + + # Fallback model lookup + fb_result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert fb_result is not None + assert fb_result["effort"] == "xhigh" + + # Primary model lookup (for comparison) + primary_result = resolve_per_model_reasoning_effort("gemini-flash", overrides) + assert primary_result is not None + assert primary_result["effort"] == "medium" + + # The key point: fallback result differs from primary + assert fb_result["effort"] != primary_result["effort"] + + def test_fallback_to_model_without_override_uses_global(self): + """Fallback to a model with no override should resolve to None (→ global).""" + from hermes_constants import resolve_per_model_reasoning_effort + + overrides = {"claude-opus-4.5": "xhigh"} + + # Fallback to gpt-5 which has no override + result = resolve_per_model_reasoning_effort("gpt-5", overrides) + assert result is None # caller falls back to global + + def test_fallback_with_normalized_model_name(self): + """Fallback model name may be normalized (dots→dashes); override should still match.""" + from hermes_constants import resolve_per_model_reasoning_effort + + # User wrote key with dots, but normalize_model_for_provider converts to dashes + overrides = {"claude-sonnet-4.6": "high"} + + result = resolve_per_model_reasoning_effort("claude-sonnet-4-6", overrides) + assert result is not None + assert result["effort"] == "high" + + def test_fallback_recovery_restores_primary_reasoning(self): + """After fallback + restore_primary_runtime, reasoning_config returns to primary's value. + + This tests the integration of Task 6 (_primary_runtime snapshot) with + Task 6b (fallback re-resolution). The full cycle: + 1. Primary model = gemini-flash, reasoning = medium + 2. /model switch → _primary_runtime captures reasoning_config + 3. Fallback activates → reasoning re-resolved for fallback model + 4. restore_primary_runtime → reasoning_config restored from snapshot + """ + from agent.agent_runtime_helpers import restore_primary_runtime + + agent = MagicMock() + # Simulate: _primary_runtime was captured during /model switch + agent._primary_runtime = { + "model": "gemini-flash", + "provider": "google", + "base_url": "", + "api_mode": "openai", + "api_key": "key", + "client_kwargs": {}, + "use_prompt_caching": False, + "use_native_cache_layout": False, + "reasoning_config": {"enabled": True, "effort": "medium"}, + "compressor_model": "gemini-flash", + "compressor_base_url": "", + "compressor_api_key": "", + "compressor_provider": "", + "compressor_context_length": 0, + "compressor_api_mode": "", + "compressor_threshold_tokens": 0, + } + agent._fallback_activated = True + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._transport_cache = {} + agent._config_context_length = None + agent._rate_limited_until = 0 + # During fallback, reasoning was changed to xhigh (fallback model's override) + agent.model = "claude-opus-4.5" + agent.provider = "anthropic" + agent.reasoning_config = {"enabled": True, "effort": "xhigh"} + agent.context_compressor = MagicMock() + agent.base_url = "" + agent._anthropic_prompt_cache_policy = MagicMock(return_value=(False, False)) + agent._create_openai_client = MagicMock(return_value=MagicMock()) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + + result = restore_primary_runtime(agent) + assert result is True + # reasoning_config should be restored to primary's value (medium) + assert agent.reasoning_config == {"enabled": True, "effort": "medium"} + + def test_fallback_global_fallback_with_yaml_false(self): + """Fallback global fallback must not coerce YAML boolean False. + + Regression: ``or ""`` turned False into "", silently re-enabling + thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + + The production code in try_activate_fallback does: + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_fb_global_effort) + We verify that passing the raw False (not coerced "") produces + the disabled config. + """ + from hermes_constants import parse_reasoning_effort + + # Simulate: no per-model override matches, global is YAML False + _fb_agent_cfg = {"reasoning_effort": False} + + # This is the exact line from try_activate_fallback's else branch. + # The bug was: _fb_global_effort = _fb_agent_cfg.get(...) or "" + # which turned False into "". The fix passes the raw value. + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + result = parse_reasoning_effort(_fb_global_effort) + + assert result is not None + assert result.get("enabled") is False diff --git a/tests/run_agent/test_switch_model_reasoning_override.py b/tests/run_agent/test_switch_model_reasoning_override.py new file mode 100644 index 000000000000..ae304e118df3 --- /dev/null +++ b/tests/run_agent/test_switch_model_reasoning_override.py @@ -0,0 +1,220 @@ +"""Tests for per-model reasoning_effort override during /model switch. + +Tests that switch_model: +1. Re-resolves reasoning_config when switching to a model with an override +2. Falls back to global when switching to a model without an override +3. Saves reasoning_config into _primary_runtime for fallback recovery +""" + +import pytest +from unittest.mock import MagicMock, patch + + +class TestSwitchModelReasoningOverride: + """Test switch_model re-resolves reasoning_config on model switch.""" + + def _make_fake_agent(self, model="gpt-5", provider="openai"): + """Create a minimal fake agent for switch_model testing.""" + agent = MagicMock() + agent.model = model + agent.provider = provider + agent.base_url = "https://api.openai.com/v1" + agent.api_mode = "openai" + agent.api_key = "test-key" + agent._client_kwargs = {"api_key": "test-key", "base_url": "https://api.openai.com/v1"} + agent._use_prompt_caching = False + agent._use_native_cache_layout = False + agent.reasoning_config = {"enabled": True, "effort": "medium"} + agent._fallback_activated = False + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._config_context_length = None + agent._transport_cache = {} + agent.context_compressor = None + agent._cached_system_prompt = None + agent._anthropic_api_key = "" + agent._anthropic_base_url = None + agent._is_anthropic_oauth = False + agent._anthropic_prompt_cache_policy = MagicMock( + return_value=(False, False) + ) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + agent._create_openai_client = MagicMock(return_value=MagicMock()) + return agent + + def test_primary_runtime_includes_reasoning_config(self): + """After switch_model, _primary_runtime should contain reasoning_config key.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="claude-opus-4.5", + new_provider="anthropic", + base_url="https://api.anthropic.com", + api_mode="anthropic_messages", + ) + except Exception: + # Client creation may fail in test env; check _primary_runtime was set + pass + + assert hasattr(agent, "_primary_runtime") + assert "reasoning_config" in agent._primary_runtime + + def test_reasoning_config_resolves_to_override_on_switch(self): + """switch_model should resolve reasoning_config to per-model override.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="claude-opus-4.5", + new_provider="anthropic", + base_url="https://api.anthropic.com", + api_mode="anthropic_messages", + ) + except Exception: + pass + + # reasoning_config should be updated to xhigh + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("effort") == "xhigh" + + def test_reasoning_config_falls_back_to_global(self): + """switch_model should fall back to global when no override for new model.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", # override for different model + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="gpt-5", + new_provider="openai", + api_mode="openai", + ) + except Exception: + pass + + # No override for gpt-5 → should fall back to global "low" + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("effort") == "low" + + def test_restore_primary_runtime_restores_reasoning(self): + """restore_primary_runtime should restore reasoning_config from snapshot.""" + from agent.agent_runtime_helpers import restore_primary_runtime + + agent = MagicMock() + agent._primary_runtime = { + "model": "claude-opus-4.5", + "provider": "anthropic", + "base_url": "https://api.anthropic.com", + "api_mode": "anthropic_messages", + "api_key": "key", + "client_kwargs": {}, + "use_prompt_caching": True, + "use_native_cache_layout": False, + "reasoning_config": {"enabled": True, "effort": "xhigh"}, + "compressor_model": "claude-opus-4.5", + "compressor_base_url": "", + "compressor_api_key": "", + "compressor_provider": "", + "compressor_context_length": 0, + "compressor_api_mode": "", + "compressor_threshold_tokens": 0, + "anthropic_api_key": "key", + "anthropic_base_url": "https://api.anthropic.com", + "is_anthropic_oauth": False, + } + agent._fallback_activated = True + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._transport_cache = {} + agent._config_context_length = None + agent._rate_limited_until = 0 + agent.model = "fallback-model" + agent.provider = "openai" + agent.reasoning_config = {"enabled": True, "effort": "medium"} + agent.context_compressor = MagicMock() + agent.base_url = "" + # Mock the methods restore_primary_runtime calls + agent._anthropic_prompt_cache_policy = MagicMock(return_value=(True, False)) + agent._create_openai_client = MagicMock(return_value=MagicMock()) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + + result = restore_primary_runtime(agent) + assert result is True + assert agent.reasoning_config == {"enabled": True, "effort": "xhigh"} + + def test_switch_model_global_fallback_with_yaml_false(self): + """switch_model global fallback must not coerce YAML boolean False. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + "reasoning_overrides": {}, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="gpt-5", + new_provider="openai", + api_mode="openai", + ) + except Exception: + pass + + # No override for gpt-5 → global fallback with raw False + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("enabled") is False \ No newline at end of file diff --git a/tests/test_hermes_constants.py b/tests/test_hermes_constants.py index 44a449d91d08..5c70ebcb952b 100644 --- a/tests/test_hermes_constants.py +++ b/tests/test_hermes_constants.py @@ -490,6 +490,206 @@ class TestParseReasoningEffort: assert documented.issubset(set(VALID_REASONING_EFFORTS)) +class TestResolvePerModelReasoningEffort: + """Tests for resolve_per_model_reasoning_effort() — spelling-tolerant + per-model override lookup from agent.reasoning_overrides dict. + + Contract: the override key the user writes in config.yaml should match + regardless of how downstream consumers normalize the model string. + normalize_model_for_provider() converts dots to dashes and + adds/strips provider prefixes. Our resolver tolerates these + variations so the user's intent ("this model always gets xhigh") + is honored no matter which code path feeds the model string. + """ + + def test_exact_match(self): + """Exact model string match returns the parsed override.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_none_when_no_matching_key(self): + """Model not in overrides returns None (caller falls back to global).""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + assert resolve_per_model_reasoning_effort("gpt-5", overrides) is None + + def test_none_value_returns_disabled(self): + """Override set to 'none' returns {'enabled': False}.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "none"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": False} + + def test_invalid_value_returns_none(self): + """Override with invalid effort falls back to None (global).""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "banana"} + assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None + + def test_none_or_empty_overrides_returns_none(self): + """None or empty overrides dict returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + assert resolve_per_model_reasoning_effort("claude-opus-4.5", None) is None + assert resolve_per_model_reasoning_effort("claude-opus-4.5", {}) is None + + def test_empty_model_returns_none(self): + """Empty model string returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + assert resolve_per_model_reasoning_effort("", {"gpt-5": "low"}) is None + + # --- Spelling tolerance layer --- + + def test_dots_to_dashes_variant(self): + """User wrote key with dots; input comes in normalized with dashes. + + normalize_model_for_provider converts claude-opus-4.5 → claude-opus-4-5 + for the anthropic provider. The user's override key should still match. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_dashes_to_dots_variant(self): + """User wrote key with dashes; input comes in with dots.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4-5": "high"} + result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_strip_provider_prefix(self): + """User wrote key WITH provider prefix; input comes in bare. + + E.g. user config: model.default: claude-opus-4.5 (no prefix), + but override key: anthropic/claude-opus-4.5. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"anthropic/claude-opus-4.5": "high"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_prepend_provider_prefix(self): + """User wrote key bare; input comes in WITH provider prefix. + + E.g. user config: model.default: anthropic/claude-opus-4.5, + but override key: claude-opus-4.5 (no prefix). + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "high"} + result = resolve_per_model_reasoning_effort("anthropic/claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_aggregator_prefix_stripping(self): + """openrouter/anthropic/claude-opus-4.5 should match key anthropic/claude-opus-4.5. + + Aggregator providers (openrouter) prepend their own name, + creating a triple-prefix. The resolver strips the aggregator + layer to find the user's two-segment key. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"anthropic/claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("openrouter/anthropic/claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_exact_match_wins_over_variant(self): + """Ambiguity resolution: exact match takes priority over a variant. + + If both 'claude-opus-4.5' (exact) and 'claude-opus-4-5' (dashes + variant) are keys, the exact input matches the exact key first. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "high", "claude-opus-4-5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_none_when_no_variant_matches(self): + """All variants exhausted without a match returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"gpt-5": "low"} + assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None + + def test_all_dotted_input_matches_canonical_key(self): + """Regression: all-dotted input (claude-opus.4.5) must match + canonical key (claude-opus-4.5). + + This was a real bug found by delegate review: the old + all_dashed = model.replace('.', '-') collapsed version dots, + making the canonical form unreachable from all-dotted input. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides) + assert result is not None + assert result["effort"] == "xhigh" + + def test_different_models_do_not_match(self): + """No false positives: gemini-2.0-flash must not match gemini-flash.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"gemini-flash": "low"} + assert resolve_per_model_reasoning_effort("gemini-2.0-flash", overrides) is None + + +class TestReasoningOverridesDefaultConfig: + """Tests for the agent.reasoning_overrides default config key (Task 2).""" + + def test_default_config_has_reasoning_overrides_key(self): + """DEFAULT_CONFIG['agent'] contains 'reasoning_overrides' as an empty dict.""" + from hermes_cli.config import DEFAULT_CONFIG + assert "reasoning_overrides" in DEFAULT_CONFIG["agent"] + assert DEFAULT_CONFIG["agent"]["reasoning_overrides"] == {} + + def test_config_version_bumped(self): + """Config version was bumped to signal the schema change for reasoning_overrides.""" + from hermes_cli.config import DEFAULT_CONFIG + assert DEFAULT_CONFIG.get("_config_version") >= 31 + + def test_load_config_preserves_user_reasoning_overrides(self, tmp_path, monkeypatch): + """User-added reasoning_overrides are preserved through load_config().""" + import yaml + from hermes_cli.config import load_config, get_config_path + + user_config = { + "agent": { + "reasoning_overrides": { + "anthropic/claude-opus-4-5": "high", + "openrouter/anthropic/claude-sonnet-4-6": "low", + } + } + } + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(user_config)) + + # load_config() reads from get_config_path() — patch its global reference + monkeypatch.setitem( + load_config.__globals__, "get_config_path", lambda: config_path + ) + + loaded = load_config() + assert loaded["agent"]["reasoning_overrides"] == { + "anthropic/claude-opus-4-5": "high", + "openrouter/anthropic/claude-sonnet-4-6": "low", + } + + def test_spelling_tolerant_lookup_works_with_user_config(self): + """resolve_per_model_reasoning_effort works with user-added overrides.""" + from hermes_constants import resolve_per_model_reasoning_effort + # User config with one override, query uses different spelling + overrides = { + "anthropic/claude-opus-4.5": "xhigh", # user wrote with dots + } + # Lookup with different spelling (bare, dashes) — should still match + result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + # Another override, bare key + overrides2 = {"gpt-5": "low"} + # Lookup with provider prefix — should match + result2 = resolve_per_model_reasoning_effort("openai/gpt-5", overrides2) + assert result2 == {"enabled": True, "effort": "low"} + + class TestSecureParentDir: """Tests for secure_parent_dir() — prevents chmod on / or top-level dirs.""" diff --git a/tests/tui_gateway/test_reasoning_config_per_model.py b/tests/tui_gateway/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..d3468ff530a9 --- /dev/null +++ b/tests/tui_gateway/test_reasoning_config_per_model.py @@ -0,0 +1,100 @@ +"""Tests for per-model reasoning_effort override in TUI gateway _load_reasoning_config.""" + +import pytest + +import tui_gateway.server as tui_server + + +class TestTUIPerModelReasoningConfig: + """Test tui_gateway _load_reasoning_config respects per-model overrides.""" + + def test_per_model_override_takes_precedence(self, monkeypatch): + """Per-model override wins over global reasoning_effort.""" + fake_cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["enabled"] is True + assert result["effort"] == "xhigh" + + def test_global_fallback_when_no_override(self, monkeypatch): + """Global reasoning_effort applies when no per-model override matches.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "high", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_spelling_tolerant_match(self, monkeypatch): + """Override matches even with different spelling (provider prefix).""" + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "high", # key has prefix, model doesn't + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_parity_with_gateway_loader(self, monkeypatch): + """TUI and gateway loaders return identical results for same config.""" + import gateway.run as gateway_run + + fake_cfg = { + "model": {"default": "openrouter/anthropic/claude-sonnet-4.6"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "claude-sonnet-4.6": "high", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + tui_result = tui_server._load_reasoning_config() + gw_result = gateway_run.GatewayRunner._load_reasoning_config() + assert tui_result == gw_result + + def test_global_fallback_with_yaml_false(self, monkeypatch): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result.get("enabled") is False diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 83c79975e2ae..db32955ddd90 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2563,13 +2563,33 @@ def _display_mouse_tracking(display: dict) -> str: def _load_reasoning_config() -> dict | None: - from hermes_constants import parse_reasoning_effort + """Load reasoning effort from config.yaml, respecting per-model overrides. - # Pass the raw value through — ``or ""`` would coerce a YAML boolean - # False (``reasoning_effort: false``/``off``/``no``) to "", silently - # re-enabling thinking for users who explicitly turned it off. + Per-model overrides (agent.reasoning_overrides) take precedence + over the global value when the current model matches a key + (spelling-tolerant). Closes #21256. + """ + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + cfg = _load_cfg() + + # Per-model override first + model_cfg = cfg.get("model") or {} + model = str( + (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") + or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") + or "" + ).strip() + overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + per_model = resolve_per_model_reasoning_effort(model, overrides) + if per_model is not None: + return per_model + + # Global fallback — pass the raw value through; ``or ""`` would coerce + # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) to + # "", silently re-enabling thinking for users who explicitly turned it off. return parse_reasoning_effort( - (_load_cfg().get("agent") or {}).get("reasoning_effort", "") + (cfg.get("agent") or {}).get("reasoning_effort", "") ) diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 40748ce23197..67f3a0ed5c5b 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1301,6 +1301,37 @@ You can also change the reasoning effort at runtime with the `/reasoning` comman /reasoning hide # Hide model thinking ``` +#### Per-Model Reasoning Overrides + +You can set different reasoning effort levels for different models. This is useful when you want high reasoning for complex models but medium for faster ones: + +```yaml +agent: + reasoning_effort: "medium" # global default + reasoning_overrides: + "openrouter/anthropic/claude-opus-4.5": "xhigh" + "openai/gpt-5": "low" + "claude-sonnet-4.6": "high" # bare model name also works +``` + +The key matching is **spelling-tolerant** — any reasonable spelling will match: +- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` (dots and dashes are interchangeable) +- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` (provider prefix optional) +- Exact matches take precedence over variants + +:::note +There is no `hermes config set` support for `reasoning_overrides` keys — edit the YAML file directly. This is because model names often contain dots (e.g. `claude-opus-4.5`), which conflict with the CLI's dotted-key syntax. +::: + +**Resolution priority:** + +1. Session-scoped `/reasoning --session` override (gateway only) +2. Per-model override from `agent.reasoning_overrides` (spelling-tolerant) +3. Global `agent.reasoning_effort` +4. Provider default + +The override applies automatically everywhere: CLI startup, messaging gateway, Desktop/TUI, cron jobs, `/model` mid-session switches, and fallback model activation. + ## Tool-Use Enforcement Some models occasionally describe intended actions as text instead of making tool calls ("I would run the tests..." instead of actually calling the terminal). Tool-use enforcement injects system prompt guidance that steers the model back to actually calling tools. From e81d18dfb449ee28826a76cfaf91029a02b0ac74 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 07:16:54 -0700 Subject: [PATCH 093/149] refactor(reasoning): unify per-model reasoning resolution behind a single chokepoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the six per-surface copies of override-then-global resolution (CLI startup, gateway, TUI, cron, /model switch, fallback activation) onto one shared resolve_reasoning_config() in hermes_constants. Also fixes the gateway resolving reasoning against config model.default instead of the session's effective model: after a session-only /model switch, the switched model's override now applies (gateway message paths pass the resolved session model through _resolve_session_reasoning_config; /reasoning status reads the session model override). Cleanup: drop docs/PER_MODEL_REASONING.md (duplicates the website docs page), drop the change-detector _config_version test (no bump needed — deep-merge handles new keys), remove a stale plan-reference comment. Adds chokepoint contract tests (13) and gateway session-effective-model regression tests (2). --- agent/agent_runtime_helpers.py | 39 ++---- agent/chat_completion_helpers.py | 39 ++---- cli.py | 17 +-- cron/scheduler.py | 24 +--- docs/PER_MODEL_REASONING.md | 101 ---------------- gateway/run.py | 56 ++++----- gateway/slash_commands.py | 6 + hermes_constants.py | 61 ++++++++++ .../test_reasoning_config_per_model.py | 65 ++++++++++ tests/test_hermes_constants.py | 111 +++++++++++++++++- tui_gateway/server.py | 36 ++---- 11 files changed, 302 insertions(+), 253 deletions(-) delete mode 100644 docs/PER_MODEL_REASONING.md diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 28f0192d8161..c6ed459e93d9 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2074,40 +2074,19 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # ── Re-resolve reasoning_config from per-model override ── # The new model may have a different reasoning_effort override. Re-read - # config so the override takes effect immediately on /model switch. - # Try both agent.model (normalized, e.g. "claude-opus-4-5") AND the raw - # config default (user's original spelling, e.g. "claude-opus-4.5") so - # override keys match regardless of how downstream consumers normalized - # the input. See plan FINDING #7 + session follow-up. + # config so the override takes effect immediately on /model switch — + # resolved through the shared chokepoint (per-model > global; YAML + # boolean False = disabled). try: - from hermes_constants import ( - parse_reasoning_effort, - resolve_per_model_reasoning_effort, - ) + from hermes_constants import resolve_reasoning_config from hermes_cli.config import load_config as _sm_load_config _reasoning_cfg = _sm_load_config() or {} - _sm_overrides = (_reasoning_cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} - # Try the normalized agent.model first, then the raw config default - _sm_raw_model_default = str((_reasoning_cfg.get("model") or {}).get("default", "") or "").strip() - _sm_per_model = None - for _candidate in (agent.model, _sm_raw_model_default): - if _candidate: - _sm_per_model = resolve_per_model_reasoning_effort(_candidate, _sm_overrides) - if _sm_per_model is not None: - break - if _sm_per_model is not None: - agent.reasoning_config = _sm_per_model - logger.info( - "switch_model: reasoning_config resolved to per-model override for %s: %s", - agent.model, _sm_per_model, - ) - else: - # Raw value — a YAML boolean False means thinking disabled, - # see parse_reasoning_effort. Do NOT str()/strip() coerce. - _sm_global = (_reasoning_cfg.get("agent") or {}).get("reasoning_effort", "") - agent.reasoning_config = parse_reasoning_effort(_sm_global) - logger.info("switch_model: reasoning_config resolved to global effort: %s", _sm_global or "(none)") + agent.reasoning_config = resolve_reasoning_config(_reasoning_cfg, agent.model) + logger.info( + "switch_model: reasoning_config resolved for %s: %s", + agent.model, agent.reasoning_config, + ) except Exception as _reasoning_err: logger.debug("switch_model: could not re-resolve reasoning_config: %s", _reasoning_err) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 57cc5e76cdb8..9a4bc8a1fe6c 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1636,37 +1636,20 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool ) # Re-resolve reasoning_config for the new fallback model (Closes #21256). - # Per-model override (if any) takes precedence, else global reasoning_effort. - # Wrapped in try/except because config load failure must not kill the swap. + # Shared chokepoint: per-model override > global reasoning_effort + # (YAML boolean False = disabled). Wrapped in try/except because a + # config load failure must not kill the swap. try: from hermes_cli.config import load_config - from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + from hermes_constants import resolve_reasoning_config - _fb_cfg = load_config() or {} - _fb_agent_cfg = _fb_cfg.get("agent", {}) or {} - _fb_overrides = _fb_agent_cfg.get("reasoning_overrides", {}) or {} - _fb_per_model = resolve_per_model_reasoning_effort(agent.model, _fb_overrides) - if _fb_per_model is not None: - agent.reasoning_config = _fb_per_model - logger.info( - "Fallback %s: reasoning_config resolved to per-model override: %s", - agent.model, _fb_per_model, - ) - else: - # Raw value — a YAML boolean False means thinking disabled, - # see parse_reasoning_effort. Do NOT coerce with ``or ""``. - _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") - agent.reasoning_config = parse_reasoning_effort(_fb_global_effort) - if agent.reasoning_config: - logger.info( - "Fallback %s: reasoning_config resolved to global effort: %s", - agent.model, _fb_global_effort, - ) - else: - logger.info( - "Fallback %s: reasoning_config resolved to None (disabled or default)", - agent.model, - ) + agent.reasoning_config = resolve_reasoning_config( + load_config() or {}, agent.model + ) + logger.info( + "Fallback %s: reasoning_config resolved: %s", + agent.model, agent.reasoning_config, + ) except Exception as _reasoning_err: logger.debug( "Failed to resolve reasoning_config for fallback %s; keeping current: %s", diff --git a/cli.py b/cli.py index c836834a2169..25cce4f95d05 100644 --- a/cli.py +++ b/cli.py @@ -3917,19 +3917,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): ) # Reasoning config (OpenRouter reasoning effort level) - # Per-model override takes precedence over global effort (Closes #21256). - _reasoning_overrides = CLI_CONFIG["agent"].get("reasoning_overrides", {}) or {} - from hermes_constants import resolve_per_model_reasoning_effort - _per_model_reasoning = resolve_per_model_reasoning_effort( - self.model, _reasoning_overrides - ) - self.reasoning_config = ( - _per_model_reasoning - if _per_model_reasoning is not None - else _parse_reasoning_config( - CLI_CONFIG["agent"].get("reasoning_effort", "") - ) - ) + # Per-model override > global reasoning_effort — resolved through the + # shared chokepoint in hermes_constants (Closes #21256). + from hermes_constants import resolve_reasoning_config + self.reasoning_config = resolve_reasoning_config(CLI_CONFIG, self.model) self.service_tier = _parse_service_tier_config( CLI_CONFIG["agent"].get("service_tier", "") ) diff --git a/cron/scheduler.py b/cron/scheduler.py index 5e80fc2d7aec..176561c89f70 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2988,25 +2988,13 @@ def run_job( except Exception: pass - # Reasoning config from config.yaml (per-model override > global) - from hermes_constants import ( - parse_reasoning_effort, - resolve_per_model_reasoning_effort, + # Reasoning config from config.yaml (per-model override > global) — + # resolved through the shared chokepoint against the job's effective + # model (per-job override > HERMES_MODEL env > config.yaml default). + from hermes_constants import resolve_reasoning_config + reasoning_config = resolve_reasoning_config( + _cfg if isinstance(_cfg, dict) else {}, str(model) ) - _cron_model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {} - _cron_model = str( - _cron_model_cfg.get("default", "") or _cron_model_cfg.get("model", "") or "" - ).strip() - _cron_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} - _cron_per_model = resolve_per_model_reasoning_effort(_cron_model, _cron_overrides) - if _cron_per_model is not None: - reasoning_config = _cron_per_model - else: - # Raw value — a YAML boolean False means thinking disabled, - # see parse_reasoning_effort. Do NOT str()/strip() coerce. - reasoning_config = parse_reasoning_effort( - _cfg.get("agent", {}).get("reasoning_effort", "") - ) # Prefill messages from env or config.yaml. The top-level # prefill_messages_file key is canonical; agent.prefill_messages_file is diff --git a/docs/PER_MODEL_REASONING.md b/docs/PER_MODEL_REASONING.md deleted file mode 100644 index 14a89466d6ee..000000000000 --- a/docs/PER_MODEL_REASONING.md +++ /dev/null @@ -1,101 +0,0 @@ -# Hermes Agent Configuration Guide - -## Per-Model Reasoning Effort Overrides - -You can configure different reasoning effort levels for different models. This allows you to set `high` effort for complex reasoning models like `claude-opus-4.5` while keeping `medium` for faster models like `gemini-flash`. - -### Configuration - -Edit your `config.yaml` (typically at `~/.hermes/config.yaml`): - -```yaml -agent: - reasoning_overrides: - claude-opus-4.5: high - gemini-flash: medium - gpt-4.5: high -``` - -### Key Matching - -The model name matching is **spelling-tolerant**. All of these variations will match: -- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` -- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` -- With or without provider prefixes - -Exact matches take precedence over variants. - -### Resolution Order - -When determining reasoning effort for a model, Hermes checks in this order: - -1. **Session override**: `/reasoning high` (current session only) -2. **Per-model override**: `agent.reasoning_overrides.` from config.yaml -3. **Global default**: `agent.reasoning_effort` from config.yaml - -### How It Works - -The override applies automatically in these scenarios: - -- **CLI startup**: Uses the override for the configured default model -- **Gateway messaging**: Each gateway session uses the override for its model -- **Desktop/TUI**: Uses the override for the configured model -- **Model switching**: When you switch models, the reasoning effort updates to the new model's override -- **Fallback activation**: When the primary model fails and Hermes falls back to a secondary model, it uses that fallback model's override -- **Reasoning recovery**: When the primary model recovers after a fallback, the original model's override is restored - -### Examples - -#### Example 1: High effort for Opus, medium for others -```yaml -agent: - reasoning_overrides: - claude-opus-4.5: high -``` - -#### Example 2: Different efforts per model -```yaml -agent: - reasoning_overrides: - claude-opus-4.5: high - gemini-2.0-flash: low - gpt-4.5: high - o3-mini: medium -``` - -#### Example 3: With provider prefixes -```yaml -agent: - reasoning_overrides: - anthropic/claude-opus-4.5: high - google/gemini-2.0-flash: low -``` - -All of these are equivalent — the provider prefix is optional. - -### Disabling Reasoning for Specific Models - -Set the override to `none` to disable reasoning for a specific model: - -```yaml -agent: - reasoning_overrides: - gemini-flash: none -``` - -### Troubleshooting - -**Override not taking effect?** -- Check the exact model name in your config with `/model` -- Verify the override is under `agent.reasoning_overrides` (not `agent.reasoning_effort`) -- Restart the gateway or CLI session after editing config.yaml -- Check logs for parsing errors - -**Override applies but reasoning doesn't work?** -- Not all models support reasoning (e.g., `gemini-flash` has limited support) -- Check the model's documentation for reasoning capability -- Use a model that explicitly supports extended thinking - -**Session override not respecting per-model override?** -- Session overrides take precedence (by design) -- Clear the session override with `/reasoning default` to return to the per-model override diff --git a/gateway/run.py b/gateway/run.py index 7ede7d4130df..3bea0eb4034b 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4851,39 +4851,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return getattr(self, "_ephemeral_system_prompt", None) or "" @staticmethod - def _load_reasoning_config() -> dict | None: + def _load_reasoning_config(model: str = "") -> dict | None: """Load reasoning effort from config.yaml, respecting per-model overrides. - Reads agent.reasoning_effort from config.yaml. Valid: "none", - "minimal", "low", "medium", "high", "xhigh", "max", "ultra". Returns None to use - default (medium). + Thin wrapper over the shared chokepoint + :func:`hermes_constants.resolve_reasoning_config` (per-model override > + global ``agent.reasoning_effort``; YAML boolean False = disabled). + Closes #21256. - Per-model overrides (agent.reasoning_overrides) take precedence - over the global value when the current model matches a key - (spelling-tolerant). Closes #21256. + Args: + model: The effective model for the calling session. When empty, + the config's ``model.default`` is used. """ - from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + from hermes_constants import resolve_reasoning_config cfg = _load_gateway_runtime_config() - # Per-model override first - model_cfg = cfg.get("model") or {} - model = str( - (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") - or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") - or "" - ).strip() - overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} - per_model = resolve_per_model_reasoning_effort(model, overrides) - if per_model is not None: - return per_model - # Global fallback — keep the raw value; coercing with ``or ""`` turns - # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) - # into "", silently re-enabling thinking for users who explicitly - # disabled it. - effort = cfg_get(cfg, "agent", "reasoning_effort", default="") - result = parse_reasoning_effort(effort) - if effort and str(effort).strip() and result is None: - logger.warning("Unknown reasoning_effort '%s', using default (medium)", effort) - return result + return resolve_reasoning_config(cfg, model) @staticmethod def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]: @@ -4916,8 +4898,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, + model: str = "", ) -> dict | None: - """Resolve reasoning effort for a session, honoring session overrides.""" + """Resolve reasoning effort for a session, honoring session overrides. + + Priority: session-scoped ``/reasoning --session`` override > + per-model override (``agent.reasoning_overrides``) > global + ``agent.reasoning_effort``. ``model`` should be the session's + *effective* model (session ``/model`` override included) so + per-model overrides track what the session actually runs — when + empty, the config's ``model.default`` is used. + """ resolved_session_key = session_key if not resolved_session_key and source is not None: try: @@ -4928,7 +4919,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew overrides = getattr(self, "_session_reasoning_overrides", {}) or {} if resolved_session_key and resolved_session_key in overrides: return overrides[resolved_session_key] - return self._load_reasoning_config() + return self._load_reasoning_config(model) def _set_session_reasoning_override( self, @@ -13498,7 +13489,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew pr = self._provider_routing max_iterations = _current_max_iterations() - reasoning_config = self._resolve_session_reasoning_config(source=source) + reasoning_config = self._resolve_session_reasoning_config( + source=source, model=model + ) self._reasoning_config = reasoning_config self._service_tier = self._load_service_tier() turn_route = self._resolve_turn_agent_config(prompt, model, runtime_kwargs) @@ -18249,6 +18242,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew reasoning_config = self._resolve_session_reasoning_config( source=source, session_key=session_key, + model=model, ) self._reasoning_config = reasoning_config self._service_tier = self._load_service_tier() diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 765369f1c5e6..731ec0d1781c 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -2658,9 +2658,15 @@ class GatewaySlashCommandsMixin: _reasoning_source = await asyncio.to_thread(self._normalize_source_for_session_key, event.source) session_key = self._session_key_for_source(_reasoning_source) self._show_reasoning = self._load_show_reasoning() + # Use the session's effective model (session /model override wins over + # config default) so per-model reasoning_overrides display correctly. + _session_model = str( + ((getattr(self, "_session_model_overrides", {}) or {}).get(session_key) or {}).get("model") or "" + ) self._reasoning_config = self._resolve_session_reasoning_config( source=event.source, session_key=session_key, + model=_session_model, ) def _save_config_key(key_path: str, value): diff --git a/hermes_constants.py b/hermes_constants.py index 5d64a619ecaa..6e344844812d 100644 --- a/hermes_constants.py +++ b/hermes_constants.py @@ -948,6 +948,67 @@ def resolve_per_model_reasoning_effort(model: str, overrides: dict | None) -> di return None +def resolve_reasoning_config(cfg: dict | None, model: str = "") -> dict | None: + """Resolve the effective reasoning config for *model* from a config dict. + + Single chokepoint for reasoning-effort resolution, shared by every + surface (CLI startup, messaging gateway, Desktop/TUI, cron, ``/model`` + switch, fallback activation). Priority: + + 1. Per-model override from ``agent.reasoning_overrides`` + (spelling-tolerant — see :func:`resolve_per_model_reasoning_effort`) + 2. Global ``agent.reasoning_effort`` — the raw value is passed through + so a YAML boolean ``False`` (``reasoning_effort: false``/``off``/ + ``no``) means "thinking disabled", never silently re-enabled. + + Session-scoped overrides (gateway ``/reasoning --session``) are resolved + by the caller BEFORE this function — they always win. + + Args: + cfg: A loaded config dict (any of the three loaders' shapes — only + the ``agent`` and ``model`` sections are read). + model: The effective model for this surface/session. When empty, + it is derived from the config's ``model`` section (string + form, or a dict's ``default``/``model`` keys). + + Returns: + The parsed reasoning config dict, or None when unset/unrecognized + (caller uses the provider default). + """ + cfg = cfg if isinstance(cfg, dict) else {} + agent_cfg = cfg.get("agent") + if not isinstance(agent_cfg, dict): + agent_cfg = {} + + if not model: + model_cfg = cfg.get("model") + if isinstance(model_cfg, str): + model = model_cfg.strip() + elif isinstance(model_cfg, dict): + model = str( + model_cfg.get("default") or model_cfg.get("model") or "" + ).strip() + else: + model = "" + + overrides = agent_cfg.get("reasoning_overrides") or {} + per_model = resolve_per_model_reasoning_effort(model, overrides) + if per_model is not None: + return per_model + + # Global fallback — keep the raw value; coercing with ``or ""`` turns a + # YAML boolean False into "", silently re-enabling thinking for users + # who explicitly disabled it. + effort = agent_cfg.get("reasoning_effort", "") + result = parse_reasoning_effort(effort) + if effort and str(effort).strip() and result is None: + import logging + logging.getLogger(__name__).warning( + "Unknown reasoning_effort '%s', using default (medium)", effort + ) + return result + + def is_termux() -> bool: """Return True when running inside a Termux (Android) environment. diff --git a/tests/gateway/test_reasoning_config_per_model.py b/tests/gateway/test_reasoning_config_per_model.py index f9799de33198..ffd6ba0b9444 100644 --- a/tests/gateway/test_reasoning_config_per_model.py +++ b/tests/gateway/test_reasoning_config_per_model.py @@ -109,3 +109,68 @@ class TestGatewayPerModelReasoningConfig: result = gateway_run.GatewayRunner._load_reasoning_config() assert result is not None assert result.get("enabled") is False + + +class TestGatewaySessionEffectiveModel: + """The reasoning override must track the SESSION's effective model. + + Regression guard: _load_reasoning_config used to always read + model.default from config.yaml, so a session-only /model switch to a + different model kept resolving the config default's override. + """ + + def test_explicit_model_beats_config_default(self, monkeypatch): + """_load_reasoning_config(model=...) resolves for that model, not model.default.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "gpt-5": "low", + "claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + # Session switched (session-only) to claude-opus-4.5 — its override + # must win over the config default model's override. + result = gateway_run.GatewayRunner._load_reasoning_config("claude-opus-4.5") + assert result is not None + assert result["effort"] == "xhigh" + + # And without a model arg, the config default's override applies. + result_default = gateway_run.GatewayRunner._load_reasoning_config() + assert result_default is not None + assert result_default["effort"] == "low" + + def test_resolve_session_reasoning_forwards_model(self, monkeypatch): + """_resolve_session_reasoning_config passes the effective model through + (and session-scoped /reasoning overrides still win over it).""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": {"claude-opus-4.5": "xhigh"}, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + runner = object.__new__(gateway_run.GatewayRunner) + runner._session_reasoning_overrides = {} + + # No session override → per-model override for the effective model. + result = runner._resolve_session_reasoning_config( + session_key="agent:main:telegram:private:1", model="claude-opus-4.5" + ) + assert result is not None + assert result["effort"] == "xhigh" + + # Session-scoped /reasoning override still wins over per-model. + runner._session_reasoning_overrides = { + "agent:main:telegram:private:1": {"enabled": True, "effort": "minimal"} + } + result = runner._resolve_session_reasoning_config( + session_key="agent:main:telegram:private:1", model="claude-opus-4.5" + ) + assert result == {"enabled": True, "effort": "minimal"} diff --git a/tests/test_hermes_constants.py b/tests/test_hermes_constants.py index 5c70ebcb952b..8eddceb70eaf 100644 --- a/tests/test_hermes_constants.py +++ b/tests/test_hermes_constants.py @@ -631,6 +631,112 @@ class TestResolvePerModelReasoningEffort: assert resolve_per_model_reasoning_effort("gemini-2.0-flash", overrides) is None +class TestResolveReasoningConfig: + """Tests for resolve_reasoning_config() — the single shared chokepoint + every surface (CLI, gateway, TUI, cron, /model switch, fallback) calls. + + Contract: per-model override > global agent.reasoning_effort; the raw + global value passes through uncoerced (YAML False = disabled); an + explicit model argument wins over the config's model.default. + """ + + def _cfg(self, effort: object = "medium", overrides=None, default_model="gpt-5"): + return { + "model": {"default": default_model}, + "agent": { + "reasoning_effort": effort, + "reasoning_overrides": overrides or {}, + }, + } + + def test_per_model_override_wins(self): + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(overrides={"claude-opus-4.5": "xhigh"}) + result = resolve_reasoning_config(cfg, "claude-opus-4.5") + assert result == {"enabled": True, "effort": "xhigh"} + + def test_global_fallback_when_no_override(self): + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort="low", overrides={"claude-opus-4.5": "xhigh"}) + assert resolve_reasoning_config(cfg, "gpt-5") == {"enabled": True, "effort": "low"} + + def test_explicit_model_wins_over_config_default(self): + """The session's effective model (e.g. after a session-only /model + switch) must be used for override lookup — NOT model.default.""" + from hermes_constants import resolve_reasoning_config + cfg = self._cfg( + effort="medium", + overrides={"gpt-5": "low", "claude-opus-4.5": "xhigh"}, + default_model="gpt-5", + ) + # Session switched to opus; its override must win over gpt-5's. + result = resolve_reasoning_config(cfg, "claude-opus-4.5") + assert result == {"enabled": True, "effort": "xhigh"} + + def test_empty_model_derives_from_config_default(self): + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(overrides={"gpt-5": "high"}, default_model="gpt-5") + assert resolve_reasoning_config(cfg) == {"enabled": True, "effort": "high"} + + def test_empty_model_derives_from_model_alias_key(self): + """model: {model: ...} alias shape (older configs) also resolves.""" + from hermes_constants import resolve_reasoning_config + cfg = { + "model": {"model": "gpt-5"}, + "agent": {"reasoning_effort": "medium", "reasoning_overrides": {"gpt-5": "high"}}, + } + assert resolve_reasoning_config(cfg) == {"enabled": True, "effort": "high"} + + def test_string_model_section(self): + """Top-level ``model: `` config shape (cron raw-YAML path).""" + from hermes_constants import resolve_reasoning_config + cfg = { + "model": "claude-opus-4.5", + "agent": {"reasoning_effort": "low", "reasoning_overrides": {"claude-opus-4.5": "xhigh"}}, + } + assert resolve_reasoning_config(cfg) == {"enabled": True, "effort": "xhigh"} + + def test_yaml_false_global_uncoerced(self): + """YAML boolean False must mean disabled — never coerced to ''.""" + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort=False) + assert resolve_reasoning_config(cfg, "gpt-5") == {"enabled": False} + + def test_yaml_false_not_shadowed_by_other_models_override(self): + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort=False, overrides={"claude-opus-4.5": "xhigh"}) + assert resolve_reasoning_config(cfg, "gpt-5") == {"enabled": False} + + def test_override_none_disables_for_model(self): + """Per-model override value 'none' disables thinking for that model.""" + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort="high", overrides={"gemini-flash": "none"}) + assert resolve_reasoning_config(cfg, "gemini-flash") == {"enabled": False} + + def test_unknown_global_returns_none(self): + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort="bogus-level") + assert resolve_reasoning_config(cfg, "gpt-5") is None + + def test_empty_config_returns_none(self): + from hermes_constants import resolve_reasoning_config + assert resolve_reasoning_config({}) is None + assert resolve_reasoning_config(None) is None + + def test_malformed_sections_tolerated(self): + """Non-dict agent/model sections must not raise.""" + from hermes_constants import resolve_reasoning_config + assert resolve_reasoning_config({"agent": "oops", "model": 42}) is None + assert resolve_reasoning_config({"agent": None, "model": None}) is None + assert resolve_reasoning_config({"agent": {"reasoning_overrides": "bad"}}) is None + + def test_invalid_override_value_falls_back_to_global(self): + """A junk override value for the matching model falls through to global.""" + from hermes_constants import resolve_reasoning_config + cfg = self._cfg(effort="medium", overrides={"gpt-5": "turbo-max"}) + assert resolve_reasoning_config(cfg, "gpt-5") == {"enabled": True, "effort": "medium"} + + class TestReasoningOverridesDefaultConfig: """Tests for the agent.reasoning_overrides default config key (Task 2).""" @@ -640,11 +746,6 @@ class TestReasoningOverridesDefaultConfig: assert "reasoning_overrides" in DEFAULT_CONFIG["agent"] assert DEFAULT_CONFIG["agent"]["reasoning_overrides"] == {} - def test_config_version_bumped(self): - """Config version was bumped to signal the schema change for reasoning_overrides.""" - from hermes_cli.config import DEFAULT_CONFIG - assert DEFAULT_CONFIG.get("_config_version") >= 31 - def test_load_config_preserves_user_reasoning_overrides(self, tmp_path, monkeypatch): """User-added reasoning_overrides are preserved through load_config().""" import yaml diff --git a/tui_gateway/server.py b/tui_gateway/server.py index db32955ddd90..af5cead103c1 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2562,35 +2562,17 @@ def _display_mouse_tracking(display: dict) -> str: return "all" -def _load_reasoning_config() -> dict | None: +def _load_reasoning_config(model: str = "") -> dict | None: """Load reasoning effort from config.yaml, respecting per-model overrides. - Per-model overrides (agent.reasoning_overrides) take precedence - over the global value when the current model matches a key - (spelling-tolerant). Closes #21256. + Thin wrapper over the shared chokepoint + :func:`hermes_constants.resolve_reasoning_config` (per-model override > + global ``agent.reasoning_effort``; YAML boolean False = disabled). + Closes #21256. """ - from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + from hermes_constants import resolve_reasoning_config - cfg = _load_cfg() - - # Per-model override first - model_cfg = cfg.get("model") or {} - model = str( - (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") - or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") - or "" - ).strip() - overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} - per_model = resolve_per_model_reasoning_effort(model, overrides) - if per_model is not None: - return per_model - - # Global fallback — pass the raw value through; ``or ""`` would coerce - # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) to - # "", silently re-enabling thinking for users who explicitly turned it off. - return parse_reasoning_effort( - (cfg.get("agent") or {}).get("reasoning_effort", "") - ) + return resolve_reasoning_config(_load_cfg(), model) def _load_service_tier() -> str | None: @@ -4231,7 +4213,7 @@ def _background_agent_kwargs(agent, task_id: str) -> dict: "openrouter_min_coding_score": getattr(agent, "openrouter_min_coding_score", None), "session_id": task_id, "reasoning_config": getattr(agent, "reasoning_config", None) - or _load_reasoning_config(), + or _load_reasoning_config(str(getattr(agent, "model", "") or "")), "service_tier": getattr(agent, "service_tier", None) or _load_service_tier(), "request_overrides": dict(getattr(agent, "request_overrides", {}) or {}), "platform": "tui", @@ -4660,7 +4642,7 @@ def _make_agent( reasoning_config=( reasoning_config_override if reasoning_config_override is not None - else _load_reasoning_config() + else _load_reasoning_config(str(model or "")) ), service_tier=( service_tier_override From 7bb409b2d0cde1b83f891ef8bd3e82dacbe4be28 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 07:37:54 -0700 Subject: [PATCH 094/149] test: update stale _load_reasoning_config mocks for new model parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two test mocks stubbed the old zero-arg signature; the chokepoint refactor added an optional model param that call sites now pass. Swept the full test tree for other stale stubs of the changed functions — the rest use MagicMock/patch(return_value=...), which tolerate the new arg. --- tests/gateway/test_api_server.py | 2 +- tests/test_tui_gateway_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 3892652c3be7..abe8cdd64d34 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -3887,7 +3887,7 @@ def _patch_create_agent_runtime(monkeypatch, captured: dict, fake_agent_cls): monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda: "global/model") monkeypatch.setattr("gateway.run._load_gateway_config", lambda: {}) monkeypatch.setattr( - "gateway.run.GatewayRunner._load_reasoning_config", staticmethod(lambda: {}) + "gateway.run.GatewayRunner._load_reasoning_config", staticmethod(lambda model="": {}) ) monkeypatch.setattr( "gateway.run.GatewayRunner._load_fallback_model", staticmethod(lambda: None) diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index b8b0bbd77322..f117af57a63e 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -7596,7 +7596,7 @@ def _setup_make_agent_mocks(monkeypatch, cfg): }, ) monkeypatch.setattr(server, "_load_tool_progress_mode", lambda: "off") - monkeypatch.setattr(server, "_load_reasoning_config", lambda: None) + monkeypatch.setattr(server, "_load_reasoning_config", lambda model="": None) monkeypatch.setattr(server, "_load_service_tier", lambda: None) monkeypatch.setattr(server, "_load_enabled_toolsets", lambda: None) monkeypatch.setattr(server, "_get_db", lambda: None) From 271a9d8ec6ada347375921a7995001b35ad89954 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:53:05 -0700 Subject: [PATCH 095/149] perf(agent): segment mixed tool batches to recover lost concurrency (#64460) A model response containing several parallel-safe reads plus one unsafe tool used to lose ALL concurrency: _should_parallelize_tool_batch was all-or-nothing, so a single barrier call (terminal, clarify, unknown tool, malformed args) forced the entire batch onto the sequential path. _plan_tool_batch_segments now splits the batch into ordered segments: maximal contiguous runs of parallel-safe calls execute on the existing concurrent path, barrier calls on the sequential path, strictly in the model's emission order. Invariants preserved: - one tool result per call, appended in emission order (segments are contiguous, so no result reordering across a barrier) - side-effect boundaries: no call starts before an earlier barrier ends - overlapping file targets split into separate ordered parallel runs - turn-end budget enforcement + /steer injection run exactly once per batch (segment executors run with finalize=False; the segmented dispatcher owns the whole-turn finalize) - interrupt during segment k drains segments k+1..n with cancelled results, keeping one result per tool_call_id Homogeneous batches keep their original single-path dispatch (zero behavior delta); _should_parallelize_tool_batch remains as a thin view over the planner for existing callers and tests. --- agent/tool_dispatch_helpers.py | 105 ++++- agent/tool_executor.py | 78 +++- run_agent.py | 33 +- .../run_agent/test_tool_batch_segmentation.py | 400 ++++++++++++++++++ 4 files changed, 584 insertions(+), 32 deletions(-) create mode 100644 tests/run_agent/test_tool_batch_segmentation.py diff --git a/agent/tool_dispatch_helpers.py b/agent/tool_dispatch_helpers.py index 1b6cae98ac66..4360f60f7eb9 100644 --- a/agent/tool_dispatch_helpers.py +++ b/agent/tool_dispatch_helpers.py @@ -102,50 +102,118 @@ def _is_mcp_tool_parallel_safe(tool_name: str) -> bool: return False -def _should_parallelize_tool_batch(tool_calls) -> bool: - """Return True when a tool-call batch is safe to run concurrently.""" - if len(tool_calls) <= 1: - return False +def _plan_tool_batch_segments(tool_calls) -> List[tuple]: + """Split a tool-call batch into ordered ``(kind, calls)`` segments. - tool_names = [tc.function.name for tc in tool_calls] - if any(name in _NEVER_PARALLEL_TOOLS for name in tool_names): - return False + ``kind`` is ``"parallel"`` (a maximal contiguous run of parallel-safe + calls) or ``"sequential"`` (one or more barrier calls that must run + in-order on the sequential path). Segments preserve the model's + original call order exactly — a later call never crosses an earlier + barrier — so tool-result ordering and side-effect boundaries are + identical to fully-sequential execution. The per-call safety rules + are the same ones the old all-or-nothing gate applied to the whole + batch: + * ``_NEVER_PARALLEL_TOOLS`` (interactive tools) → barrier. + * Unparseable / non-dict arguments → barrier. + * Path-scoped tools (``read_file``/``write_file``/``patch``) join a + parallel run only when their target path does not overlap another + path already reserved in the same run; an overlap closes the run so + the conflicting call starts a NEW run after the first completes. + * Anything not in ``_PARALLEL_SAFE_TOOLS`` and not an opted-in MCP + tool → barrier. + + Parallel runs shorter than two calls are demoted to sequential (no + concurrency win, and the sequential executor owns the richer inline + dispatch), and adjacent sequential segments are merged. + """ + segments: list[list] = [] # [kind, calls] pairs, normalized to tuples on return + current: list = [] reserved_paths: list[Path] = [] + + def _close_parallel() -> None: + nonlocal current, reserved_paths + if current: + segments.append(["parallel", current]) + current = [] + reserved_paths = [] + + def _add_sequential(tc) -> None: + _close_parallel() + if segments and segments[-1][0] == "sequential": + segments[-1][1].append(tc) + else: + segments.append(["sequential", [tc]]) + for tool_call in tool_calls: tool_name = tool_call.function.name + + if tool_name in _NEVER_PARALLEL_TOOLS: + _add_sequential(tool_call) + continue + try: function_args = json.loads(tool_call.function.arguments) except Exception: logging.debug( - "Could not parse args for %s — defaulting to sequential; raw=%s", + "Could not parse args for %s — treating as sequential barrier; raw=%s", tool_name, tool_call.function.arguments[:200], ) - return False + _add_sequential(tool_call) + continue if not isinstance(function_args, dict): logging.debug( - "Non-dict args for %s (%s) — defaulting to sequential", + "Non-dict args for %s (%s) — treating as sequential barrier", tool_name, type(function_args).__name__, ) - return False + _add_sequential(tool_call) + continue if tool_name in _PATH_SCOPED_TOOLS: scoped_path = _extract_parallel_scope_path(tool_name, function_args) if scoped_path is None: - return False + _add_sequential(tool_call) + continue if any(_paths_overlap(scoped_path, existing) for existing in reserved_paths): - return False + # Same-subtree conflict inside this run: close it so this + # call starts a fresh run AFTER the conflicting one lands. + _close_parallel() reserved_paths.append(scoped_path) + current.append(tool_call) continue - if tool_name not in _PARALLEL_SAFE_TOOLS: - # Check if it's an MCP tool from a server that opted into parallel calls. - if not _is_mcp_tool_parallel_safe(tool_name): - return False + if tool_name in _PARALLEL_SAFE_TOOLS or _is_mcp_tool_parallel_safe(tool_name): + current.append(tool_call) + continue - return True + _add_sequential(tool_call) + + _close_parallel() + + normalized: list[list] = [] + for kind, calls in segments: + if kind == "parallel" and len(calls) < 2: + kind = "sequential" + if normalized and normalized[-1][0] == "sequential" and kind == "sequential": + normalized[-1][1].extend(calls) + else: + normalized.append([kind, calls]) + return [(kind, calls) for kind, calls in normalized] + + +def _should_parallelize_tool_batch(tool_calls) -> bool: + """Return True when the WHOLE tool-call batch is safe to run concurrently. + + Thin view over ``_plan_tool_batch_segments`` kept for callers/tests that + only care about the homogeneous case: True iff the planner produces a + single all-parallel segment. + """ + if len(tool_calls) <= 1: + return False + segments = _plan_tool_batch_segments(tool_calls) + return len(segments) == 1 and segments[0][0] == "parallel" def _extract_parallel_scope_path(tool_name: str, function_args: dict) -> Optional[Path]: @@ -542,6 +610,7 @@ __all__ = [ "_DESTRUCTIVE_PATTERNS", "_REDIRECT_OVERWRITE", "_is_destructive_command", + "_plan_tool_batch_segments", "_should_parallelize_tool_batch", "_extract_parallel_scope_path", "_paths_overlap", diff --git a/agent/tool_executor.py b/agent/tool_executor.py index ac505c6d8299..a5871442b449 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -36,6 +36,7 @@ from agent.tool_dispatch_helpers import ( _is_multimodal_tool_result, _multimodal_text_summary, _append_subdir_hint_to_multimodal, + _plan_tool_batch_segments, make_tool_result_message, ) from tools.terminal_tool import ( @@ -322,11 +323,15 @@ def _run_agent_tool_execution_middleware( return result, observed_args -def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: +def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: """Execute multiple tool calls concurrently using a thread pool. Results are collected in the original tool-call order and appended to messages so the API sees them in the expected sequence. + + ``finalize=False`` skips the end-of-batch aggregate budget enforcement + and /steer injection — used when this call is one segment of a larger + mixed batch and the segmented dispatcher owns the turn-end work. """ tool_calls = assistant_message.tool_calls num_tools = len(tool_calls) @@ -1006,7 +1011,7 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe # ── Per-turn aggregate budget enforcement ───────────────────────── num_tools = len(parsed_calls) - if num_tools > 0: + if finalize and num_tools > 0: turn_tool_msgs = messages[-num_tools:] enforce_turn_budget(turn_tool_msgs, env=get_active_env(effective_task_id), config=_tool_budget) @@ -1014,13 +1019,18 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe # Append any pending user steer text to the last tool result so the # agent sees it on its next iteration. Runs AFTER budget enforcement # so the steer marker is never truncated. See steer() for details. - if num_tools > 0: + if finalize and num_tools > 0: agent._apply_pending_steer_to_tool_results(messages, num_tools) -def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: - """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools.""" +def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: + """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools. + + ``finalize=False`` skips the end-of-batch aggregate budget enforcement + and /steer injection — used when this call is one segment of a larger + mixed batch and the segmented dispatcher owns the turn-end work. + """ # Resolve the context-scaled tool-output budget once per turn. _tool_budget = _budget_for_agent(agent) for i, tool_call in enumerate(assistant_message.tool_calls, 1): @@ -1716,19 +1726,73 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe # ── Per-turn aggregate budget enforcement ───────────────────────── num_tools_seq = len(assistant_message.tool_calls) - if num_tools_seq > 0: + if finalize and num_tools_seq > 0: enforce_turn_budget(messages[-num_tools_seq:], env=get_active_env(effective_task_id), config=_tool_budget) # ── /steer injection ────────────────────────────────────────────── # See _execute_tool_calls_parallel for the rationale. Same hook, # applied to sequential execution as well. - if num_tools_seq > 0: + if finalize and num_tools_seq > 0: agent._apply_pending_steer_to_tool_results(messages, num_tools_seq) +def execute_tool_calls_segmented(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, segments=None) -> None: + """Execute a mixed tool-call batch as ordered parallel/sequential segments. + + ``segments`` is the ``(kind, calls)`` plan from + ``_plan_tool_batch_segments``: maximal contiguous runs of parallel-safe + calls execute on the concurrent path, barrier calls on the sequential + path, strictly in the model's original call order. Because segments are + contiguous, every tool result is still appended one-per-call in emission + order and no call ever starts before an earlier barrier finishes — + identical ordering and side-effect boundaries to fully-sequential + execution, with I/O parallelism recovered inside the safe runs. + + Turn-end work (aggregate budget enforcement + /steer injection) is done + once here for the WHOLE batch; the per-segment executor calls run with + ``finalize=False`` so a multi-segment turn cannot multiply the budget or + truncate a steer marker. + + Interrupt semantics: each segment executor already checks + ``agent._interrupt_requested`` up front and appends a cancelled/skipped + result per call, so an interrupt during segment *k* drains segments + *k+1..n* without executing them while preserving one result per + tool_call_id. + """ + from types import SimpleNamespace + + if segments is None: + segments = _plan_tool_batch_segments(assistant_message.tool_calls) + + for kind, calls in segments: + segment_message = SimpleNamespace(tool_calls=list(calls)) + if kind == "parallel": + execute_tool_calls_concurrent( + agent, segment_message, messages, effective_task_id, api_call_count, + finalize=False, + ) + else: + execute_tool_calls_sequential( + agent, segment_message, messages, effective_task_id, api_call_count, + finalize=False, + ) + + # ── Whole-turn finalize (budget + /steer) ───────────────────────── + total_tools = len(assistant_message.tool_calls) + if total_tools > 0: + _tool_budget = _budget_for_agent(agent) + enforce_turn_budget( + messages[-total_tools:], + env=get_active_env(effective_task_id), + config=_tool_budget, + ) + agent._apply_pending_steer_to_tool_results(messages, total_tools) + + __all__ = [ "execute_tool_calls_concurrent", "execute_tool_calls_sequential", + "execute_tool_calls_segmented", ] diff --git a/run_agent.py b/run_agent.py index 31cca895592f..504402459c21 100644 --- a/run_agent.py +++ b/run_agent.py @@ -198,7 +198,7 @@ from agent.trajectory import ( save_trajectory as _save_trajectory_to_file, ) from agent.tool_dispatch_helpers import ( - _should_parallelize_tool_batch, + _should_parallelize_tool_batch, # noqa: F401 # re-exported for tests that `from run_agent import _should_parallelize_tool_batch` _is_destructive_command, # noqa: F401 # re-exported for tests that access `run_agent._is_destructive_command` _extract_parallel_scope_path, # noqa: F401 # re-exported for tests that `from run_agent import _extract_parallel_scope_path` _paths_overlap, # noqa: F401 # re-exported for tests that `from run_agent import _paths_overlap` @@ -5714,22 +5714,41 @@ class AIAgent: def _execute_tool_calls(self, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: """Execute tool calls from the assistant message and append results to messages. - Dispatches to concurrent execution only for batches that look - independent: read-only tools may always share the parallel path, while - file reads/writes may do so only when their target paths do not overlap. + The segment planner splits the batch into maximal contiguous runs of + parallel-safe calls (read-only tools, non-overlapping file targets, + opted-in MCP tools) separated by sequential barriers (interactive, + unsafe, or unrecognized tools). Homogeneous batches keep their + original single-path dispatch; mixed batches execute segment by + segment in emission order so safe subsets still run concurrently + while side-effect ordering is preserved. """ tool_calls = assistant_message.tool_calls # Allow _vprint during tool execution even with stream consumers self._executing_tools = True try: - if not _should_parallelize_tool_batch(tool_calls): + if len(tool_calls) <= 1: return self._execute_tool_calls_sequential( assistant_message, messages, effective_task_id, api_call_count ) - return self._execute_tool_calls_concurrent( - assistant_message, messages, effective_task_id, api_call_count + from agent.tool_dispatch_helpers import _plan_tool_batch_segments + segments = _plan_tool_batch_segments(tool_calls) + + if len(segments) == 1: + kind = segments[0][0] + if kind == "parallel": + return self._execute_tool_calls_concurrent( + assistant_message, messages, effective_task_id, api_call_count + ) + return self._execute_tool_calls_sequential( + assistant_message, messages, effective_task_id, api_call_count + ) + + from agent.tool_executor import execute_tool_calls_segmented + return execute_tool_calls_segmented( + self, assistant_message, messages, effective_task_id, api_call_count, + segments=segments, ) finally: self._executing_tools = False diff --git a/tests/run_agent/test_tool_batch_segmentation.py b/tests/run_agent/test_tool_batch_segmentation.py new file mode 100644 index 000000000000..9cde31479d00 --- /dev/null +++ b/tests/run_agent/test_tool_batch_segmentation.py @@ -0,0 +1,400 @@ +"""Segment-aware mixed tool-batch dispatch. + +A model response containing several parallel-safe reads plus one unsafe +tool used to lose ALL concurrency: `_should_parallelize_tool_batch` was +all-or-nothing, so one barrier call forced the entire batch onto the +sequential path. `_plan_tool_batch_segments` now splits the batch into +ordered segments — maximal contiguous runs of parallel-safe calls execute +concurrently, barrier calls sequentially — while preserving: + + * model tool-result ordering (one result per call, in emission order), + * side-effect boundaries (no call starts before an earlier barrier ends). +""" + +import json +import threading +import time +import uuid +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from run_agent import AIAgent +from agent.tool_dispatch_helpers import ( + _plan_tool_batch_segments, + _should_parallelize_tool_batch, +) + + +def _tc(name="web_search", arguments="{}", call_id=None): + return SimpleNamespace( + id=call_id or f"call_{uuid.uuid4().hex[:8]}", + type="function", + function=SimpleNamespace(name=name, arguments=arguments), + ) + + +def _kinds(segments): + return [kind for kind, _ in segments] + + +def _flatten_ids(segments): + return [tc.id for _, calls in segments for tc in calls] + + +# --------------------------------------------------------------------------- +# Planner unit tests +# --------------------------------------------------------------------------- + + +class TestPlanToolBatchSegments: + def test_all_safe_batch_is_single_parallel_segment(self): + calls = [_tc("web_search"), _tc("read_file", '{"path":"a.py"}'), _tc("web_extract")] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel"] + assert _flatten_ids(segments) == [c.id for c in calls] + + def test_three_safe_reads_plus_trailing_unsafe_keeps_reads_parallel(self): + """The headline case: 3 safe reads + 1 unsafe tool must NOT go fully sequential.""" + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("read_file", '{"path":"a.py"}', call_id="r3"), + _tc("terminal", '{"command":"echo hi"}', call_id="b1"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential"] + assert [tc.id for tc in segments[0][1]] == ["r1", "r2", "r3"] + assert [tc.id for tc in segments[1][1]] == ["b1"] + + def test_barrier_in_middle_splits_runs_and_preserves_order(self): + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("terminal", '{"command":"make"}', call_id="b1"), + _tc("web_search", call_id="r3"), + _tc("web_search", call_id="r4"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential", "parallel"] + assert _flatten_ids(segments) == ["r1", "r2", "b1", "r3", "r4"] + + def test_single_safe_call_after_barrier_is_demoted_and_merged(self): + # parallel run of 1 gains nothing — demote to sequential and merge + # with the adjacent barrier segment. + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("terminal", '{"command":"make"}', call_id="b1"), + _tc("web_search", call_id="r3"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential"] + assert [tc.id for tc in segments[1][1]] == ["b1", "r3"] + + def test_adjacent_barriers_merge_into_one_sequential_segment(self): + calls = [ + _tc("terminal", '{"command":"a"}', call_id="b1"), + _tc("terminal", '{"command":"b"}', call_id="b2"), + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["sequential", "parallel"] + assert [tc.id for tc in segments[0][1]] == ["b1", "b2"] + + def test_never_parallel_tool_is_a_barrier(self): + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("clarify", '{"question":"?"}', call_id="c1"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential"] + assert [tc.id for tc in segments[1][1]] == ["c1"] + + def test_malformed_args_call_is_a_barrier_not_a_batch_poison(self): + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("web_search", "{not json", call_id="bad"), + _tc("web_search", call_id="r3"), + _tc("web_search", call_id="r4"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential", "parallel"] + assert [tc.id for tc in segments[1][1]] == ["bad"] + + def test_non_dict_args_call_is_a_barrier(self): + calls = [ + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + _tc("web_search", '"just a string"', call_id="bad"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["parallel", "sequential"] + + def test_overlapping_paths_split_across_segments(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + calls = [ + _tc("read_file", '{"path":"a.py"}', call_id="w1"), + _tc("web_search", call_id="r1"), + _tc("write_file", '{"path":"a.py","content":"x"}', call_id="w2"), + _tc("web_search", call_id="r2"), + ] + segments = _plan_tool_batch_segments(calls) + # w2 conflicts with w1 → closes the first run; w2+r2 form the second. + assert _kinds(segments) == ["parallel", "parallel"] + assert [tc.id for tc in segments[0][1]] == ["w1", "r1"] + assert [tc.id for tc in segments[1][1]] == ["w2", "r2"] + # Order and completeness preserved. + assert _flatten_ids(segments) == ["w1", "r1", "w2", "r2"] + + def test_path_scoped_tool_without_path_is_a_barrier(self): + calls = [ + _tc("read_file", "{}", call_id="nopath"), + _tc("web_search", call_id="r1"), + _tc("web_search", call_id="r2"), + ] + segments = _plan_tool_batch_segments(calls) + assert _kinds(segments) == ["sequential", "parallel"] + + def test_flattened_segments_always_preserve_emission_order(self): + calls = [ + _tc("terminal", '{"command":"x"}', call_id="b1"), + _tc("web_search", call_id="r1"), + _tc("clarify", '{"question":"?"}', call_id="c1"), + _tc("read_file", '{"path":"a.py"}', call_id="r2"), + _tc("read_file", '{"path":"b.py"}', call_id="r3"), + ] + segments = _plan_tool_batch_segments(calls) + assert _flatten_ids(segments) == ["b1", "r1", "c1", "r2", "r3"] + + +class TestShouldParallelizeBackwardCompat: + """The boolean gate is now a view over the planner — same answers as before.""" + + def test_single_call_is_sequential(self): + assert not _should_parallelize_tool_batch([_tc("web_search")]) + + def test_all_safe_batch_is_parallel(self): + assert _should_parallelize_tool_batch([_tc("web_search"), _tc("web_extract")]) + + def test_mixed_batch_is_not_wholly_parallel(self): + assert not _should_parallelize_tool_batch( + [_tc("web_search"), _tc("terminal", '{"command":"ls"}')] + ) + + def test_clarify_anywhere_blocks_whole_batch_parallelism(self): + assert not _should_parallelize_tool_batch( + [_tc("web_search"), _tc("clarify", '{"question":"?"}')] + ) + + +# --------------------------------------------------------------------------- +# Dispatcher integration +# --------------------------------------------------------------------------- + + +def _make_tool_defs(*names: str) -> list: + return [ + { + "type": "function", + "function": { + "name": n, + "description": f"{n} tool", + "parameters": {"type": "object", "properties": {}}, + }, + } + for n in names + ] + + +@pytest.fixture() +def agent(): + with ( + patch( + "run_agent.get_tool_definitions", + return_value=_make_tool_defs("web_search", "terminal"), + ), + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + ): + a = AIAgent( + api_key="test-key-1234567890", + base_url="https://openrouter.ai/api/v1", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + a.client = MagicMock() + return a + + +class TestSegmentedDispatchIntegration: + def test_mixed_batch_runs_safe_prefix_concurrently_and_barrier_after(self, agent): + """Two web_search calls must overlap in time; terminal must start only + after both finish; results land in the model's emission order.""" + calls = [ + _tc("web_search", '{"query":"a"}', call_id="s1"), + _tc("web_search", '{"query":"b"}', call_id="s2"), + _tc("terminal", '{"command":"echo done"}', call_id="t1"), + ] + msg = SimpleNamespace(content="", tool_calls=calls) + messages = [] + + rendezvous = threading.Barrier(2, timeout=10) + events = [] + events_lock = threading.Lock() + + def fake_handle(name, args, task_id, **kwargs): + with events_lock: + events.append(("start", name, kwargs["tool_call_id"])) + if name == "web_search": + # Both searches must be in flight at once to pass this + # barrier — proves genuine concurrency for the safe prefix. + rendezvous.wait() + with events_lock: + events.append(("end", name, kwargs["tool_call_id"])) + return json.dumps({"ok": name}) + + with patch("run_agent.handle_function_call", side_effect=fake_handle): + agent._execute_tool_calls(msg, messages, "task-1") + + # One result per call, in emission order. + assert [m["tool_call_id"] for m in messages] == ["s1", "s2", "t1"] + assert all(m["role"] == "tool" for m in messages) + + # The barrier (terminal) started only after BOTH searches ended. + terminal_start = events.index(("start", "terminal", "t1")) + search_ends = [ + i for i, e in enumerate(events) if e[0] == "end" and e[1] == "web_search" + ] + assert len(search_ends) == 2 + assert all(i < terminal_start for i in search_ends) + + def test_mixed_batch_preserves_order_with_barrier_in_middle(self, agent): + calls = [ + _tc("web_search", '{"query":"a"}', call_id="s1"), + _tc("web_search", '{"query":"b"}', call_id="s2"), + _tc("terminal", '{"command":"touch x"}', call_id="t1"), + _tc("web_search", '{"query":"c"}', call_id="s3"), + _tc("web_search", '{"query":"d"}', call_id="s4"), + ] + msg = SimpleNamespace(content="", tool_calls=calls) + messages = [] + executed = [] + lock = threading.Lock() + + def fake_handle(name, args, task_id, **kwargs): + with lock: + executed.append(kwargs["tool_call_id"]) + return json.dumps({"ok": True}) + + with patch("run_agent.handle_function_call", side_effect=fake_handle): + agent._execute_tool_calls(msg, messages, "task-1") + + assert [m["tool_call_id"] for m in messages] == ["s1", "s2", "t1", "s3", "s4"] + # Barrier ordering: t1 executed after {s1,s2} and before {s3,s4}. + t1_pos = executed.index("t1") + assert {"s1", "s2"} == set(executed[:t1_pos]) + assert {"s3", "s4"} == set(executed[t1_pos + 1:]) + + def test_homogeneous_safe_batch_still_uses_plain_concurrent_path(self, agent): + calls = [_tc("web_search", '{"query":"a"}'), _tc("web_search", '{"query":"b"}')] + msg = SimpleNamespace(content="", tool_calls=calls) + + with ( + patch.object(agent, "_execute_tool_calls_concurrent") as conc, + patch.object(agent, "_execute_tool_calls_sequential") as seq, + ): + agent._execute_tool_calls(msg, [], "task-1") + + conc.assert_called_once() + seq.assert_not_called() + + def test_homogeneous_unsafe_batch_still_uses_plain_sequential_path(self, agent): + calls = [ + _tc("terminal", '{"command":"a"}'), + _tc("terminal", '{"command":"b"}'), + ] + msg = SimpleNamespace(content="", tool_calls=calls) + + with ( + patch.object(agent, "_execute_tool_calls_concurrent") as conc, + patch.object(agent, "_execute_tool_calls_sequential") as seq, + ): + agent._execute_tool_calls(msg, [], "task-1") + + seq.assert_called_once() + conc.assert_not_called() + + def test_single_call_uses_sequential_path(self, agent): + msg = SimpleNamespace(content="", tool_calls=[_tc("web_search", '{"query":"a"}')]) + + with ( + patch.object(agent, "_execute_tool_calls_concurrent") as conc, + patch.object(agent, "_execute_tool_calls_sequential") as seq, + ): + agent._execute_tool_calls(msg, [], "task-1") + + seq.assert_called_once() + conc.assert_not_called() + + def test_interrupt_during_barrier_drains_later_segments(self, agent): + """Interrupt raised while the barrier tool runs: the trailing parallel + segment must be drained with cancelled results — one per call — + without executing.""" + calls = [ + _tc("web_search", '{"query":"a"}', call_id="s1"), + _tc("web_search", '{"query":"b"}', call_id="s2"), + _tc("terminal", '{"command":"long"}', call_id="t1"), + _tc("web_search", '{"query":"c"}', call_id="s3"), + _tc("web_search", '{"query":"d"}', call_id="s4"), + ] + msg = SimpleNamespace(content="", tool_calls=calls) + messages = [] + executed = [] + lock = threading.Lock() + + def fake_handle(name, args, task_id, **kwargs): + with lock: + executed.append(kwargs["tool_call_id"]) + if kwargs["tool_call_id"] == "t1": + agent._interrupt_requested = True + return json.dumps({"ok": True}) + + with patch("run_agent.handle_function_call", side_effect=fake_handle): + agent._execute_tool_calls(msg, messages, "task-1") + + # Every call still gets exactly one result, in order. + assert [m["tool_call_id"] for m in messages] == ["s1", "s2", "t1", "s3", "s4"] + # s3/s4 were never executed. + assert "s3" not in executed and "s4" not in executed + for m in messages[-2:]: + assert "cancelled" in m["content"] or "skipped" in m["content"] + + def test_steer_lands_exactly_once_in_mixed_batch(self, agent): + """Steer is drained once (per-tool drains + one dispatcher-level + finalize) — the marker must appear exactly once across the batch, + never duplicated by segment boundaries.""" + calls = [ + _tc("web_search", '{"query":"a"}', call_id="s1"), + _tc("web_search", '{"query":"b"}', call_id="s2"), + _tc("terminal", '{"command":"echo hi"}', call_id="t1"), + ] + msg = SimpleNamespace(content="", tool_calls=calls) + messages = [] + + def fake_handle(name, args, task_id, **kwargs): + return json.dumps({"ok": True}) + + agent.steer("focus on the tests") + with patch("run_agent.handle_function_call", side_effect=fake_handle): + agent._execute_tool_calls(msg, messages, "task-1") + + contents = [m["content"] for m in messages] + hits = [c for c in contents if "focus on the tests" in c] + assert len(hits) == 1 From 7e84d2b5a43d47b1da33cfa662d0f87991774b1c Mon Sep 17 00:00:00 2001 From: ethernet Date: Fri, 10 Jul 2026 22:12:59 -0400 Subject: [PATCH 096/149] fix(terminal): ignore stale env.cwd from a different session's cd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The terminal environment is shared process-globally (collapsed to the default key), so env.cwd tracks the LAST session that ran a command. _resolve_command_cwd() trusted env.cwd unconditionally — no ownership check — so when session A left env.cwd pointing at A's checkout, session B's first terminal command inherited A's stale cwd and ran in the wrong workspace. The file tools already solved this exact shared-env problem with _live_cwd_if_owned() checking env.cwd_owner. The terminal tool never got the same guard. Fix: capture env.cwd_owner BEFORE the current session claims it, and pass it as prev_owner to _resolve_command_cwd. When the previous owner was a different session, env.cwd is stale — fall through to default_cwd (the config/override cwd for this session) instead. Once the session has claimed the env, subsequent calls in the same session still trust env.cwd so in-session state survives. --- tests/tools/test_terminal_task_cwd.py | 83 +++++++++++++++++++++++++++ tools/terminal_tool.py | 24 ++++++++ 2 files changed, 107 insertions(+) diff --git a/tests/tools/test_terminal_task_cwd.py b/tests/tools/test_terminal_task_cwd.py index 85817fa53351..009e3f7e089d 100644 --- a/tests/tools/test_terminal_task_cwd.py +++ b/tests/tools/test_terminal_task_cwd.py @@ -223,6 +223,89 @@ def test_registering_non_cwd_override_leaves_live_env_cwd_untouched(monkeypatch) assert fake_env.cwd == "/workspace/keep" +def test_stale_env_cwd_from_different_session_is_ignored(monkeypatch): + """A different session's `cd` left env.cwd pointing at its checkout. + + The terminal env is shared (collapsed to "default"), so env.cwd tracks the + LAST session that ran a command. When session B claims the env after + session A left it in A's worktree, the first command must NOT run in A's + leftover cwd — it must fall through to the config/override cwd (this + session's own workspace). + """ + calls = [] + + class FakeEnv: + env = {} + cwd = "/home/user/src/hermes-desktop-tipc/apps/desktop" + cwd_owner = "session-A-key" + + def execute(self, command, **kwargs): + calls.append((command, kwargs)) + return {"output": "ok", "returncode": 0} + + task_id = "session-B" + monkeypatch.setattr(terminal_tool, "_active_environments", {"default": FakeEnv()}) + monkeypatch.setattr(terminal_tool, "_last_activity", {}) + monkeypatch.setattr(terminal_tool, "_task_env_overrides", {}) + monkeypatch.setattr(terminal_tool, "_get_env_config", lambda: _minimal_terminal_config(cwd="/home/user/src/hermes-agent")) + monkeypatch.setattr(terminal_tool, "_start_cleanup_thread", lambda: None) + monkeypatch.setattr(terminal_tool, "_resolve_container_task_id", lambda value: "default") + monkeypatch.setattr( + terminal_tool, + "_check_all_guards", + lambda command, env_type, **kwargs: {"approved": True}, + ) + + result = json.loads(terminal_tool.terminal_tool(command="pwd", task_id=task_id)) + + assert result["exit_code"] == 0 + # The command must run in the config cwd (hermes-agent), NOT the stale + # env.cwd left by session A (hermes-desktop-tipc). + assert calls == [("pwd", {"timeout": 60, "cwd": "/home/user/src/hermes-agent"})] + + +def test_same_session_env_cwd_is_trusted_after_first_claim(monkeypatch): + """Once a session has claimed the env, subsequent commands trust env.cwd. + + The prev_owner check only rejects env.cwd when a DIFFERENT session owned it + before this call. After the first command (which claims ownership), + subsequent calls in the same session should trust the live env.cwd so that + in-session `cd` state survives. + """ + calls = [] + + class FakeEnv: + env = {} + cwd = "/workspace/deep" + cwd_owner = "session-X" + + def execute(self, command, **kwargs): + calls.append((command, kwargs)) + return {"output": "ok", "returncode": 0} + + env = FakeEnv() + task_id = "session-X" + monkeypatch.setattr(terminal_tool, "_active_environments", {"default": env}) + monkeypatch.setattr(terminal_tool, "_last_activity", {}) + monkeypatch.setattr(terminal_tool, "_task_env_overrides", {}) + monkeypatch.setattr(terminal_tool, "_get_env_config", lambda: _minimal_terminal_config(cwd="/workspace/config")) + monkeypatch.setattr(terminal_tool, "_start_cleanup_thread", lambda: None) + monkeypatch.setattr(terminal_tool, "_resolve_container_task_id", lambda value: "default") + monkeypatch.setattr( + terminal_tool, + "_check_all_guards", + lambda command, env_type, **kwargs: {"approved": True}, + ) + + # First call: env was owned by "session-X" (same session_key since + # get_current_session_key falls back to task_id). prev_owner == current + # session, so env.cwd is trusted. + result = json.loads(terminal_tool.terminal_tool(command="pwd", task_id=task_id)) + + assert result["exit_code"] == 0 + assert calls == [("pwd", {"timeout": 60, "cwd": "/workspace/deep"})] + + def test_safe_getcwd_returns_real_cwd(monkeypatch): monkeypatch.setattr(terminal_tool.os, "getcwd", lambda: "/home/user/project") assert terminal_tool._safe_getcwd() == "/home/user/project" diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index fc13367eb116..692aa1e4a81c 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -1988,6 +1988,7 @@ def _resolve_command_cwd( workdir: Optional[str], env: Any, default_cwd: str, + prev_owner: Optional[str] = None, ) -> str: """Return the cwd for a command, preferring the live session cwd. @@ -1996,12 +1997,29 @@ def _resolve_command_cwd( new directory in ``env.cwd``, but foreground/background calls kept forcing the old cwd back through ``env.execute(..., cwd=...)``. Explicit ``workdir=`` must still override everything. + + When ``prev_owner`` is provided and differs from the current session, + ``env.cwd`` was mutated by a *different* session's ``cd`` and must NOT be + trusted — fall through to ``default_cwd`` (the config/override cwd) so + the command runs in this session's own workspace, not the previous + session's leftover checkout. This mirrors the ``_live_cwd_if_owned`` + guard file_tools uses for the same shared-env problem. """ if workdir: return workdir live_cwd = getattr(env, "cwd", None) if isinstance(live_cwd, str) and live_cwd.strip(): + # The env is shared (collapsed to "default"); its cwd tracks the LAST + # session that ran a command. If a different session owned the env + # before this call claimed it, env.cwd is that session's leftover `cd` + # — not ours. Don't use it. + if prev_owner is not None: + session_key = getattr(env, "cwd_owner", "") + # cwd_owner was already overwritten to the current session at the + # call site, so compare against the captured previous owner. + if prev_owner and prev_owner != "default" and session_key != prev_owner: + return default_cwd return live_cwd return default_cwd @@ -2354,6 +2372,10 @@ def terminal_tool( from tools.approval import get_current_session_key session_key = get_current_session_key(default="") or (task_id or "") + # Capture the env's previous owner BEFORE claiming it — _resolve_command_cwd + # needs to know whether env.cwd was left by a *different* session's `cd` + # (in which case it's stale for this session and must be ignored). + prev_cwd_owner = getattr(env, "cwd_owner", "") or "" try: env.cwd_owner = session_key except Exception: @@ -2369,6 +2391,7 @@ def terminal_tool( workdir=workdir, env=env, default_cwd=cwd, + prev_owner=prev_cwd_owner, ) try: if env_type == "local": @@ -2629,6 +2652,7 @@ def terminal_tool( workdir=workdir, env=env, default_cwd=cwd, + prev_owner=prev_cwd_owner, ) execute_kwargs = { "timeout": effective_timeout, From 1813d3046c77271e9a2d284781b601dab5c2f2ea Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Tue, 14 Jul 2026 16:50:03 -0400 Subject: [PATCH 097/149] feat(desktop): add a chat backdrop on/off toggle The faint statue backdrop behind the transcript was only switchable via the DEV-only leva panel. Add a persisted Appearance toggle (default on) so users can hide it; the Backdrop simply skips rendering when off. --- .../src/app/settings/appearance-settings.tsx | 20 ++++++++++++++++++ apps/desktop/src/components/Backdrop.tsx | 6 +++++- apps/desktop/src/i18n/en.ts | 2 ++ apps/desktop/src/i18n/ja.ts | 2 ++ apps/desktop/src/i18n/types.ts | 2 ++ apps/desktop/src/i18n/zh-hant.ts | 2 ++ apps/desktop/src/i18n/zh.ts | 2 ++ apps/desktop/src/store/backdrop.ts | 21 +++++++++++++++++++ 8 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/store/backdrop.ts diff --git a/apps/desktop/src/app/settings/appearance-settings.tsx b/apps/desktop/src/app/settings/appearance-settings.tsx index 559978e41568..32deb34d2d47 100644 --- a/apps/desktop/src/app/settings/appearance-settings.tsx +++ b/apps/desktop/src/app/settings/appearance-settings.tsx @@ -12,6 +12,7 @@ import { Check, Download, Loader2, Palette, Trash2 } from '@/lib/icons' import { selectableCardClass } from '@/lib/selectable-card' import { normalize } from '@/lib/text' import { cn } from '@/lib/utils' +import { $backdrop, setBackdrop } from '@/store/backdrop' import { $embedAllowed, $embedMode, clearEmbedAllowed, type EmbedMode, setEmbedMode } from '@/store/embed-consent' import { $activeGatewayProfile, $profiles, normalizeProfileKey } from '@/store/profile' import { $toolViewMode, setToolViewMode } from '@/store/tool-view' @@ -248,6 +249,7 @@ export function AppearanceSettings() { const embedMode = useStore($embedMode) const embedAllowed = useStore($embedAllowed) const translucency = useStore($translucency) + const backdrop = useStore($backdrop) const installs = useStore($marketplaceInstalls) const profiles = useStore($profiles) const activeProfileKey = normalizeProfileKey(useStore($activeGatewayProfile)) @@ -451,6 +453,24 @@ export function AppearanceSettings() { title={a.translucencyTitle} /> + { + triggerHaptic('selection') + setBackdrop(id === 'on') + }} + options={[ + { id: 'off', label: t.common.off }, + { id: 'on', label: t.common.on } + ]} + value={backdrop ? 'on' : 'off'} + /> + } + description={a.backdropDesc} + title={a.backdropTitle} + /> + `${import.meta.env.BASE_URL}${path.replace(/ export function Backdrop() { const [controlsOpen, setControlsOpen] = useState(false) + const on = useStore($backdrop) useEffect(() => { if (!import.meta.env.DEV) { @@ -87,7 +91,7 @@ export function Backdrop() { <>