diff --git a/tests/tools/test_local_env_cwd_recovery.py b/tests/tools/test_local_env_cwd_recovery.py new file mode 100644 index 0000000000..96e61fa1cc --- /dev/null +++ b/tests/tools/test_local_env_cwd_recovery.py @@ -0,0 +1,151 @@ +"""Tests for LocalEnvironment recovery when ``self.cwd`` is deleted. + +When a tool call inside the persistent terminal session ``rm -rf``'s its own +working directory, the next ``subprocess.Popen(..., cwd=self.cwd)`` would +otherwise raise ``FileNotFoundError`` before bash starts, wedging every +subsequent terminal/file-tool call until the gateway restarts. + +Regression coverage for https://github.com/NousResearch/hermes-agent/issues/17558. +""" + +import os +import shutil +import tempfile +import threading +from unittest.mock import MagicMock, patch + +from tools.environments.local import ( + LocalEnvironment, + _resolve_safe_cwd, +) + + +class TestResolveSafeCwd: + """Pure-function unit tests for the recovery helper.""" + + def test_returns_cwd_when_directory_exists(self, tmp_path): + path = str(tmp_path) + assert _resolve_safe_cwd(path) == path + + def test_walks_up_to_first_existing_ancestor(self, tmp_path): + nested = tmp_path / "child" / "grandchild" + nested.mkdir(parents=True) + deleted = str(nested) + shutil.rmtree(tmp_path / "child") + + # The deepest existing ancestor on the path is tmp_path itself. + assert _resolve_safe_cwd(deleted) == str(tmp_path) + + def test_falls_back_when_path_is_empty(self): + assert _resolve_safe_cwd("") == tempfile.gettempdir() + + def test_returns_tempdir_when_nothing_on_path_exists(self, monkeypatch): + monkeypatch.setattr(os.path, "isdir", lambda p: False) + assert _resolve_safe_cwd("/no/such/dir") == tempfile.gettempdir() + + +def _fake_interrupt(): + return threading.Event() + + +def _make_fake_popen(captured: dict): + def fake_popen(cmd, **kwargs): + captured["cwd"] = kwargs.get("cwd") + captured["env"] = kwargs.get("env", {}) + proc = MagicMock() + proc.poll.return_value = 0 + proc.returncode = 0 + proc.stdout = MagicMock( + __iter__=lambda s: iter([]), + __next__=lambda s: (_ for _ in ()).throw(StopIteration), + ) + proc.stdin = MagicMock() + return proc + return fake_popen + + +class TestRunBashCwdRecovery: + """End-to-end recovery: deleted ``self.cwd`` must not crash Popen.""" + + def test_recovers_when_cwd_deleted_after_init(self, tmp_path, caplog): + """Reproduces the wedge from #17558: cwd was valid when the + snapshot was taken, but a subsequent command deleted it before the + next ``Popen``.""" + wedged = tmp_path / "wedge-repro" + wedged.mkdir() + + with patch.object(LocalEnvironment, "init_session", autospec=True, return_value=None): + env = LocalEnvironment(cwd=str(wedged), timeout=10) + + # The previous tool call deleted the working directory. + shutil.rmtree(wedged) + assert env.cwd == str(wedged) and not os.path.isdir(env.cwd) + + captured = {} + with patch("tools.environments.local._find_bash", return_value="/bin/bash"), \ + patch("subprocess.Popen", side_effect=_make_fake_popen(captured)), \ + patch("tools.terminal_tool._interrupt_event", _fake_interrupt()), \ + caplog.at_level("WARNING", logger="tools.environments.local"): + env.execute("echo hello") + + # Popen must have been handed a real, existing directory. + assert captured["cwd"] == str(tmp_path) + assert os.path.isdir(captured["cwd"]) + + # ``self.cwd`` is updated so the next call doesn't re-warn. + assert env.cwd == str(tmp_path) + + # The warning surfaces the wedge so it isn't silently masked. + assert any("missing on disk" in rec.message for rec in caplog.records) + + def test_no_warning_when_cwd_still_exists(self, tmp_path, caplog): + with patch.object(LocalEnvironment, "init_session", autospec=True, return_value=None): + env = LocalEnvironment(cwd=str(tmp_path), timeout=10) + + captured = {} + with patch("tools.environments.local._find_bash", return_value="/bin/bash"), \ + patch("subprocess.Popen", side_effect=_make_fake_popen(captured)), \ + patch("tools.terminal_tool._interrupt_event", _fake_interrupt()), \ + caplog.at_level("WARNING", logger="tools.environments.local"): + env.execute("echo hello") + + assert captured["cwd"] == str(tmp_path) + assert env.cwd == str(tmp_path) + assert not any("missing on disk" in rec.message for rec in caplog.records) + + +class TestUpdateCwdRejectsMissingPaths: + """``_update_cwd`` must not propagate a deleted path back into ``self.cwd``.""" + + def test_skips_assignment_when_marker_path_missing(self, tmp_path): + original = tmp_path / "starting" + original.mkdir() + + with patch.object(LocalEnvironment, "init_session", autospec=True, return_value=None): + env = LocalEnvironment(cwd=str(original), timeout=10) + + # Simulate the stale-marker case: the prior command's ``pwd -P`` left + # a path in the cwd file, but that path has since been deleted. + deleted = tmp_path / "wedge-repro" + with open(env._cwd_file, "w") as f: + f.write(str(deleted)) + + env._update_cwd({"output": "", "returncode": 0}) + + assert env.cwd == str(original) + + def test_accepts_assignment_when_marker_path_exists(self, tmp_path): + original = tmp_path / "starting" + original.mkdir() + new_dir = tmp_path / "next" + new_dir.mkdir() + + with patch.object(LocalEnvironment, "init_session", autospec=True, return_value=None): + env = LocalEnvironment(cwd=str(original), timeout=10) + + with open(env._cwd_file, "w") as f: + f.write(str(new_dir)) + + env._update_cwd({"output": "", "returncode": 0}) + + assert env.cwd == str(new_dir) diff --git a/tools/environments/local.py b/tools/environments/local.py index 3200e63e60..99a7d9d12c 100644 --- a/tools/environments/local.py +++ b/tools/environments/local.py @@ -1,5 +1,6 @@ """Local execution environment — spawn-per-call with session snapshot.""" +import logging import os import platform import shutil @@ -12,6 +13,30 @@ from tools.environments.base import BaseEnvironment, _pipe_stdin _IS_WINDOWS = platform.system() == "Windows" +logger = logging.getLogger(__name__) + + +def _resolve_safe_cwd(cwd: str) -> str: + """Return ``cwd`` if it exists as a directory, else the nearest existing + ancestor. Falls back to ``tempfile.gettempdir()`` only if walking up the + path can't find any existing directory (effectively never on a healthy + filesystem, but cheap belt-and-braces). + + Used by ``_run_bash`` to recover when the configured cwd is gone — most + commonly because a previous tool call deleted its own working directory + (issue #17558). Without this guard, ``subprocess.Popen(..., cwd=...)`` + raises ``FileNotFoundError`` before bash starts, wedging every subsequent + terminal call until the gateway restarts. + """ + if cwd and os.path.isdir(cwd): + return cwd + parent = os.path.dirname(cwd) if cwd else "" + while parent and parent != os.path.dirname(parent): + if os.path.isdir(parent): + return parent + parent = os.path.dirname(parent) + return tempfile.gettempdir() + # Hermes-internal env vars that should NOT leak into terminal subprocesses. _HERMES_PROVIDER_ENV_FORCE_PREFIX = "_HERMES_FORCE_" @@ -358,6 +383,21 @@ class LocalEnvironment(BaseEnvironment): args = [bash, "-l", "-c", cmd_string] if login else [bash, "-c", cmd_string] run_env = _make_run_env(self.env) + # Recover when the cwd has been deleted out from under us — usually by + # a previous tool call that ran ``rm -rf`` on its own working dir + # (issue #17558). Popen would otherwise raise FileNotFoundError on + # the cwd before bash starts, wedging every subsequent call until the + # gateway restarts. + safe_cwd = _resolve_safe_cwd(self.cwd) + if safe_cwd != self.cwd: + logger.warning( + "LocalEnvironment cwd %r is missing on disk; " + "falling back to %r so terminal commands keep working.", + self.cwd, + safe_cwd, + ) + self.cwd = safe_cwd + proc = subprocess.Popen( args, text=True, @@ -452,11 +492,17 @@ class LocalEnvironment(BaseEnvironment): pass def _update_cwd(self, result: dict): - """Read CWD from temp file (local-only, no round-trip needed).""" + """Read CWD from temp file (local-only, no round-trip needed). + + Skip the assignment when the path no longer exists as a directory — + ``pwd -P`` on a deleted cwd can leave a stale value in the marker + file, and propagating it would re-wedge the next ``Popen``. The + ``_run_bash`` recovery path will resolve a safe fallback if needed. + """ try: with open(self._cwd_file) as f: cwd_path = f.read().strip() - if cwd_path: + if cwd_path and os.path.isdir(cwd_path): self.cwd = cwd_path except (OSError, FileNotFoundError): pass