mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
298 lines
10 KiB
Python
298 lines
10 KiB
Python
"""Regression tests for delegate_task isolation from parent Kanban workers."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shlex
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
# The subprocess-boundary tests below spawn ``sys.executable -c`` with a tmp
|
|
# cwd. Without an explicit PYTHONPATH the child resolves ``hermes_cli`` /
|
|
# ``agent`` through whatever install is on sys.path (in a worktree that is the
|
|
# MAIN checkout's editable install, which may not contain the code under
|
|
# test). Pin the repo root so the child always imports the tree being tested.
|
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _python_with_repo_path(code: str) -> str:
|
|
"""Build a shell command running *code* with the repo under test on PYTHONPATH."""
|
|
return (
|
|
f"PYTHONPATH={shlex.quote(str(_REPO_ROOT))} "
|
|
f"{shlex.quote(sys.executable)} -c {shlex.quote(code)}"
|
|
)
|
|
|
|
|
|
def _make_running_kanban_task(monkeypatch, tmp_path):
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
attachments_root = tmp_path / "attachments"
|
|
workspace = tmp_path / "parent-workspace"
|
|
workspace.mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
monkeypatch.setenv("HERMES_PROFILE", "parent-worker")
|
|
monkeypatch.setenv("HERMES_KANBAN_WORKSPACE", str(workspace))
|
|
monkeypatch.setenv("HERMES_KANBAN_ATTACHMENTS_ROOT", str(attachments_root))
|
|
|
|
from hermes_cli import kanban_db as kb
|
|
|
|
kb._INITIALIZED_PATHS.clear()
|
|
kb.init_db()
|
|
conn = kb.connect()
|
|
try:
|
|
tid = kb.create_task(
|
|
conn,
|
|
title="parent",
|
|
assignee="parent-worker",
|
|
workspace_kind="scratch",
|
|
workspace_path=str(workspace),
|
|
)
|
|
claim = kb.claim_task(conn, tid)
|
|
assert claim is not None
|
|
run_id = claim.id
|
|
finally:
|
|
conn.close()
|
|
|
|
monkeypatch.setenv("HERMES_KANBAN_TASK", tid)
|
|
monkeypatch.setenv("HERMES_KANBAN_RUN_ID", str(run_id))
|
|
return kb, tid, workspace, attachments_root
|
|
|
|
|
|
def test_delegated_child_context_suppresses_env_gated_kanban_tools(monkeypatch, tmp_path):
|
|
"""A delegate_task child must not inherit the parent's Kanban tool schema.
|
|
|
|
The parent process may be a dispatcher worker with HERMES_KANBAN_TASK set;
|
|
the child is only a subagent, not the run owner.
|
|
"""
|
|
monkeypatch.setenv("HERMES_KANBAN_TASK", "t_parent")
|
|
monkeypatch.setenv("HERMES_KANBAN_RUN_ID", "123")
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
|
|
import tools.kanban_tools # noqa: F401 - ensure registered
|
|
from agent.delegation_context import delegated_child_context
|
|
from model_tools import _clear_tool_defs_cache, get_tool_definitions
|
|
from tools.registry import invalidate_check_fn_cache
|
|
|
|
invalidate_check_fn_cache()
|
|
_clear_tool_defs_cache()
|
|
with delegated_child_context():
|
|
schema = get_tool_definitions(enabled_toolsets=["terminal"], quiet_mode=True)
|
|
|
|
names = {s["function"].get("name") for s in schema if "function" in s}
|
|
assert "terminal" in names
|
|
assert {n for n in names if n and n.startswith("kanban_")} == set()
|
|
|
|
|
|
def test_build_child_agent_strips_kanban_toolset_even_when_parent_is_worker(monkeypatch):
|
|
"""Child construction must fail closed even if the parent exposes kanban."""
|
|
captured = {}
|
|
|
|
class FakeAgent:
|
|
def __init__(self, **kwargs):
|
|
captured.update(kwargs)
|
|
self.valid_tool_names = {"terminal"}
|
|
self.session_id = "child-session"
|
|
|
|
import run_agent
|
|
from tools import delegate_tool
|
|
|
|
monkeypatch.setattr(run_agent, "AIAgent", FakeAgent)
|
|
monkeypatch.setattr(delegate_tool, "_load_config", lambda: {})
|
|
|
|
class Parent:
|
|
enabled_toolsets = ["terminal", "kanban"]
|
|
valid_tool_names = {"terminal", "kanban_complete", "kanban_comment"}
|
|
model = "test-model"
|
|
provider = "test-provider"
|
|
base_url = "http://example.invalid"
|
|
api_mode = "chat_completions"
|
|
platform = "cli"
|
|
session_id = "parent-session"
|
|
|
|
child = delegate_tool._build_child_agent(
|
|
task_index=0,
|
|
goal="review only",
|
|
context=None,
|
|
toolsets=None,
|
|
model=None,
|
|
max_iterations=3,
|
|
task_count=1,
|
|
parent_agent=Parent(),
|
|
)
|
|
|
|
assert child.valid_tool_names == {"terminal"}
|
|
assert "kanban" not in captured["enabled_toolsets"]
|
|
assert "kanban" in captured["disabled_toolsets"]
|
|
|
|
|
|
def test_delegate_child_execute_code_env_bridges_contextvar_and_scrubs_kanban(
|
|
monkeypatch,
|
|
tmp_path,
|
|
):
|
|
"""The real execute_code child-env builder must bridge ContextVar lineage.
|
|
|
|
Regression coverage for the vulnerable path: delegate_task marks child
|
|
execution with a ContextVar, while execute_code used to scrub plain
|
|
``os.environ`` and therefore never wrote HERMES_DELEGATED_CHILD_CONTEXT into
|
|
the sandbox env.
|
|
"""
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
monkeypatch.setenv("HERMES_KANBAN_TASK", "t_parent")
|
|
monkeypatch.setenv("HERMES_KANBAN_RUN_ID", "123")
|
|
monkeypatch.setenv("HERMES_KANBAN_DB", str(home / "kanban.db"))
|
|
monkeypatch.setenv("HERMES_KANBAN_WORKSPACE", str(tmp_path / "parent-workspace"))
|
|
monkeypatch.setenv("HERMES_KANBAN_CLAIM_LOCK", "lock")
|
|
monkeypatch.delenv("HERMES_DELEGATED_CHILD_CONTEXT", raising=False)
|
|
|
|
from agent.delegation_context import delegated_child_context
|
|
from tools.code_execution_tool import _scrub_child_env
|
|
|
|
with delegated_child_context():
|
|
env = _scrub_child_env(
|
|
dict(os.environ),
|
|
is_passthrough=lambda k: k.startswith("HERMES_KANBAN_"),
|
|
is_windows=False,
|
|
)
|
|
|
|
assert os.environ.get("HERMES_DELEGATED_CHILD_CONTEXT") is None
|
|
assert env["HERMES_HOME"] == str(home)
|
|
assert env["HERMES_DELEGATED_CHILD_CONTEXT"] == "1"
|
|
assert "HERMES_KANBAN_TASK" not in env
|
|
assert "HERMES_KANBAN_RUN_ID" not in env
|
|
assert "HERMES_KANBAN_DB" not in env
|
|
assert "HERMES_KANBAN_WORKSPACE" not in env
|
|
assert "HERMES_KANBAN_CLAIM_LOCK" not in env
|
|
|
|
|
|
def test_delegate_child_kanban_cli_cannot_delete_parent_board(
|
|
monkeypatch,
|
|
tmp_path,
|
|
):
|
|
kb, _tid, _workspace, _attachments_root = _make_running_kanban_task(
|
|
monkeypatch,
|
|
tmp_path,
|
|
)
|
|
kb.create_board("victim")
|
|
assert kb.board_exists("victim")
|
|
|
|
from agent.delegation_context import delegated_child_context
|
|
from tools.environments.local import LocalEnvironment
|
|
|
|
code = (
|
|
"from hermes_cli import kanban; "
|
|
"import argparse; "
|
|
"p=argparse.ArgumentParser(); "
|
|
"sub=p.add_subparsers(dest='cmd'); "
|
|
"kanban.build_parser(sub); "
|
|
"args=p.parse_args(['kanban','boards','rm','victim','--delete']); "
|
|
"raise SystemExit(kanban.kanban_command(args))"
|
|
)
|
|
env = LocalEnvironment(cwd=str(tmp_path), timeout=15)
|
|
try:
|
|
with delegated_child_context():
|
|
result = env.execute(
|
|
_python_with_repo_path(code),
|
|
timeout=15,
|
|
)
|
|
finally:
|
|
env.cleanup()
|
|
|
|
assert result["returncode"] == 1
|
|
assert "delegate_task child contexts cannot mutate Kanban tasks" in result["output"]
|
|
assert kb.board_exists("victim")
|
|
assert kb.board_dir("victim").is_dir()
|
|
|
|
|
|
def test_delegate_child_attach_url_guard_leaves_no_row_or_file(monkeypatch, tmp_path):
|
|
kb, tid, _workspace, attachments_root = _make_running_kanban_task(monkeypatch, tmp_path)
|
|
|
|
from agent.delegation_context import delegated_child_context
|
|
from tools import kanban_tools
|
|
|
|
def forbidden_download(*_args, **_kwargs):
|
|
raise AssertionError("delegated child guard must run before URL download")
|
|
|
|
monkeypatch.setattr(kanban_tools, "_download_url_with_cap", forbidden_download)
|
|
|
|
with delegated_child_context():
|
|
raw = kanban_tools._handle_attach_url({
|
|
"task_id": tid,
|
|
"url": "https://example.com/leak.txt",
|
|
})
|
|
|
|
payload = json.loads(raw)
|
|
assert payload["error"]
|
|
assert "delegate_task child" in payload["error"]
|
|
|
|
conn = kb.connect()
|
|
try:
|
|
assert kb.list_attachments(conn, tid) == []
|
|
finally:
|
|
conn.close()
|
|
task_dir = attachments_root / tid
|
|
assert not task_dir.exists() or list(task_dir.iterdir()) == []
|
|
|
|
|
|
def test_child_attempting_default_complete_does_not_finish_parent_or_delete_workspace(
|
|
monkeypatch,
|
|
tmp_path,
|
|
):
|
|
"""Deterministic E2E: a delegated child cannot complete its parent task."""
|
|
kb, tid, workspace, _attachments_root = _make_running_kanban_task(monkeypatch, tmp_path)
|
|
from tools import delegate_tool
|
|
from tools import kanban_tools
|
|
|
|
class Parent:
|
|
_current_task_id = tid
|
|
|
|
def _touch_activity(self, _desc):
|
|
return None
|
|
|
|
class Child:
|
|
tool_progress_callback = None
|
|
_delegate_saved_tool_names = []
|
|
_credential_pool = None
|
|
_subagent_id = "sa-test"
|
|
_delegate_depth = 1
|
|
_parent_subagent_id = None
|
|
model = "test-model"
|
|
session_prompt_tokens = 0
|
|
session_completion_tokens = 0
|
|
session_estimated_cost_usd = 0.0
|
|
session_reasoning_tokens = 0
|
|
|
|
def get_activity_summary(self):
|
|
return {"api_call_count": 0, "max_iterations": 1, "current_tool": None}
|
|
|
|
def run_conversation(self, user_message, task_id, **_kwargs):
|
|
attempted = kanban_tools._handle_complete({"summary": "wrong child completion"})
|
|
return {
|
|
"final_response": attempted,
|
|
"completed": True,
|
|
"api_calls": 0,
|
|
"messages": [],
|
|
}
|
|
|
|
def close(self):
|
|
return None
|
|
|
|
result = delegate_tool._run_single_child(0, "try to complete parent", Child(), Parent())
|
|
|
|
conn = kb.connect()
|
|
try:
|
|
task = kb.get_task(conn, tid)
|
|
run = kb.latest_run(conn, tid)
|
|
finally:
|
|
conn.close()
|
|
|
|
assert result["status"] == "completed"
|
|
assert "delegate_task child" in result["summary"]
|
|
assert task.status == "running"
|
|
assert run.status == "running"
|
|
assert workspace.is_dir()
|