mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-18 04:41:56 +00:00
The shutdown forensics added in #23285 caught tests/hermes_cli/ pytest runs sending SIGTERM to the developer's live gateway 5+ times in 3 days. Root cause: when a single test forgets to mock os.kill or find_gateway_pids, the real call leaks past the hermetic HERMES_HOME isolation — find_gateway_pids' psutil scan walks the whole machine and returns the live gateway PID, then the unmocked os.kill delivers the signal. Rather than audit and patch ~30 tests across cmd_update, kill_gateway_processes, and stop_profile_gateway code paths, install a single autouse guard in tests/conftest.py that blocks the two primitives that actually cause the damage: - os.kill rejects any PID outside the test process subtree with a hard RuntimeError so the offending test gets a stack trace instead of silently murdering the real gateway. - subprocess.run / Popen / call / check_call / check_output reject any 'systemctl <verb> hermes-gateway' invocation that would mutate the live unit. Read-only systemctl calls (status, show, list-units) still pass through. We intentionally do NOT stub find_gateway_pids / _scan_gateway_pids — tests of those functions themselves need the real implementation. Discovery without delivery is harmless; the os.kill + systemctl guards catch the actual damage path. Tests that legitimately need real signal delivery (e.g. PTY tests signalling their own child) opt out via @pytest.mark.live_system_guard_bypass. Validation: tests/hermes_cli/ + tests/cli/ + tests/gateway/ produce the same 17 failures with and without this guard (all pre-existing on main, unrelated to gateway-kill leaks). The live gateway survives the test run that previously SIGTERMed it.
804 lines
29 KiB
Python
804 lines
29 KiB
Python
"""Shared fixtures for the hermes-agent test suite.
|
|
|
|
Hermetic-test invariants enforced here (see AGENTS.md for rationale):
|
|
|
|
1. **No credential env vars.** All provider/credential-shaped env vars
|
|
(ending in _API_KEY, _TOKEN, _SECRET, _PASSWORD, _CREDENTIALS, etc.)
|
|
are unset before every test. Local developer keys cannot leak in.
|
|
2. **Isolated HERMES_HOME.** HERMES_HOME points to a per-test tempdir so
|
|
code reading ``~/.hermes/*`` via ``get_hermes_home()`` can't see the
|
|
real one. (We do NOT also redirect HOME — that broke subprocesses in
|
|
CI. Code using ``Path.home() / ".hermes"`` instead of the canonical
|
|
``get_hermes_home()`` is a bug to fix at the callsite.)
|
|
3. **Deterministic runtime.** TZ=UTC, LANG=C.UTF-8, PYTHONHASHSEED=0.
|
|
4. **No HERMES_SESSION_* inheritance** — the agent's current gateway
|
|
session must not leak into tests.
|
|
|
|
These invariants make the local test run match CI closely. Gaps that
|
|
remain (CPU count, xdist worker count) are addressed by the canonical
|
|
test runner at ``scripts/run_tests.sh``.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import re
|
|
import signal
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
# Ensure project root is importable
|
|
PROJECT_ROOT = Path(__file__).parent.parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
|
|
# ── Credential env-var filter ──────────────────────────────────────────────
|
|
#
|
|
# Any env var in the current process matching ONE of these patterns is
|
|
# unset for every test. Developers' local keys cannot leak into assertions
|
|
# about "auto-detect provider when key present".
|
|
|
|
_CREDENTIAL_SUFFIXES = (
|
|
"_API_KEY",
|
|
"_TOKEN",
|
|
"_SECRET",
|
|
"_PASSWORD",
|
|
"_CREDENTIALS",
|
|
"_ACCESS_KEY",
|
|
"_SECRET_ACCESS_KEY",
|
|
"_PRIVATE_KEY",
|
|
"_OAUTH_TOKEN",
|
|
"_WEBHOOK_SECRET",
|
|
"_ENCRYPT_KEY",
|
|
"_APP_SECRET",
|
|
"_CLIENT_SECRET",
|
|
"_CORP_SECRET",
|
|
"_AES_KEY",
|
|
)
|
|
|
|
# Explicit names (for ones that don't fit the suffix pattern)
|
|
_CREDENTIAL_NAMES = frozenset({
|
|
"AWS_ACCESS_KEY_ID",
|
|
"AWS_SECRET_ACCESS_KEY",
|
|
"AWS_SESSION_TOKEN",
|
|
"ANTHROPIC_TOKEN",
|
|
"FAL_KEY",
|
|
"GH_TOKEN",
|
|
"GITHUB_TOKEN",
|
|
"OPENAI_API_KEY",
|
|
"OPENROUTER_API_KEY",
|
|
"NOUS_API_KEY",
|
|
"GEMINI_API_KEY",
|
|
"GOOGLE_API_KEY",
|
|
"GROQ_API_KEY",
|
|
"XAI_API_KEY",
|
|
"MISTRAL_API_KEY",
|
|
"DEEPSEEK_API_KEY",
|
|
"KIMI_API_KEY",
|
|
"MOONSHOT_API_KEY",
|
|
"GLM_API_KEY",
|
|
"ZAI_API_KEY",
|
|
"MINIMAX_API_KEY",
|
|
"OLLAMA_API_KEY",
|
|
"OPENVIKING_API_KEY",
|
|
"COPILOT_API_KEY",
|
|
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
"BROWSERBASE_API_KEY",
|
|
"FIRECRAWL_API_KEY",
|
|
"PARALLEL_API_KEY",
|
|
"EXA_API_KEY",
|
|
"TAVILY_API_KEY",
|
|
"WANDB_API_KEY",
|
|
"ELEVENLABS_API_KEY",
|
|
"HONCHO_API_KEY",
|
|
"MEM0_API_KEY",
|
|
"SUPERMEMORY_API_KEY",
|
|
"RETAINDB_API_KEY",
|
|
"HINDSIGHT_API_KEY",
|
|
"HINDSIGHT_LLM_API_KEY",
|
|
"TINKER_API_KEY",
|
|
"DAYTONA_API_KEY",
|
|
"TWILIO_AUTH_TOKEN",
|
|
"TELEGRAM_BOT_TOKEN",
|
|
"DISCORD_BOT_TOKEN",
|
|
"SLACK_BOT_TOKEN",
|
|
"SLACK_APP_TOKEN",
|
|
"MATTERMOST_TOKEN",
|
|
"MATRIX_ACCESS_TOKEN",
|
|
"MATRIX_PASSWORD",
|
|
"MATRIX_RECOVERY_KEY",
|
|
"HASS_TOKEN",
|
|
"EMAIL_PASSWORD",
|
|
"BLUEBUBBLES_PASSWORD",
|
|
"FEISHU_APP_SECRET",
|
|
"FEISHU_ENCRYPT_KEY",
|
|
"FEISHU_VERIFICATION_TOKEN",
|
|
"DINGTALK_CLIENT_SECRET",
|
|
"QQ_CLIENT_SECRET",
|
|
"QQ_STT_API_KEY",
|
|
"WECOM_SECRET",
|
|
"WECOM_CALLBACK_CORP_SECRET",
|
|
"WECOM_CALLBACK_TOKEN",
|
|
"WECOM_CALLBACK_ENCODING_AES_KEY",
|
|
"WEIXIN_TOKEN",
|
|
"MODAL_TOKEN_ID",
|
|
"MODAL_TOKEN_SECRET",
|
|
"TERMINAL_SSH_KEY",
|
|
"SUDO_PASSWORD",
|
|
"GATEWAY_PROXY_KEY",
|
|
"API_SERVER_KEY",
|
|
"TOOL_GATEWAY_USER_TOKEN",
|
|
"TELEGRAM_WEBHOOK_SECRET",
|
|
"WEBHOOK_SECRET",
|
|
"AI_GATEWAY_API_KEY",
|
|
"VOICE_TOOLS_OPENAI_KEY",
|
|
"BROWSER_USE_API_KEY",
|
|
"CUSTOM_API_KEY",
|
|
"GATEWAY_PROXY_URL",
|
|
"GEMINI_BASE_URL",
|
|
"OPENAI_BASE_URL",
|
|
"OPENROUTER_BASE_URL",
|
|
"OLLAMA_BASE_URL",
|
|
"GROQ_BASE_URL",
|
|
"XAI_BASE_URL",
|
|
"AI_GATEWAY_BASE_URL",
|
|
"ANTHROPIC_BASE_URL",
|
|
})
|
|
|
|
|
|
def _looks_like_credential(name: str) -> bool:
|
|
"""True if env var name matches a credential-shaped pattern."""
|
|
if name in _CREDENTIAL_NAMES:
|
|
return True
|
|
return any(name.endswith(suf) for suf in _CREDENTIAL_SUFFIXES)
|
|
|
|
|
|
# HERMES_* vars that change test behavior by being set. Unset all of these
|
|
# unconditionally — individual tests that need them set do so explicitly.
|
|
_HERMES_BEHAVIORAL_VARS = frozenset({
|
|
"HERMES_YOLO_MODE",
|
|
"HERMES_INTERACTIVE",
|
|
"HERMES_QUIET",
|
|
"HERMES_TOOL_PROGRESS",
|
|
"HERMES_TOOL_PROGRESS_MODE",
|
|
"HERMES_MAX_ITERATIONS",
|
|
"HERMES_SESSION_PLATFORM",
|
|
"HERMES_SESSION_CHAT_ID",
|
|
"HERMES_SESSION_CHAT_NAME",
|
|
"HERMES_SESSION_THREAD_ID",
|
|
"HERMES_SESSION_SOURCE",
|
|
"HERMES_SESSION_KEY",
|
|
"HERMES_GATEWAY_SESSION",
|
|
"HERMES_PLATFORM",
|
|
"HERMES_MODEL",
|
|
"HERMES_INFERENCE_MODEL",
|
|
"HERMES_INFERENCE_PROVIDER",
|
|
"HERMES_TUI_PROVIDER",
|
|
"HERMES_MANAGED",
|
|
"HERMES_DEV",
|
|
"HERMES_CONTAINER",
|
|
"HERMES_EPHEMERAL_SYSTEM_PROMPT",
|
|
"HERMES_TIMEZONE",
|
|
"HERMES_REDACT_SECRETS",
|
|
"HERMES_BACKGROUND_NOTIFICATIONS",
|
|
"HERMES_EXEC_ASK",
|
|
"HERMES_HOME_MODE",
|
|
# Kanban path/board pins must never leak from a developer shell or
|
|
# dispatched worker into tests; otherwise tests can write fake tasks to
|
|
# the real ~/.hermes/kanban.db instead of the per-test HERMES_HOME.
|
|
"HERMES_KANBAN_DB",
|
|
"HERMES_KANBAN_BOARD",
|
|
"HERMES_KANBAN_WORKSPACES_ROOT",
|
|
"HERMES_KANBAN_LOGS_ROOT",
|
|
"HERMES_KANBAN_TASK",
|
|
"HERMES_KANBAN_WORKSPACE",
|
|
"HERMES_TENANT",
|
|
"TERMINAL_CWD",
|
|
"TERMINAL_ENV",
|
|
"TERMINAL_VERCEL_RUNTIME",
|
|
"TERMINAL_CONTAINER_CPU",
|
|
"TERMINAL_CONTAINER_DISK",
|
|
"TERMINAL_CONTAINER_MEMORY",
|
|
"TERMINAL_CONTAINER_PERSISTENT",
|
|
"TERMINAL_DOCKER_RUN_AS_HOST_USER",
|
|
"BROWSER_CDP_URL",
|
|
"CAMOFOX_URL",
|
|
# Platform allowlists — not credentials, but if set from any source
|
|
# (user shell, earlier leaky test, CI env), they change gateway auth
|
|
# behavior and flake button-authorization tests.
|
|
"TELEGRAM_ALLOWED_USERS",
|
|
"DISCORD_ALLOWED_USERS",
|
|
"WHATSAPP_ALLOWED_USERS",
|
|
"SLACK_ALLOWED_USERS",
|
|
"SIGNAL_ALLOWED_USERS",
|
|
"SIGNAL_GROUP_ALLOWED_USERS",
|
|
"EMAIL_ALLOWED_USERS",
|
|
"SMS_ALLOWED_USERS",
|
|
"MATTERMOST_ALLOWED_USERS",
|
|
"MATRIX_ALLOWED_USERS",
|
|
"DINGTALK_ALLOWED_USERS",
|
|
"FEISHU_ALLOWED_USERS",
|
|
"WECOM_ALLOWED_USERS",
|
|
"GATEWAY_ALLOWED_USERS",
|
|
"GATEWAY_ALLOW_ALL_USERS",
|
|
"TELEGRAM_ALLOW_ALL_USERS",
|
|
"DISCORD_ALLOW_ALL_USERS",
|
|
"WHATSAPP_ALLOW_ALL_USERS",
|
|
"SLACK_ALLOW_ALL_USERS",
|
|
"SIGNAL_ALLOW_ALL_USERS",
|
|
"EMAIL_ALLOW_ALL_USERS",
|
|
"SMS_ALLOW_ALL_USERS",
|
|
# Gateway home channels are set by /sethome in real profiles. Tests that
|
|
# exercise dashboard notification toggles must opt in explicitly or they
|
|
# can accidentally subscribe against a developer's real home channel.
|
|
"TELEGRAM_HOME_CHANNEL",
|
|
"TELEGRAM_HOME_CHANNEL_THREAD_ID",
|
|
"TELEGRAM_HOME_CHANNEL_NAME",
|
|
"DISCORD_HOME_CHANNEL",
|
|
"DISCORD_HOME_CHANNEL_THREAD_ID",
|
|
"DISCORD_HOME_CHANNEL_NAME",
|
|
"SLACK_HOME_CHANNEL",
|
|
"SLACK_HOME_CHANNEL_THREAD_ID",
|
|
"SLACK_HOME_CHANNEL_NAME",
|
|
"WHATSAPP_HOME_CHANNEL",
|
|
"WHATSAPP_HOME_CHANNEL_THREAD_ID",
|
|
"WHATSAPP_HOME_CHANNEL_NAME",
|
|
"SIGNAL_HOME_CHANNEL",
|
|
"SIGNAL_HOME_CHANNEL_THREAD_ID",
|
|
"SIGNAL_HOME_CHANNEL_NAME",
|
|
"EMAIL_HOME_CHANNEL",
|
|
"EMAIL_HOME_CHANNEL_THREAD_ID",
|
|
"EMAIL_HOME_CHANNEL_NAME",
|
|
"SMS_HOME_CHANNEL",
|
|
"SMS_HOME_CHANNEL_THREAD_ID",
|
|
"SMS_HOME_CHANNEL_NAME",
|
|
"MATTERMOST_HOME_CHANNEL",
|
|
"MATTERMOST_HOME_CHANNEL_THREAD_ID",
|
|
"MATTERMOST_HOME_CHANNEL_NAME",
|
|
"MATRIX_HOME_CHANNEL",
|
|
"MATRIX_HOME_CHANNEL_THREAD_ID",
|
|
"MATRIX_HOME_CHANNEL_NAME",
|
|
"DINGTALK_HOME_CHANNEL",
|
|
"DINGTALK_HOME_CHANNEL_THREAD_ID",
|
|
"DINGTALK_HOME_CHANNEL_NAME",
|
|
"FEISHU_HOME_CHANNEL",
|
|
"FEISHU_HOME_CHANNEL_THREAD_ID",
|
|
"FEISHU_HOME_CHANNEL_NAME",
|
|
"WECOM_HOME_CHANNEL",
|
|
"WECOM_HOME_CHANNEL_THREAD_ID",
|
|
"WECOM_HOME_CHANNEL_NAME",
|
|
# Platform gating — set by load_gateway_config() as a side effect when
|
|
# a config.yaml is present, so individual test bodies that call the
|
|
# loader leak these values into later tests on the same xdist worker.
|
|
# Force-clear on every test setup so the leak can't happen.
|
|
"SLACK_REQUIRE_MENTION",
|
|
"SLACK_STRICT_MENTION",
|
|
"SLACK_FREE_RESPONSE_CHANNELS",
|
|
"SLACK_ALLOW_BOTS",
|
|
"SLACK_REACTIONS",
|
|
"DISCORD_REQUIRE_MENTION",
|
|
"DISCORD_FREE_RESPONSE_CHANNELS",
|
|
"TELEGRAM_REQUIRE_MENTION",
|
|
"WHATSAPP_REQUIRE_MENTION",
|
|
"DINGTALK_REQUIRE_MENTION",
|
|
"MATRIX_REQUIRE_MENTION",
|
|
})
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _hermetic_environment(tmp_path, monkeypatch):
|
|
"""Blank out all credential/behavioral env vars so local and CI match.
|
|
|
|
Also redirects HOME and HERMES_HOME to per-test tempdirs so code that
|
|
reads ``~/.hermes/*`` can't touch the real one, and pins TZ/LANG so
|
|
datetime/locale-sensitive tests are deterministic.
|
|
"""
|
|
# 1. Blank every credential-shaped env var that's currently set.
|
|
for name in list(os.environ.keys()):
|
|
if _looks_like_credential(name):
|
|
monkeypatch.delenv(name, raising=False)
|
|
|
|
# 2. Blank behavioral HERMES_* vars that could change test semantics.
|
|
for name in _HERMES_BEHAVIORAL_VARS:
|
|
monkeypatch.delenv(name, raising=False)
|
|
|
|
# 3. Redirect HERMES_HOME to a per-test tempdir. Code that reads
|
|
# ``~/.hermes/*`` via ``get_hermes_home()`` now gets the tempdir.
|
|
#
|
|
# NOTE: We do NOT also redirect HOME. Doing so broke CI because
|
|
# some tests (and their transitive deps) spawn subprocesses that
|
|
# inherit HOME and expect it to be stable. If a test genuinely
|
|
# needs HOME isolated, it should set it explicitly in its own
|
|
# fixture. Any code in the codebase reading ``~/.hermes/*`` via
|
|
# ``Path.home() / ".hermes"`` instead of ``get_hermes_home()``
|
|
# is a bug to fix at the callsite.
|
|
fake_hermes_home = tmp_path / "hermes_test"
|
|
fake_hermes_home.mkdir()
|
|
(fake_hermes_home / "sessions").mkdir()
|
|
(fake_hermes_home / "cron").mkdir()
|
|
(fake_hermes_home / "memories").mkdir()
|
|
(fake_hermes_home / "skills").mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(fake_hermes_home))
|
|
|
|
# 4. Deterministic locale / timezone / hashseed. CI runs in UTC with
|
|
# C.UTF-8 locale; local dev often doesn't. Pin everything.
|
|
monkeypatch.setenv("TZ", "UTC")
|
|
monkeypatch.setenv("LANG", "C.UTF-8")
|
|
monkeypatch.setenv("LC_ALL", "C.UTF-8")
|
|
monkeypatch.setenv("PYTHONHASHSEED", "0")
|
|
|
|
# 4b. Disable AWS IMDS lookups. Without this, any test that ends up
|
|
# calling has_aws_credentials() / resolve_aws_auth_env_var()
|
|
# (e.g. provider auto-detect, status command, cron run_job) burns
|
|
# ~2s waiting for the metadata service at 169.254.169.254 to time
|
|
# out. Tests don't run on EC2 — IMDS is always unreachable here.
|
|
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
|
|
monkeypatch.setenv("AWS_METADATA_SERVICE_TIMEOUT", "1")
|
|
monkeypatch.setenv("AWS_METADATA_SERVICE_NUM_ATTEMPTS", "1")
|
|
|
|
# 5. Reset plugin singleton so tests don't leak plugins from
|
|
# ~/.hermes/plugins/ (which, per step 3, is now empty — but the
|
|
# singleton might still be cached from a previous test).
|
|
try:
|
|
import hermes_cli.plugins as _plugins_mod
|
|
monkeypatch.setattr(_plugins_mod, "_plugin_manager", None)
|
|
except Exception:
|
|
pass
|
|
# Explicitly clear provider-specific base URL overrides that don't match
|
|
# the generic credential-shaped env-var filter above.
|
|
monkeypatch.delenv("GMI_API_KEY", raising=False)
|
|
monkeypatch.delenv("GMI_BASE_URL", raising=False)
|
|
|
|
|
|
# Backward-compat alias — old tests reference this fixture name. Keep it
|
|
# as a no-op wrapper so imports don't break.
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_hermes_home(_hermetic_environment):
|
|
"""Alias preserved for any test that yields this name explicitly."""
|
|
return None
|
|
|
|
|
|
# ── Module-level state reset ───────────────────────────────────────────────
|
|
#
|
|
# Python modules are singletons per process, and pytest-xdist workers are
|
|
# long-lived. Module-level dicts/sets (tool registries, approval state,
|
|
# interrupt flags) and ContextVars persist across tests in the same worker,
|
|
# causing tests that pass alone to fail when run with siblings.
|
|
#
|
|
# Each entry in this fixture clears state that belongs to a specific module.
|
|
# New state buckets go here too — this is the single gate that prevents
|
|
# "works alone, flakes in CI" bugs from state leakage.
|
|
#
|
|
# The skill `test-suite-cascade-diagnosis` documents the concrete patterns
|
|
# this closes; the running example was `test_command_guards` failing 12/15
|
|
# CI runs because ``tools.approval._session_approved`` carried approvals
|
|
# from one test's session into another's.
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_module_state():
|
|
"""Clear module-level mutable state and ContextVars between tests.
|
|
|
|
Keeps state from leaking across tests on the same xdist worker. Modules
|
|
that don't exist yet (test collection before production import) are
|
|
skipped silently — production import later creates fresh empty state.
|
|
"""
|
|
# --- logging — quiet/one-shot paths mutate process-global logger state ---
|
|
logging.disable(logging.NOTSET)
|
|
for _logger_name in ("tools", "run_agent", "trajectory_compressor", "cron", "hermes_cli"):
|
|
_logger = logging.getLogger(_logger_name)
|
|
_logger.disabled = False
|
|
_logger.setLevel(logging.NOTSET)
|
|
_logger.propagate = True
|
|
|
|
# --- tools.approval — the single biggest source of cross-test pollution ---
|
|
try:
|
|
from tools import approval as _approval_mod
|
|
_approval_mod._session_approved.clear()
|
|
_approval_mod._session_yolo.clear()
|
|
_approval_mod._permanent_approved.clear()
|
|
_approval_mod._pending.clear()
|
|
_approval_mod._gateway_queues.clear()
|
|
_approval_mod._gateway_notify_cbs.clear()
|
|
# ContextVar: reset to empty string so get_current_session_key()
|
|
# falls through to the env var / default path, matching a fresh
|
|
# process.
|
|
_approval_mod._approval_session_key.set("")
|
|
except Exception:
|
|
pass
|
|
|
|
# --- tools.interrupt — per-thread interrupt flag set ---
|
|
try:
|
|
from tools import interrupt as _interrupt_mod
|
|
with _interrupt_mod._lock:
|
|
_interrupt_mod._interrupted_threads.clear()
|
|
except Exception:
|
|
pass
|
|
|
|
# --- gateway.session_context — 9 ContextVars that represent
|
|
# the active gateway session. If set in one test and not reset,
|
|
# the next test's get_session_env() reads stale values.
|
|
try:
|
|
from gateway import session_context as _sc_mod
|
|
for _cv in (
|
|
_sc_mod._SESSION_PLATFORM,
|
|
_sc_mod._SESSION_CHAT_ID,
|
|
_sc_mod._SESSION_CHAT_NAME,
|
|
_sc_mod._SESSION_THREAD_ID,
|
|
_sc_mod._SESSION_USER_ID,
|
|
_sc_mod._SESSION_USER_NAME,
|
|
_sc_mod._SESSION_KEY,
|
|
_sc_mod._CRON_AUTO_DELIVER_PLATFORM,
|
|
_sc_mod._CRON_AUTO_DELIVER_CHAT_ID,
|
|
_sc_mod._CRON_AUTO_DELIVER_THREAD_ID,
|
|
):
|
|
_cv.set(_sc_mod._UNSET)
|
|
except Exception:
|
|
pass
|
|
|
|
# --- tools.env_passthrough — ContextVar<set[str]> with no default ---
|
|
# LookupError is normal if the test never set it. Setting it to an
|
|
# empty set unconditionally normalizes the starting state.
|
|
try:
|
|
from tools import env_passthrough as _envp_mod
|
|
_envp_mod._allowed_env_vars_var.set(set())
|
|
except Exception:
|
|
pass
|
|
|
|
# --- tools.terminal_tool — active environment/cwd cache ---
|
|
# File tools prefer a live terminal cwd when one is cached for the task.
|
|
# Clear terminal environments between tests so a prior terminal call can't
|
|
# override TERMINAL_CWD in path-resolution tests.
|
|
try:
|
|
from tools import terminal_tool as _term_mod
|
|
_envs_to_cleanup = []
|
|
with _term_mod._env_lock:
|
|
_envs_to_cleanup = list(_term_mod._active_environments.values())
|
|
_term_mod._active_environments.clear()
|
|
_term_mod._last_activity.clear()
|
|
_term_mod._creation_locks.clear()
|
|
for _env in _envs_to_cleanup:
|
|
try:
|
|
_env.cleanup()
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
# --- tools.credential_files — ContextVar<dict> ---
|
|
try:
|
|
from tools import credential_files as _credf_mod
|
|
_credf_mod._registered_files_var.set({})
|
|
except Exception:
|
|
pass
|
|
|
|
# --- agent.auxiliary_client — runtime main provider/model override ---
|
|
# Set per-turn by AIAgent.run_conversation; tests that import it must
|
|
# see a clean state so config.yaml fallback works as expected.
|
|
try:
|
|
from agent import auxiliary_client as _aux_mod
|
|
_aux_mod.clear_runtime_main()
|
|
except Exception:
|
|
pass
|
|
|
|
# --- tools.file_tools — per-task read history + file-ops cache ---
|
|
# _read_tracker accumulates per-task_id read history for loop detection,
|
|
# capped by _READ_HISTORY_CAP. If entries from a prior test persist, the
|
|
# cap is hit faster than expected and capacity-related tests flake.
|
|
try:
|
|
from tools import file_tools as _ft_mod
|
|
with _ft_mod._read_tracker_lock:
|
|
_ft_mod._read_tracker.clear()
|
|
with _ft_mod._file_ops_lock:
|
|
_ft_mod._file_ops_cache.clear()
|
|
except Exception:
|
|
pass
|
|
|
|
yield
|
|
|
|
|
|
@pytest.fixture()
|
|
def tmp_dir(tmp_path):
|
|
"""Provide a temporary directory that is cleaned up automatically."""
|
|
return tmp_path
|
|
|
|
|
|
@pytest.fixture()
|
|
def mock_config():
|
|
"""Return a minimal hermes config dict suitable for unit tests."""
|
|
return {
|
|
"model": "test/mock-model",
|
|
"toolsets": ["terminal", "file"],
|
|
"max_turns": 10,
|
|
"terminal": {
|
|
"backend": "local",
|
|
"cwd": "/tmp",
|
|
"timeout": 30,
|
|
},
|
|
"compression": {"enabled": False},
|
|
"memory": {"memory_enabled": False, "user_profile_enabled": False},
|
|
"command_allowlist": [],
|
|
}
|
|
|
|
|
|
# ── Global test timeout ─────────────────────────────────────────────────────
|
|
# Kill any individual test that takes longer than 30 seconds.
|
|
# Prevents hanging tests (subprocess spawns, blocking I/O) from stalling the
|
|
# entire test suite.
|
|
|
|
def _timeout_handler(signum, frame):
|
|
raise TimeoutError("Test exceeded 30 second timeout")
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _ensure_current_event_loop(request):
|
|
"""Provide a default event loop for sync tests that call get_event_loop().
|
|
|
|
Python 3.11+ no longer guarantees a current loop for plain synchronous tests.
|
|
A number of gateway tests still use asyncio.get_event_loop().run_until_complete(...).
|
|
Ensure they always have a usable loop without interfering with pytest-asyncio's
|
|
own loop management for @pytest.mark.asyncio tests.
|
|
|
|
On Python 3.12+, ``asyncio.get_event_loop_policy().get_event_loop()`` with no
|
|
*running* loop emits DeprecationWarning; skip that path and install a fresh
|
|
loop via ``new_event_loop()`` instead.
|
|
"""
|
|
if request.node.get_closest_marker("asyncio") is not None:
|
|
yield
|
|
return
|
|
|
|
loop = None
|
|
try:
|
|
loop = asyncio.get_running_loop()
|
|
except RuntimeError:
|
|
pass
|
|
|
|
if loop is None and sys.version_info < (3, 12):
|
|
try:
|
|
loop = asyncio.get_event_loop_policy().get_event_loop()
|
|
except RuntimeError:
|
|
loop = None
|
|
|
|
created = loop is None or loop.is_closed()
|
|
if created:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
if created and loop is not None:
|
|
try:
|
|
loop.close()
|
|
finally:
|
|
asyncio.set_event_loop(None)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _enforce_test_timeout():
|
|
"""Kill any individual test that takes longer than 30 seconds.
|
|
SIGALRM is Unix-only; skip on Windows."""
|
|
if sys.platform == "win32":
|
|
yield
|
|
return
|
|
old = signal.signal(signal.SIGALRM, _timeout_handler)
|
|
signal.alarm(30)
|
|
yield
|
|
signal.alarm(0)
|
|
signal.signal(signal.SIGALRM, old)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_tool_registry_caches():
|
|
"""Clear tool-registry-level caches between tests.
|
|
|
|
The production registry caches ``check_fn()`` results for 30 s
|
|
(see tools/registry.py) and :func:`get_tool_definitions` memoizes
|
|
its result (see model_tools.py). Both are keyed on state that tests
|
|
routinely mutate (env vars, registry._generation, config.yaml mtime)
|
|
— but a stale result from test A can still be served to test B
|
|
because 30 s covers the entire suite, and xdist worker reuse means
|
|
one test's cache lands in another's process. Clearing before every
|
|
test keeps hermetic behavior.
|
|
"""
|
|
try:
|
|
from tools.registry import invalidate_check_fn_cache
|
|
invalidate_check_fn_cache()
|
|
except ImportError:
|
|
pass
|
|
try:
|
|
from model_tools import _clear_tool_defs_cache
|
|
_clear_tool_defs_cache()
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
# ── Live-system guard ──────────────────────────────────────────────────────
|
|
#
|
|
# Several test files exercise the gateway-restart / kill code paths
|
|
# (``cmd_update``, ``kill_gateway_processes``, ``stop_profile_gateway``).
|
|
# When a single test forgets to mock either ``os.kill`` or the global
|
|
# ``find_gateway_pids`` helper, the real call leaks out of the hermetic
|
|
# environment and finds the developer's live ``hermes-gateway`` process
|
|
# via ``psutil`` — sending it SIGTERM mid-test. The shutdown forensics in
|
|
# PR #23285 caught this happening 5+ times in 3 days, every time
|
|
# correlated with a ``tests/hermes_cli/`` pytest run starting up.
|
|
#
|
|
# This fixture makes the leak impossible by intercepting the two
|
|
# primitives that actually do damage:
|
|
#
|
|
# • ``os.kill`` rejects any PID outside the test process subtree with
|
|
# a hard ``RuntimeError`` so the offending test gets a stack trace
|
|
# instead of silently murdering the real gateway.
|
|
# • ``subprocess.run`` / ``subprocess.Popen`` / ``call`` / ``check_call`` /
|
|
# ``check_output`` reject any ``systemctl ... <verb> hermes-gateway``
|
|
# invocation that would mutate the live unit. Read-only systemctl
|
|
# calls (``status``, ``show``, ``list-units``) still pass through.
|
|
#
|
|
# We intentionally do NOT stub ``find_gateway_pids`` / ``_scan_gateway_pids``
|
|
# here — tests of those functions themselves need the real implementation.
|
|
# Even if a test gets the live gateway PID back from a real scan, the
|
|
# ``os.kill`` guard above catches the actual signal call, and the
|
|
# ``systemctl`` guard catches the systemd path. Discovery without
|
|
# delivery is harmless.
|
|
|
|
_LIVE_SYSTEM_GUARD_BYPASS_MARK = "live_system_guard_bypass"
|
|
|
|
|
|
def pytest_configure(config): # noqa: D401 — pytest hook
|
|
"""Register markers used by hermetic conftest."""
|
|
config.addinivalue_line(
|
|
"markers",
|
|
f"{_LIVE_SYSTEM_GUARD_BYPASS_MARK}: bypass the live-system guard "
|
|
"(only for tests that genuinely need real os.kill / subprocess "
|
|
"behaviour — e.g. PTY tests that signal their own child).",
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _live_system_guard(request, monkeypatch):
|
|
"""Block real os.kill / systemctl / gateway-pid scans during tests.
|
|
|
|
See block comment above for the why. Tests that genuinely need
|
|
real signal delivery (e.g. PTY tests that SIGINT their own child)
|
|
can opt out with ``@pytest.mark.live_system_guard_bypass``.
|
|
"""
|
|
if request.node.get_closest_marker(_LIVE_SYSTEM_GUARD_BYPASS_MARK):
|
|
yield
|
|
return
|
|
|
|
import os as _os
|
|
import subprocess as _subprocess
|
|
|
|
test_pid = _os.getpid()
|
|
# Capture the test process's existing children at fixture start —
|
|
# any *new* children spawned by the test are also allowlisted via
|
|
# the live psutil walk below. Static set keeps the fast path cheap.
|
|
try:
|
|
import psutil as _psutil
|
|
_initial_children = {
|
|
c.pid for c in _psutil.Process(test_pid).children(recursive=True)
|
|
}
|
|
except Exception:
|
|
_psutil = None
|
|
_initial_children = set()
|
|
|
|
def _is_own_subtree(pid: int) -> bool:
|
|
if pid == test_pid or pid in _initial_children:
|
|
return True
|
|
if _psutil is None:
|
|
return False
|
|
try:
|
|
walker = _psutil.Process(pid)
|
|
except Exception:
|
|
# Stale PID — kill would be a no-op anyway, allow it.
|
|
return True
|
|
try:
|
|
for parent in walker.parents():
|
|
if parent.pid == test_pid:
|
|
return True
|
|
except Exception:
|
|
return False
|
|
return False
|
|
|
|
real_kill = _os.kill
|
|
|
|
def _guarded_kill(pid, sig, *args, **kwargs):
|
|
if _is_own_subtree(int(pid)):
|
|
return real_kill(pid, sig, *args, **kwargs)
|
|
raise RuntimeError(
|
|
f"tests/conftest.py live-system guard: blocked os.kill("
|
|
f"{pid}, {sig}) — PID is outside the test process subtree. "
|
|
"If this fired in CI it means the test reached a real "
|
|
"kill_gateway_processes / stop_profile_gateway / cmd_update "
|
|
"code path without mocking find_gateway_pids and os.kill. "
|
|
"Mock both, or mark the test with "
|
|
"@pytest.mark.live_system_guard_bypass if real signal "
|
|
"delivery is genuinely required."
|
|
)
|
|
|
|
monkeypatch.setattr(_os, "kill", _guarded_kill)
|
|
|
|
# ``os.killpg`` is the same risk class — sends a signal to every
|
|
# process in a group. The gateway is a session leader (its own
|
|
# PGID == its PID), so killpg(gateway_pid, SIGTERM) is a one-shot
|
|
# kill of the live process. Allow it only when the target PGID is
|
|
# the test process's own group.
|
|
if hasattr(_os, "killpg"):
|
|
real_killpg = _os.killpg
|
|
own_pgid = _os.getpgrp()
|
|
|
|
def _guarded_killpg(pgid, sig, *args, **kwargs):
|
|
if int(pgid) == own_pgid or _is_own_subtree(int(pgid)):
|
|
return real_killpg(pgid, sig, *args, **kwargs)
|
|
raise RuntimeError(
|
|
f"tests/conftest.py live-system guard: blocked "
|
|
f"os.killpg({pgid}, {sig}) — PGID is outside the test "
|
|
"process group. See _live_system_guard for the why."
|
|
)
|
|
|
|
monkeypatch.setattr(_os, "killpg", _guarded_killpg)
|
|
|
|
real_run = _subprocess.run
|
|
real_popen = _subprocess.Popen
|
|
real_call = _subprocess.call
|
|
real_check_call = _subprocess.check_call
|
|
real_check_output = _subprocess.check_output
|
|
|
|
def _is_blocked_systemctl(cmd) -> bool:
|
|
if isinstance(cmd, (list, tuple)):
|
|
tokens = [str(t) for t in cmd]
|
|
elif isinstance(cmd, (bytes, bytearray)):
|
|
tokens = bytes(cmd).decode(errors="replace").split()
|
|
elif isinstance(cmd, str):
|
|
tokens = cmd.split()
|
|
else:
|
|
return False
|
|
if not tokens:
|
|
return False
|
|
head = tokens[0].rsplit("/", 1)[-1]
|
|
if head != "systemctl":
|
|
return False
|
|
joined = " ".join(tokens).lower()
|
|
# Block any systemctl invocation that targets a hermes-gateway
|
|
# unit AND would change its run state. Status reads, list-units,
|
|
# and show calls remain allowed because they're side-effect-free.
|
|
if "hermes-gateway" not in joined and "hermes.service" not in joined:
|
|
return False
|
|
mutating = (
|
|
"restart", "start", "stop", "kill", "reload",
|
|
"reset-failed", "enable", "disable", "mask", "unmask",
|
|
"daemon-reload",
|
|
)
|
|
return any(verb in tokens for verb in mutating)
|
|
|
|
def _wrap_subprocess(name, real):
|
|
def _guarded(cmd, *args, **kwargs):
|
|
if _is_blocked_systemctl(cmd):
|
|
raise RuntimeError(
|
|
f"tests/conftest.py live-system guard: blocked "
|
|
f"subprocess.{name}({cmd!r}) — would mutate the "
|
|
"live hermes-gateway systemd unit. Mock "
|
|
"subprocess.run / _run_systemctl in the test, or "
|
|
"mark with @pytest.mark.live_system_guard_bypass."
|
|
)
|
|
return real(cmd, *args, **kwargs)
|
|
_guarded.__name__ = f"_guarded_{name}"
|
|
return _guarded
|
|
|
|
monkeypatch.setattr(_subprocess, "run", _wrap_subprocess("run", real_run))
|
|
monkeypatch.setattr(_subprocess, "Popen", _wrap_subprocess("Popen", real_popen))
|
|
monkeypatch.setattr(_subprocess, "call", _wrap_subprocess("call", real_call))
|
|
monkeypatch.setattr(
|
|
_subprocess, "check_call", _wrap_subprocess("check_call", real_check_call)
|
|
)
|
|
monkeypatch.setattr(
|
|
_subprocess,
|
|
"check_output",
|
|
_wrap_subprocess("check_output", real_check_output),
|
|
)
|
|
|
|
yield
|