mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-08 13:12:08 +00:00
fix(gateway,cron): guard cron model-tool path + add auto-resume loop breaker (#30719)
Completes the #30719 restart-loop defenses. Defenses 1-2 (the _HERMES_GATEWAY guard on `hermes gateway stop|restart` + terminal_tool, and the cron-creation lifecycle filter) already landed on main, but two gaps remained: - The agent's `cronjob` model tool calls cron.jobs.create_job directly, bypassing the hermes_cli.cron.cron_create CLI filter, so lifecycle commands scheduled via the model tool were only blocked at execution time (terminal_tool), not at creation. Moved the filter to a shared cron/lifecycle_guard.py enforced at create_job — the single chokepoint every job-creation path hits (CLI + model tool). Re-exported _contains_gateway_lifecycle_command from hermes_cli.cron so terminal_tool's import keeps working. - No breaker for the auto-resume loop itself. Defenses 1-2 cover the cron/CLI/terminal paths, but any other SIGTERM source (e.g. a raw terminal("launchctl kickstart ai.hermes.gateway")) still triggers the boot->auto-resume->re-run cycle. Added gateway/restart_loop_guard.py: counts restart-interrupted boots in a rolling window (config gateway.restart_loop_guard, default 3 boots / 60s) and skips auto-resume for that boot once tripped. The gateway still comes up and serves real inbound messages; it just stops replaying the session that keeps killing it, putting a human back in the loop. Also tightened the lifecycle regex over main's version: dropped `hermes gateway start` (benign), required the gateway identifier on the launchctl/systemctl branches (so `launchctl unload ai.hermes.update-checker.plist` and `systemctl restart hermes-meta.service` no longer false-positive), added the inverse pkill token order, and fixed the binary-script bypass (decode with errors='replace' instead of swallowing UnicodeDecodeError). The create_job guard resolves relative script paths under HERMES_HOME/scripts the same way the scheduler does, so a bare script name is scanned as the file that actually runs. Design and much of defense-2 originate from PR #33395 (@kshitijk4poor), which itself salvaged #30728 (@SimoKiihamaki). Rebuilt against current main since defenses 1-2 had already landed under different names. Closes #30719. Co-authored-by: SimoKiihamaki <simo.kiihamaki@gmail.com> Co-authored-by: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com>
This commit is contained in:
teknium1
Teknium
parent
c71f816956
commit
b48cacb97b
7 changed files with 555 additions and 46 deletions
|
|
@ -960,6 +960,15 @@ def create_job(
|
|||
context_from = None
|
||||
|
||||
prompt_text = _coerce_job_text(prompt)
|
||||
|
||||
# Reject cron jobs that schedule gateway-lifecycle commands. Prevents
|
||||
# agent-driven SIGTERM-respawn loops under launchd/systemd KeepAlive
|
||||
# (#30719). Enforced here (not only in the CLI layer) so the agent's
|
||||
# `cronjob` model tool — which calls create_job directly — is also
|
||||
# covered, not just `hermes cron create`.
|
||||
from cron.lifecycle_guard import check_gateway_lifecycle
|
||||
check_gateway_lifecycle(prompt_text, normalized_script)
|
||||
|
||||
label_source = (prompt_text or (normalized_skills[0] if normalized_skills else None) or (normalized_script if normalized_no_agent else None)) or "cron job"
|
||||
|
||||
provider_snapshot, model_snapshot = _compute_provider_model_snapshots(
|
||||
|
|
|
|||
141
cron/lifecycle_guard.py
Normal file
141
cron/lifecycle_guard.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
"""Gateway lifecycle guard for cron job creation (#30719).
|
||||
|
||||
An agent running inside a gateway can schedule a cron job that calls
|
||||
``hermes gateway restart`` (or ``launchctl kickstart ai.hermes.gateway``
|
||||
or ``systemctl restart hermes-gateway``). When the cron fires, the
|
||||
gateway dies, the supervisor (launchd KeepAlive / systemd Restart=)
|
||||
revives it, auto-resume picks up the offending session, and the resumed
|
||||
turn re-runs the same logic — a SIGTERM-respawn loop every ~10 seconds
|
||||
until manually broken.
|
||||
|
||||
This module rejects cron job specs whose prompt or script contains a
|
||||
direct shell-level gateway-lifecycle command. It is enforced at
|
||||
``cron.jobs.create_job`` so it fires on every job-creation path: the
|
||||
``hermes cron create`` CLI subcommand AND the agent's ``cronjob`` model
|
||||
tool (which calls ``create_job`` directly, bypassing the CLI layer).
|
||||
|
||||
The pattern is intentionally command-shaped: it anchors on a concrete
|
||||
command identifier (``hermes gateway``, ``launchctl ... hermes-gateway``,
|
||||
``systemctl ... hermes-gateway``, ``pkill`` against the gateway) so it
|
||||
cannot fire on prose. A cron ``prompt`` is fed to a future LLM, not a
|
||||
shell, so an over-broad substring match on English ("Kong API gateway
|
||||
autoscaling and restart behavior") would produce a high false-positive
|
||||
rate without preventing the actual foot-gun, which requires a real
|
||||
command shape.
|
||||
|
||||
This is a defence-in-depth layer. ``tools/terminal_tool.py`` already
|
||||
blocks these commands at *execution* time when ``_HERMES_GATEWAY=1``, and
|
||||
``hermes gateway stop|restart`` refuse to self-target from inside the
|
||||
gateway. Blocking at *creation* time as well means the agent gets an
|
||||
immediate, informative rejection instead of scheduling a job that will
|
||||
only fail (silently) when it fires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class GatewayLifecycleBlocked(ValueError):
|
||||
"""Raised when a cron job spec contains a gateway-lifecycle command."""
|
||||
|
||||
|
||||
# Shell-level command shapes that target the gateway lifecycle. Each branch
|
||||
# is anchored on a concrete command identifier so a match can only fire on
|
||||
# actual shell-command-shaped strings, not on prose.
|
||||
_GATEWAY_LIFECYCLE_PATTERN = re.compile(
|
||||
r"(?i)"
|
||||
# Branch A: `hermes gateway restart|stop` — the canonical foot-gun.
|
||||
# `start` is intentionally excluded: starting a gateway from inside a
|
||||
# gateway is benign (a no-op or "already running" error), and a
|
||||
# legitimate cron job might start a sibling profile's gateway.
|
||||
r"(?:hermes\s+gateway\s+(?:restart|stop))"
|
||||
# Branch B: launchctl ops on a hermes-gateway label. macOS launchd
|
||||
# labels look like `ai.hermes.gateway` / `hermes-gateway`. Requiring the
|
||||
# gateway identifier prevents blocking unrelated hermes services (e.g.
|
||||
# `launchctl unload ai.hermes.update-checker.plist`).
|
||||
r"|(?:launchctl\s+(?:kickstart|unload|load|stop|restart)\b[^\n]*\bhermes[.\-]?gateway)"
|
||||
# Branch C: systemctl ops on a hermes-gateway unit.
|
||||
r"|(?:systemctl\s+(?:-\S+\s+)*(?:restart|stop|start)\b[^\n]*\bhermes[.\-]?gateway)"
|
||||
# Branch D: pkill / kill targeting the hermes gateway process. Both
|
||||
# token orders because real reproductions show both.
|
||||
r"|(?:p?kill\b[^\n]*\bhermes\b[^\n]*\bgateway)"
|
||||
r"|(?:p?kill\b[^\n]*\bgateway\b[^\n]*\bhermes)"
|
||||
)
|
||||
|
||||
|
||||
def contains_gateway_lifecycle_command(text: str) -> bool:
|
||||
"""Return True if *text* contains a gateway lifecycle command pattern."""
|
||||
if not text:
|
||||
return False
|
||||
return bool(_GATEWAY_LIFECYCLE_PATTERN.search(text))
|
||||
|
||||
|
||||
def _resolve_script_path(script_path: str) -> Path:
|
||||
"""Resolve a cron ``script`` value the same way the scheduler does.
|
||||
|
||||
The scheduler (``cron.scheduler``) resolves a bare/relative script path
|
||||
under ``<HERMES_HOME>/scripts/`` and only accepts absolute paths as-is.
|
||||
We MUST mirror that here so the guard scans the file that will actually
|
||||
run — otherwise a job whose script lives at the scheduler's real location
|
||||
(``~/.hermes/scripts/restart.sh``) but is passed as the bare name
|
||||
``restart.sh`` would read as a nonexistent relative path and silently
|
||||
scan prompt-only content, letting the command through.
|
||||
"""
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
raw = Path(script_path).expanduser()
|
||||
if raw.is_absolute():
|
||||
return raw
|
||||
return get_hermes_home() / "scripts" / raw
|
||||
|
||||
|
||||
def _read_script_for_scanning(script_path: str) -> str:
|
||||
"""Read a script file for lifecycle-pattern scanning.
|
||||
|
||||
Decodes with ``errors="replace"`` so binary or non-UTF-8 content does not
|
||||
silently bypass the check — a plain text-mode read raises
|
||||
``UnicodeDecodeError`` on such files, and swallowing that error would let
|
||||
an attacker hide the command in binary noise. Returns an empty string
|
||||
only when the file cannot be read at all.
|
||||
"""
|
||||
try:
|
||||
return _resolve_script_path(script_path).read_bytes().decode(
|
||||
"utf-8", errors="replace"
|
||||
)
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def check_gateway_lifecycle(
|
||||
prompt: Optional[str],
|
||||
script: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Raise ``GatewayLifecycleBlocked`` if *prompt* or *script* contains a
|
||||
gateway-lifecycle command pattern.
|
||||
|
||||
``prompt`` is scanned directly. ``script``, when supplied, is read from
|
||||
disk and concatenated for the scan. Both are considered together so a
|
||||
job cannot slip through by splitting the command across the prompt and
|
||||
the script.
|
||||
|
||||
Callers should let the exception propagate when they want the create to
|
||||
fail with a ``ValueError``-shaped error (the agent's ``cronjob`` tool
|
||||
surfaces this as a tool error; the CLI prints it in red and exits 1).
|
||||
"""
|
||||
combined = prompt or ""
|
||||
if script:
|
||||
script_text = _read_script_for_scanning(script)
|
||||
if script_text:
|
||||
combined = f"{combined}\n{script_text}"
|
||||
|
||||
if contains_gateway_lifecycle_command(combined):
|
||||
raise GatewayLifecycleBlocked(
|
||||
"Blocked: cron job contains a gateway lifecycle command "
|
||||
"(restart/stop/kill). This is blocked to prevent agent-driven "
|
||||
"SIGTERM-respawn loops under launchd/systemd supervision "
|
||||
"(#30719). Run `hermes gateway restart` from a shell outside "
|
||||
"the running gateway instead."
|
||||
)
|
||||
150
gateway/restart_loop_guard.py
Normal file
150
gateway/restart_loop_guard.py
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
"""Auto-resume restart-loop breaker (#30719, defense-3).
|
||||
|
||||
Defenses 1 and 2 (the ``_HERMES_GATEWAY`` guard on ``hermes gateway
|
||||
stop|restart`` + ``terminal_tool``, and the cron-creation lifecycle
|
||||
filter) stop the agent from scheduling its own restart via the cron and
|
||||
CLI paths. They do NOT cover every SIGTERM source: an agent running a
|
||||
raw ``terminal("launchctl kickstart -k gui/<uid>/ai.hermes.gateway")``,
|
||||
an external monitor with a bad trigger, or any other repeated crash can
|
||||
still drive the supervisor (launchd ``KeepAlive`` / systemd ``Restart=``)
|
||||
into a tight respawn loop. On each boot the gateway auto-resumes the
|
||||
restart-interrupted session, whose next turn re-runs the offending
|
||||
logic — SIGTERM every ~10 seconds until manually broken.
|
||||
|
||||
This module is the last-resort circuit breaker: it records a timestamp
|
||||
each time the gateway boots with restart-interrupted sessions pending,
|
||||
keeps a rolling window of recent boots persisted across processes (each
|
||||
boot is a fresh process, so in-memory state is useless), and reports the
|
||||
loop as "tripped" once too many such boots happen inside a short window.
|
||||
When tripped, the caller SKIPS auto-resume for that boot — the gateway
|
||||
still starts and serves real inbound messages, it just stops replaying
|
||||
the session that keeps killing it, which breaks the cycle and puts a
|
||||
human back in the loop.
|
||||
|
||||
State lives in ``<HERMES_HOME>/gateway/restart_loop.json`` so it is
|
||||
profile-scoped and survives process death. It is intentionally tiny and
|
||||
best-effort: any read/write failure fails OPEN (no false trip) because a
|
||||
broken breaker must never wedge a healthy gateway.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger("gateway.run")
|
||||
|
||||
# Defaults chosen so a legitimate operator restart (or two) never trips the
|
||||
# breaker, but the documented ~10s respawn loop does within a few cycles.
|
||||
DEFAULT_MAX_RESTARTS = 3
|
||||
DEFAULT_WINDOW_SECONDS = 60
|
||||
|
||||
|
||||
def _state_path():
|
||||
return get_hermes_home() / "gateway" / "restart_loop.json"
|
||||
|
||||
|
||||
def _load_boots() -> List[float]:
|
||||
try:
|
||||
raw = _state_path().read_text(encoding="utf-8")
|
||||
data = json.loads(raw)
|
||||
boots = data.get("boots", [])
|
||||
return [float(t) for t in boots if isinstance(t, (int, float))]
|
||||
except (OSError, ValueError, TypeError):
|
||||
return []
|
||||
|
||||
|
||||
def _save_boots(boots: List[float]) -> None:
|
||||
try:
|
||||
path = _state_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"boots": boots}), encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def record_restart_interrupted_boot(
|
||||
window_seconds: int = DEFAULT_WINDOW_SECONDS,
|
||||
*,
|
||||
now: Optional[float] = None,
|
||||
) -> List[float]:
|
||||
"""Record that the gateway just booted with restart-interrupted sessions.
|
||||
|
||||
Prunes boots older than ``window_seconds`` and appends the current time.
|
||||
Returns the pruned+appended list (most recent last). Best-effort — a
|
||||
persistence failure returns the in-memory list without raising.
|
||||
"""
|
||||
ts = time.time() if now is None else now
|
||||
cutoff = ts - max(1, window_seconds)
|
||||
boots = [t for t in _load_boots() if t >= cutoff]
|
||||
boots.append(ts)
|
||||
_save_boots(boots)
|
||||
return boots
|
||||
|
||||
|
||||
def is_restart_loop_tripped(
|
||||
max_restarts: int = DEFAULT_MAX_RESTARTS,
|
||||
window_seconds: int = DEFAULT_WINDOW_SECONDS,
|
||||
*,
|
||||
now: Optional[float] = None,
|
||||
) -> bool:
|
||||
"""Return True if the gateway has restarted ``>= max_restarts`` times with
|
||||
restart-interrupted sessions inside the last ``window_seconds``.
|
||||
|
||||
Reads the persisted boot log written by
|
||||
``record_restart_interrupted_boot`` and counts boots within the window.
|
||||
Fails OPEN (returns False) on any error — a broken breaker must never
|
||||
wedge a healthy gateway.
|
||||
"""
|
||||
if max_restarts <= 0:
|
||||
return False
|
||||
ts = time.time() if now is None else now
|
||||
cutoff = ts - max(1, window_seconds)
|
||||
try:
|
||||
recent = [t for t in _load_boots() if t >= cutoff]
|
||||
except Exception: # pragma: no cover — _load_boots already guards
|
||||
return False
|
||||
return len(recent) >= max_restarts
|
||||
|
||||
|
||||
def clear() -> None:
|
||||
"""Remove the persisted boot log (used on clean shutdown / by tests)."""
|
||||
try:
|
||||
_state_path().unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def check_and_record(
|
||||
max_restarts: int = DEFAULT_MAX_RESTARTS,
|
||||
window_seconds: int = DEFAULT_WINDOW_SECONDS,
|
||||
*,
|
||||
now: Optional[float] = None,
|
||||
) -> bool:
|
||||
"""Record this restart-interrupted boot and report whether the loop is now
|
||||
tripped.
|
||||
|
||||
This is the single entry point the gateway calls: it appends the current
|
||||
boot, then checks whether the (now-updated) window has reached the
|
||||
threshold. Returns True when auto-resume should be SKIPPED to break the
|
||||
loop.
|
||||
"""
|
||||
boots = record_restart_interrupted_boot(window_seconds, now=now)
|
||||
tripped = len(boots) >= max_restarts if max_restarts > 0 else False
|
||||
if tripped:
|
||||
logger.warning(
|
||||
"Restart-loop breaker TRIPPED: %d restart-interrupted gateway "
|
||||
"boots within %ds (threshold %d). Skipping auto-resume to break "
|
||||
"a suspected SIGTERM-respawn loop (#30719). Restart-interrupted "
|
||||
"sessions stay resume-pending and will continue on the next real "
|
||||
"user message. If this is a false positive, delete %s.",
|
||||
len(boots),
|
||||
window_seconds,
|
||||
max_restarts,
|
||||
_state_path(),
|
||||
)
|
||||
return tripped
|
||||
|
|
@ -3801,6 +3801,29 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
raw = None
|
||||
return parse_idle_timeout_seconds(raw)
|
||||
|
||||
def _restart_loop_guard_config(self) -> tuple:
|
||||
"""Return ``(max_restarts, window_seconds)`` for the auto-resume
|
||||
restart-loop breaker (#30719, defense-3), read from
|
||||
``gateway.restart_loop_guard`` in config.yaml with the module defaults
|
||||
as fallback. ``max_restarts <= 0`` disables the breaker.
|
||||
"""
|
||||
from gateway import restart_loop_guard as _rlg
|
||||
|
||||
max_restarts = _rlg.DEFAULT_MAX_RESTARTS
|
||||
window_seconds = _rlg.DEFAULT_WINDOW_SECONDS
|
||||
try:
|
||||
user_cfg = _load_gateway_config()
|
||||
gw = user_cfg.get("gateway") if isinstance(user_cfg, dict) else None
|
||||
rlg = gw.get("restart_loop_guard") if isinstance(gw, dict) else None
|
||||
if isinstance(rlg, dict):
|
||||
if isinstance(rlg.get("max_restarts"), int):
|
||||
max_restarts = rlg["max_restarts"]
|
||||
if isinstance(rlg.get("window_seconds"), int) and rlg["window_seconds"] > 0:
|
||||
window_seconds = rlg["window_seconds"]
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return max_restarts, window_seconds
|
||||
|
||||
def _scale_to_zero_should_arm(self) -> bool:
|
||||
"""Whether to start the idle watcher (D1/D11/§3.4(1))."""
|
||||
from gateway.relay import relay_wake_url
|
||||
|
|
@ -5962,6 +5985,26 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
logger.warning("Failed to enumerate resume-pending sessions: %s", exc)
|
||||
return 0
|
||||
|
||||
# Defense-3 (#30719): break the SIGTERM-respawn loop. Only count this
|
||||
# boot when there are restart-interrupted sessions to resume — a clean
|
||||
# boot must not accrue toward the breaker. If too many such boots have
|
||||
# happened in the configured window, skip auto-resume for THIS boot:
|
||||
# the gateway still comes up and serves real inbound messages, it just
|
||||
# stops replaying the session that keeps killing it. The session stays
|
||||
# resume_pending, so a real user message can still continue it (a human
|
||||
# is now in the loop). Defenses 1-2 cover the cron/CLI/terminal paths;
|
||||
# this catches every other SIGTERM source (e.g. a raw `terminal(
|
||||
# "launchctl kickstart ai.hermes.gateway")`).
|
||||
if candidates:
|
||||
try:
|
||||
from gateway import restart_loop_guard as _rlg
|
||||
|
||||
_max_restarts, _window = self._restart_loop_guard_config()
|
||||
if _rlg.check_and_record(_max_restarts, _window):
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001 — breaker must fail OPEN
|
||||
logger.debug("Restart-loop guard check skipped: %s", exc)
|
||||
|
||||
now = datetime.now()
|
||||
scheduled = 0
|
||||
for entry in candidates:
|
||||
|
|
|
|||
|
|
@ -2717,6 +2717,23 @@ DEFAULT_CONFIG = {
|
|||
"idle_timeout_minutes": 5,
|
||||
},
|
||||
|
||||
# Auto-resume restart-loop breaker (#30719, defense-3). When the
|
||||
# gateway is killed mid-turn (SIGTERM) and revived by a supervisor
|
||||
# (launchd KeepAlive / systemd Restart=), it auto-resumes the
|
||||
# restart-interrupted session on the next boot. If the resumed turn
|
||||
# keeps triggering another kill (e.g. the agent runs a raw
|
||||
# `launchctl kickstart ai.hermes.gateway` that defenses 1-2 don't
|
||||
# cover), the result is a tight SIGTERM-respawn loop. This breaker
|
||||
# counts restart-interrupted boots in a rolling window and, once
|
||||
# `max_restarts` boots happen within `window_seconds`, SKIPS
|
||||
# auto-resume for that boot — the gateway still starts and serves
|
||||
# real inbound messages, it just stops replaying the session that
|
||||
# keeps killing it. Set `max_restarts` to 0 to disable the breaker.
|
||||
"restart_loop_guard": {
|
||||
"max_restarts": 3,
|
||||
"window_seconds": 60,
|
||||
},
|
||||
|
||||
# Inject a human-readable timestamp prefix (e.g.
|
||||
# "[Tue 2026-04-28 13:40:53 CEST]") onto user messages IN THE MODEL'S
|
||||
# CONTEXT so the agent has temporal awareness of when each message was
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ pause/resume/run/remove, status, and tick.
|
|||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Optional
|
||||
|
|
@ -16,25 +15,17 @@ sys.path.insert(0, str(PROJECT_ROOT))
|
|||
|
||||
from hermes_cli.colors import Colors, color
|
||||
|
||||
# Patterns that indicate a cron job targets the gateway lifecycle.
|
||||
# Matches commands that restart/stop the gateway or its service manager.
|
||||
# Deliberately specific — a bare "gateway ... restart" catch-all would block
|
||||
# legitimate prompts that merely mention an unrelated gateway (e.g. "summarize
|
||||
# the API gateway logs and report restart events").
|
||||
_GATEWAY_LIFECYCLE_PATTERNS = re.compile(
|
||||
r"(?i)"
|
||||
r"(hermes\s+gateway\s+(restart|stop|start))"
|
||||
r"|(launchctl\s+(kickstart|unload|load|stop|restart)\s+.*hermes)"
|
||||
r"|(systemctl\s+(-\S+\s+)*(restart|stop|start)\s+.*hermes)"
|
||||
r"|(p?kill\s+.*hermes.*gateway)"
|
||||
# Gateway-lifecycle command detection lives in ``cron.lifecycle_guard`` so it
|
||||
# can be shared across every job-creation path (CLI + the agent's ``cronjob``
|
||||
# model tool via ``cron.jobs.create_job``) without a circular import. Re-export
|
||||
# ``_contains_gateway_lifecycle_command`` here for back-compat: ``tools/
|
||||
# terminal_tool.py`` imports it from this module to hard-block the same
|
||||
# commands at execution time when ``_HERMES_GATEWAY=1``.
|
||||
from cron.lifecycle_guard import ( # noqa: F401 (re-exported for terminal_tool)
|
||||
contains_gateway_lifecycle_command as _contains_gateway_lifecycle_command,
|
||||
)
|
||||
|
||||
|
||||
def _contains_gateway_lifecycle_command(text: str) -> bool:
|
||||
"""Return True if *text* contains a gateway lifecycle command pattern."""
|
||||
return bool(_GATEWAY_LIFECYCLE_PATTERNS.search(text))
|
||||
|
||||
|
||||
def _normalize_skills(single_skill=None, skills: Optional[Iterable[str]] = None) -> Optional[List[str]]:
|
||||
if skills is None:
|
||||
if single_skill is None:
|
||||
|
|
@ -301,28 +292,12 @@ def _print_active_jobs_summary(jobs) -> None:
|
|||
|
||||
|
||||
def cron_create(args):
|
||||
# Defense: reject cron jobs that contain gateway lifecycle commands.
|
||||
# Prevents agents from scheduling their own restart/stop, which creates
|
||||
# SIGTERM-respawn loops under launchd/systemd KeepAlive (#30719).
|
||||
prompt = getattr(args, "prompt", None) or ""
|
||||
script = getattr(args, "script", None)
|
||||
combined = prompt
|
||||
if script:
|
||||
try:
|
||||
script_text = Path(script).read_text(encoding="utf-8")
|
||||
combined = f"{combined}\n{script_text}"
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
if _contains_gateway_lifecycle_command(combined):
|
||||
print(color(
|
||||
"Blocked: cron job contains a gateway lifecycle command "
|
||||
"(restart/stop/kill).\n"
|
||||
"This is blocked to prevent restart loops (#30719).\n"
|
||||
"Use `hermes gateway restart` from a shell outside the gateway.",
|
||||
Colors.RED,
|
||||
))
|
||||
return 1
|
||||
|
||||
# The gateway-lifecycle guard lives in cron.jobs.create_job so it fires on
|
||||
# every job-creation path (this CLI subcommand AND the agent's `cronjob`
|
||||
# model tool, which calls create_job directly). When it blocks, create_job
|
||||
# raises GatewayLifecycleBlocked, the `cronjob` tool wrapper catches it and
|
||||
# returns it as result["error"], and the `if not result.get("success")`
|
||||
# branch below prints it in red and exits 1 — same UX as before.
|
||||
result = _cron_api(
|
||||
action="create",
|
||||
schedule=args.schedule,
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ class TestGatewayLifecyclePattern:
|
|||
@pytest.mark.parametrize("text", [
|
||||
"hermes gateway restart",
|
||||
"hermes gateway stop",
|
||||
"hermes gateway start",
|
||||
"hermes gateway restart", # double spaces
|
||||
"Hermez Gateway Restart".lower().replace("z", "s"), # case handled
|
||||
"HERMES GATEWAY RESTART", # uppercase
|
||||
|
|
@ -50,6 +49,7 @@ class TestGatewayLifecyclePattern:
|
|||
@pytest.mark.parametrize("text", [
|
||||
"kill hermes gateway process",
|
||||
"pkill -f hermes.*gateway",
|
||||
"pkill -f gateway.*hermes", # inverse token order
|
||||
])
|
||||
def test_kill_commands(self, text):
|
||||
assert _contains_gateway_lifecycle_command(text), f"Should match: {text!r}"
|
||||
|
|
@ -62,11 +62,28 @@ class TestGatewayLifecyclePattern:
|
|||
"echo 'just a normal cron job'",
|
||||
"run the backup script",
|
||||
"gateway is running fine",
|
||||
# `hermes gateway start` is benign — starting a gateway from inside a
|
||||
# gateway is a no-op / "already running", and a legit cron job may
|
||||
# start a sibling profile's gateway. Only restart/stop/kill are the
|
||||
# foot-gun (#30719 lists only those).
|
||||
"hermes gateway start",
|
||||
"hermes gateway start --all",
|
||||
# Tightened launchctl/systemctl branches: ops on NON-gateway hermes
|
||||
# services must not be falsely blocked (the old `.*hermes` matched any
|
||||
# hermes token).
|
||||
"launchctl unload ai.hermes.update-checker.plist",
|
||||
"launchctl restart ai.hermes.daemon",
|
||||
"systemctl restart hermes-meta.service",
|
||||
"systemctl restart hermes-cron-helper",
|
||||
# Regression (#30728 follow-up): legit prompts that merely mention an
|
||||
# unrelated gateway + a restart must NOT be blocked.
|
||||
# unrelated gateway + a restart must NOT be blocked. The cron prompt is
|
||||
# fed to an LLM, not a shell, so substring detection on English text is
|
||||
# a high-FP no-op — only concrete command shapes trigger the block.
|
||||
"Summarize the API gateway logs and report any restart events from last night",
|
||||
"Check if the payment gateway needs a restart after the deploy",
|
||||
"Monitor the gateway and tell me if a restart is recommended",
|
||||
"research how the OpenAI API gateway handles restart after rate limiting",
|
||||
"compare AWS API Gateway vs Cloudflare on restart latency",
|
||||
])
|
||||
def test_safe_commands(self, text):
|
||||
assert not _contains_gateway_lifecycle_command(text), f"Should NOT match: {text!r}"
|
||||
|
|
@ -122,9 +139,14 @@ class TestCronCreateLifecycleBlock:
|
|||
out = capsys.readouterr().out
|
||||
assert "Blocked" in out
|
||||
|
||||
def test_block_script_with_lifecycle_command(self, tmp_path, capsys):
|
||||
script = tmp_path / "restart.sh"
|
||||
script.write_text("#!/bin/bash\nhermes gateway restart\n")
|
||||
def test_block_script_with_lifecycle_command(self, tmp_path, capsys, monkeypatch):
|
||||
# A no_agent job whose script IS the job (the issue's real abuse path:
|
||||
# restart_hermes_gateway_once.sh). The script must live under
|
||||
# HERMES_HOME/scripts so the scheduler — and the guard — resolve it.
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
scripts_dir = tmp_path / ".hermes" / "scripts"
|
||||
scripts_dir.mkdir(parents=True)
|
||||
(scripts_dir / "restart.sh").write_text("#!/bin/bash\nhermes gateway restart\n")
|
||||
args = Namespace(
|
||||
cron_command="create",
|
||||
schedule="1h",
|
||||
|
|
@ -134,10 +156,10 @@ class TestCronCreateLifecycleBlock:
|
|||
repeat=None,
|
||||
skill=None,
|
||||
skills=None,
|
||||
script=str(script),
|
||||
script="restart.sh",
|
||||
workdir=None,
|
||||
profile=None,
|
||||
no_agent=False,
|
||||
no_agent=True,
|
||||
)
|
||||
rc = cron_command(args)
|
||||
assert rc == 1
|
||||
|
|
@ -357,3 +379,155 @@ class TestTerminalToolGatewayLifecycleGuard:
|
|||
# approval flow handles it (here mocked as approved).
|
||||
assert result["exit_code"] == 0
|
||||
assert calls == ["systemctl restart hermes-gateway"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cron.lifecycle_guard module — the shared checker create_job/CLI/terminal use
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestLifecycleGuardModule:
|
||||
"""Direct tests for cron.lifecycle_guard.check_gateway_lifecycle."""
|
||||
|
||||
def test_prompt_with_command_raises(self):
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked, check_gateway_lifecycle
|
||||
with pytest.raises(GatewayLifecycleBlocked) as exc:
|
||||
check_gateway_lifecycle("please run hermes gateway restart", None)
|
||||
assert "#30719" in str(exc.value)
|
||||
|
||||
def test_clean_prompt_does_not_raise(self):
|
||||
from cron.lifecycle_guard import check_gateway_lifecycle
|
||||
check_gateway_lifecycle("research the gateway architecture", None)
|
||||
check_gateway_lifecycle("check server health and restart watchers", None)
|
||||
|
||||
def test_script_with_command_raises(self, tmp_path, monkeypatch):
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked, check_gateway_lifecycle
|
||||
script = tmp_path / "restart.sh"
|
||||
script.write_text("#!/bin/bash\nhermes gateway restart\n")
|
||||
with pytest.raises(GatewayLifecycleBlocked):
|
||||
check_gateway_lifecycle("clean prompt", str(script))
|
||||
|
||||
def test_split_across_prompt_and_script_still_blocks(self, tmp_path):
|
||||
"""Concatenated scan prevents splitting the command between prompt and
|
||||
script to slip through."""
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked, check_gateway_lifecycle
|
||||
script = tmp_path / "ops.sh"
|
||||
script.write_text("hermes gateway stop\n")
|
||||
with pytest.raises(GatewayLifecycleBlocked):
|
||||
check_gateway_lifecycle("daily ops job", str(script))
|
||||
|
||||
def test_binary_script_does_not_silently_bypass(self, tmp_path):
|
||||
"""Non-UTF-8 bytes used to be swallowed by UnicodeDecodeError; now we
|
||||
decode with errors='replace' so the scan always sees the command."""
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked, check_gateway_lifecycle
|
||||
script = tmp_path / "weird.bin"
|
||||
script.write_bytes(b"\xfehermes gateway restart\xff")
|
||||
with pytest.raises(GatewayLifecycleBlocked):
|
||||
check_gateway_lifecycle("", str(script))
|
||||
|
||||
def test_missing_script_does_not_raise(self, tmp_path):
|
||||
from cron.lifecycle_guard import check_gateway_lifecycle
|
||||
check_gateway_lifecycle("clean prompt", str(tmp_path / "nonexistent.sh"))
|
||||
|
||||
def test_relative_script_resolved_under_scripts_dir(self, tmp_path, monkeypatch):
|
||||
"""A bare/relative script name resolves under HERMES_HOME/scripts (the
|
||||
same place the scheduler runs it from) — otherwise the guard would read
|
||||
a nonexistent relative path and scan prompt-only content."""
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked, check_gateway_lifecycle
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
scripts_dir = tmp_path / ".hermes" / "scripts"
|
||||
scripts_dir.mkdir(parents=True)
|
||||
(scripts_dir / "restart.sh").write_text(
|
||||
"launchctl kickstart -k gui/501/ai.hermes.gateway\n"
|
||||
)
|
||||
with pytest.raises(GatewayLifecycleBlocked):
|
||||
check_gateway_lifecycle("daily", "restart.sh")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defense 2 (chokepoint): cron.jobs.create_job blocks the AGENT model-tool path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCreateJobBlocksLifecycleCommands:
|
||||
"""The regression the CLI-layer-only guard could not catch: the agent's
|
||||
`cronjob` model tool calls cron.jobs.create_job directly, bypassing
|
||||
hermes_cli.cron.cron_create. Enforcing at create_job covers both."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup_cron_dir(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("cron.jobs.CRON_DIR", tmp_path / "cron")
|
||||
monkeypatch.setattr("cron.jobs.JOBS_FILE", tmp_path / "cron" / "jobs.json")
|
||||
monkeypatch.setattr("cron.jobs.OUTPUT_DIR", tmp_path / "cron" / "output")
|
||||
|
||||
def test_create_job_blocks_prompt_command(self):
|
||||
from cron.jobs import create_job
|
||||
from cron.lifecycle_guard import GatewayLifecycleBlocked
|
||||
with pytest.raises(GatewayLifecycleBlocked):
|
||||
create_job(prompt="then run hermes gateway restart", schedule="30m")
|
||||
|
||||
def test_create_job_allows_benign_prompt(self):
|
||||
from cron.jobs import create_job
|
||||
job = create_job(prompt="summarize the API gateway logs and note restart events",
|
||||
schedule="30m")
|
||||
assert job["id"]
|
||||
|
||||
def test_cronjob_tool_surfaces_block_as_error(self, tmp_path, monkeypatch):
|
||||
"""End-to-end through the model tool: the block comes back as
|
||||
result['error'] with the #30719 hint, not an unhandled exception."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
(tmp_path / ".hermes").mkdir(parents=True)
|
||||
from tools.cronjob_tools import cronjob
|
||||
result = json.loads(cronjob(
|
||||
action="create", schedule="0 9 * * *",
|
||||
prompt="please run hermes gateway restart nightly",
|
||||
))
|
||||
assert result.get("success") is False
|
||||
assert "#30719" in result.get("error", "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defense 3: auto-resume restart-loop breaker
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRestartLoopGuard:
|
||||
"""gateway.restart_loop_guard trips after >= max_restarts
|
||||
restart-interrupted boots inside window_seconds, breaking a
|
||||
SIGTERM-respawn loop that defenses 1-2 don't cover."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_state(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
(tmp_path / ".hermes").mkdir(parents=True)
|
||||
import gateway.restart_loop_guard as rlg
|
||||
rlg.clear()
|
||||
|
||||
def test_burst_trips_on_threshold(self):
|
||||
import gateway.restart_loop_guard as rlg
|
||||
assert rlg.check_and_record(3, 60, now=1000.0) is False
|
||||
assert rlg.check_and_record(3, 60, now=1005.0) is False
|
||||
assert rlg.check_and_record(3, 60, now=1010.0) is True
|
||||
|
||||
def test_spread_boots_never_trip(self):
|
||||
import gateway.restart_loop_guard as rlg
|
||||
assert rlg.check_and_record(3, 60, now=1000.0) is False
|
||||
assert rlg.check_and_record(3, 60, now=1070.0) is False
|
||||
assert rlg.check_and_record(3, 60, now=1140.0) is False
|
||||
|
||||
def test_disabled_when_max_restarts_zero(self):
|
||||
import gateway.restart_loop_guard as rlg
|
||||
for i in range(5):
|
||||
assert rlg.check_and_record(0, 60, now=1000.0 + i) is False
|
||||
|
||||
def test_is_tripped_reads_without_recording(self):
|
||||
import gateway.restart_loop_guard as rlg
|
||||
rlg.record_restart_interrupted_boot(60, now=1000.0)
|
||||
rlg.record_restart_interrupted_boot(60, now=1001.0)
|
||||
assert rlg.is_restart_loop_tripped(3, 60, now=1002.0) is False
|
||||
rlg.record_restart_interrupted_boot(60, now=1002.0)
|
||||
assert rlg.is_restart_loop_tripped(3, 60, now=1003.0) is True
|
||||
|
||||
def test_clear_resets(self):
|
||||
import gateway.restart_loop_guard as rlg
|
||||
rlg.check_and_record(3, 60, now=1000.0)
|
||||
rlg.check_and_record(3, 60, now=1001.0)
|
||||
rlg.clear()
|
||||
assert rlg.check_and_record(3, 60, now=1002.0) is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue