mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
fix(gateway): unify reset boundaries vs recovery — promote accidental ends, honor mode=none, adapter-aware resume guidance
Unifies the two gateway subsystems that were fighting each other: the 'never lose a session' recovery machinery (#54878 stale-route self-heal, find_latest_gateway_session_for_peer reopening agent_close/ws_orphan_reap rows) and the session reset/expiry machinery (expiry watcher, /new, /resume, resume_pending freshness gate). The unified contract: - INTENTIONAL boundaries (expiry finalization, auto-reset, /new, /resume switch) are recorded durably via promote_to_session_reset(), which upgrades accidental recoverable end_reasons (agent_close, ws_orphan_reap) to the explicit boundary while preserving other explicit reasons (compression, etc.). Recovery then correctly refuses to resurrect them. - ACCIDENTAL ends (crash, cleanup bug, mistaken reaper) stay recoverable — genuine crash recovery is untouched. On top of the cherry-picked contributor commits: - promote_to_session_reset widened to ws_orphan_reap + parameterized reason so auto-reset paths stay auditable (idle/daily/suspended/ resume_pending_expired) (#61220, #61993, #63539) - get_or_create_session auto-reset, reset_session (/new), and switch_session (/resume) all write through the promote path — the first-reason-wins end_session no-op could previously leave a reset session resurrectable behind a stale agent_close row (#61993) - resume_pending freshness gate now honors session_reset.mode=none: explicit opt-out of automatic resets also opts out of the zombie gate (#61052) - resume recovery note extracted to build_resume_recovery_note() and made adapter-aware via a new interactive_resume capability flag: webhook/api_server auto-resume turns now CONTINUE the interrupted task instead of emitting an unanswerable 'session restored' acknowledgement that abandoned the work (#57056) - tests updated to call the real note builder instead of mirroring it E2E-validated against a real SessionDB + SessionStore in a temp HERMES_HOME: expiry->agent_close->no-resurrection, /new promote, crash recovery preserved, mode=none opt-out, routing-table flag sync.
This commit is contained in:
parent
d17daf0b12
commit
9fc0074bac
11 changed files with 273 additions and 112 deletions
|
|
@ -548,7 +548,13 @@ The `_session_expiry_watcher` task runs in the gateway event loop every 300 seco
|
|||
- Clean up cached AIAgent resources (close tool resources, shut down memory provider).
|
||||
- Evict the cached agent entry.
|
||||
- Clear per-session overrides (`_session_model_overrides`, reasoning overrides, etc.).
|
||||
- Mark `expiry_finalized=True` and persist.
|
||||
- Mark `expiry_finalized=True` and persist (sessions.json + state.db).
|
||||
- Promote the state.db session row to `end_reason='session_reset'` via
|
||||
`promote_to_session_reset()` — conditional: only live rows or rows ended with a
|
||||
recoverable accidental reason (`agent_close`, `ws_orphan_reap`) are promoted, so
|
||||
explicit boundaries (`compression`, `session_switch`, …) are never overwritten. This
|
||||
durably records the reset so stale-route recovery cannot resurrect the expired
|
||||
session with its full history (#61220, #61993, #63539).
|
||||
|
||||
2. **Sweep idle cached agents** — Calls `_sweep_idle_cached_agents()` to evict agents that
|
||||
have been idle beyond `_AGENT_CACHE_IDLE_TTL_SECS` (3600s / 1h), regardless of session
|
||||
|
|
|
|||
|
|
@ -932,6 +932,11 @@ class APIServerAdapter(BasePlatformAdapter):
|
|||
# ``async_delivery_supported()``.
|
||||
supports_async_delivery: bool = False
|
||||
|
||||
# Same statelessness applies to the startup auto-resume prompt: no client
|
||||
# is waiting to answer "session restored — what next?", so a resumed turn
|
||||
# should complete the interrupted work rather than acknowledge (#57056).
|
||||
interactive_resume: bool = False
|
||||
|
||||
def __init__(self, config: PlatformConfig):
|
||||
super().__init__(config, Platform.API_SERVER)
|
||||
extra = config.extra or {}
|
||||
|
|
|
|||
|
|
@ -2365,6 +2365,19 @@ class BasePlatformAdapter(ABC):
|
|||
# generic seam; Slack is merely the first consumer).
|
||||
supports_inchannel_continuable: bool = False
|
||||
|
||||
# Whether a human is interactively present on this platform to answer a
|
||||
# "session restored — what next?" prompt. The startup auto-resume turn
|
||||
# (``_schedule_resume_pending_sessions`` → the ``_is_resume_pending``
|
||||
# branch in ``_handle_message_with_agent``) reads this to pick its
|
||||
# guidance: interactive platforms (Telegram, Slack, Discord DMs, …) get
|
||||
# "report the restore and ask what the user wants next"; non-interactive
|
||||
# event platforms (webhook) get "finish the interrupted work" because
|
||||
# nobody is there to answer, and an acknowledgement would silently
|
||||
# abandon the task (#57056). Read generically via ``getattr(adapter,
|
||||
# "interactive_resume", True)`` — no per-platform branching at the call
|
||||
# site.
|
||||
interactive_resume: bool = True
|
||||
|
||||
# Back-reference to the running ``GatewayRunner``, injected by
|
||||
# ``gateway/run.py`` after the adapter is created. Adapters consume it via
|
||||
# ``getattr(self, "gateway_runner", None)`` for cross-platform delivery and
|
||||
|
|
|
|||
|
|
@ -147,6 +147,12 @@ def check_webhook_requirements() -> bool:
|
|||
class WebhookAdapter(BasePlatformAdapter):
|
||||
"""Generic webhook receiver that triggers agent runs from HTTP POSTs."""
|
||||
|
||||
# No human is present to answer a "session restored — what next?" prompt:
|
||||
# webhook runs are event-triggered. The startup auto-resume turn must
|
||||
# instruct the model to FINISH the interrupted work instead of emitting an
|
||||
# interactive acknowledgement that abandons the task (#57056).
|
||||
interactive_resume: bool = False
|
||||
|
||||
def __init__(self, config: PlatformConfig):
|
||||
super().__init__(config, Platform.WEBHOOK)
|
||||
# ``host`` may be None (dual-stack default) or a user-pinned string.
|
||||
|
|
|
|||
131
gateway/run.py
131
gateway/run.py
|
|
@ -712,6 +712,75 @@ def _is_fresh_gateway_interruption(
|
|||
return current - timestamp <= window
|
||||
|
||||
|
||||
def build_resume_recovery_note(
|
||||
reason: Optional[str],
|
||||
message: str = "",
|
||||
*,
|
||||
interactive: bool = True,
|
||||
) -> str:
|
||||
"""Build the resume-pending recovery system note for an interrupted turn.
|
||||
|
||||
``reason`` is the session's ``resume_reason`` (``restart_timeout``,
|
||||
``shutdown_timeout``, or anything else → generic interruption phrasing).
|
||||
``message`` is the user's NEW message text; empty means this is the
|
||||
startup auto-resume turn synthesized by
|
||||
``_schedule_resume_pending_sessions`` with no human message attached.
|
||||
|
||||
``interactive`` selects the empty-message guidance: on interactive
|
||||
platforms a human is present, so "report the restore and ask what next"
|
||||
is right. On non-interactive event platforms (webhook, API server —
|
||||
adapters with ``interactive_resume = False``) nobody can answer; the
|
||||
resumed turn must instead complete the interrupted work, or the task is
|
||||
silently abandoned behind a "restored" acknowledgement that goes
|
||||
nowhere (#57056).
|
||||
"""
|
||||
reason_phrase = (
|
||||
"a gateway restart"
|
||||
if reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
if message:
|
||||
resume_guidance = (
|
||||
"Address the user's NEW message below FIRST and focus "
|
||||
"on what the user is asking now."
|
||||
)
|
||||
tail_guidance = (
|
||||
"Do NOT re-execute old tool calls — skip any "
|
||||
"unfinished work from the conversation history."
|
||||
)
|
||||
elif interactive:
|
||||
resume_guidance = (
|
||||
"Report to the user that the session was restored "
|
||||
"successfully and ask what they would like to do next."
|
||||
)
|
||||
tail_guidance = (
|
||||
"Do NOT re-execute old tool calls — skip any "
|
||||
"unfinished work from the conversation history."
|
||||
)
|
||||
else:
|
||||
resume_guidance = (
|
||||
"No user is present on this non-interactive platform, "
|
||||
"so do NOT emit a 'session restored' acknowledgement "
|
||||
"or ask questions. Review the conversation history and "
|
||||
"CONTINUE the interrupted task to completion."
|
||||
)
|
||||
tail_guidance = (
|
||||
"Do NOT re-run tool calls whose results already "
|
||||
"appear in the history — resume from the first step "
|
||||
"that has no recorded result."
|
||||
)
|
||||
return (
|
||||
f"[System note: The previous turn was interrupted by "
|
||||
f"{reason_phrase}; the gateway is now back online. "
|
||||
f"Any restart/shutdown command in the history has already "
|
||||
f"run — do NOT re-execute or verify it. {resume_guidance} "
|
||||
f"{tail_guidance}]"
|
||||
+ (f"\n\n{message}" if message else "")
|
||||
)
|
||||
|
||||
|
||||
# Assistant-message fields that must survive transcript replay so multi-turn
|
||||
# reasoning context, prefix-cache hits, and provider-specific echo
|
||||
# requirements all behave the same on the gateway as they do in the CLI.
|
||||
|
|
@ -19520,36 +19589,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
|
||||
if _is_resume_pending:
|
||||
_reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout"
|
||||
_reason_phrase = (
|
||||
"a gateway restart"
|
||||
if _reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if _reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
_persist_user_message_override = message
|
||||
# The empty-message case is the auto-resume startup turn
|
||||
# synthesized by _schedule_resume_pending_sessions — there is
|
||||
# no NEW user message to address, so tell the model to report
|
||||
# recovery instead of the (nonexistent) "new message".
|
||||
if message:
|
||||
_resume_guidance = (
|
||||
"Address the user's NEW message below FIRST and focus "
|
||||
"on what the user is asking now."
|
||||
)
|
||||
else:
|
||||
_resume_guidance = (
|
||||
"Report to the user that the session was restored "
|
||||
"successfully and ask what they would like to do next."
|
||||
)
|
||||
message = (
|
||||
f"[System note: The previous turn was interrupted by "
|
||||
f"{_reason_phrase}; the gateway is now back online. "
|
||||
f"Any restart/shutdown command in the history has already "
|
||||
f"run — do NOT re-execute or verify it. {_resume_guidance} "
|
||||
f"Do NOT re-execute old tool calls — skip any unfinished "
|
||||
f"work from the conversation history.]"
|
||||
+ (f"\n\n{message}" if message else "")
|
||||
# no NEW user message to address. Guidance is adapter-aware:
|
||||
# interactive platforms report the restore and ask what next;
|
||||
# non-interactive event platforms (webhook, API server)
|
||||
# continue the interrupted work instead, because nobody is
|
||||
# present to answer and an acknowledgement would silently
|
||||
# abandon the task (#57056).
|
||||
_resume_adapter = self._adapter_for_source(source)
|
||||
_interactive_resume = bool(
|
||||
getattr(_resume_adapter, "interactive_resume", True)
|
||||
)
|
||||
message = build_resume_recovery_note(
|
||||
_reason, message, interactive=_interactive_resume,
|
||||
)
|
||||
elif _has_fresh_tool_tail:
|
||||
_persist_user_message_override = message
|
||||
|
|
@ -19590,22 +19644,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_sn_reason = (
|
||||
getattr(_resume_entry, "resume_reason", None) or "restart_timeout"
|
||||
)
|
||||
_sn_reason_phrase = (
|
||||
"a gateway restart"
|
||||
if _sn_reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if _sn_reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
message = (
|
||||
f"[System note: The previous turn was interrupted by "
|
||||
f"{_sn_reason_phrase}; the gateway is now back online. "
|
||||
f"Any restart/shutdown command in the history has already "
|
||||
f"run — do NOT re-execute or verify it. Report to the user "
|
||||
f"that the session was restored successfully and ask what "
|
||||
f"they would like to do next. Do NOT re-execute old tool "
|
||||
f"calls — skip any unfinished work from the conversation "
|
||||
f"history.]"
|
||||
_sn_adapter = self._adapter_for_source(source)
|
||||
message = build_resume_recovery_note(
|
||||
_sn_reason,
|
||||
"",
|
||||
interactive=bool(
|
||||
getattr(_sn_adapter, "interactive_resume", True)
|
||||
),
|
||||
)
|
||||
|
||||
_approval_session_key = session_key or ""
|
||||
|
|
|
|||
|
|
@ -1933,13 +1933,24 @@ class SessionStore:
|
|||
elif _entry_for_checks.resume_pending:
|
||||
_reset_reason = self._should_reset(_entry_for_checks, source)
|
||||
if not _reset_reason:
|
||||
_fw = auto_continue_freshness_window()
|
||||
_ref_time = (
|
||||
_entry_for_checks.last_resume_marked_at
|
||||
or _entry_for_checks.updated_at
|
||||
# Freshness-gate stale resume_pending zombies (#46934) —
|
||||
# but honor an explicit ``session_reset.mode: none``: the
|
||||
# user opted out of ALL automatic resets, so an expired
|
||||
# resume marker must fall through to a normal resume of
|
||||
# the preserved transcript, never a silent fresh session
|
||||
# (#61052).
|
||||
_policy = self.config.get_reset_policy(
|
||||
platform=source.platform,
|
||||
session_type=source.chat_type,
|
||||
)
|
||||
if _fw > 0 and (now - _ref_time).total_seconds() > _fw:
|
||||
_reset_reason = "resume_pending_expired"
|
||||
if _policy.mode != "none":
|
||||
_fw = auto_continue_freshness_window()
|
||||
_ref_time = (
|
||||
_entry_for_checks.last_resume_marked_at
|
||||
or _entry_for_checks.updated_at
|
||||
)
|
||||
if _fw > 0 and (now - _ref_time).total_seconds() > _fw:
|
||||
_reset_reason = "resume_pending_expired"
|
||||
else:
|
||||
_reset_reason = self._should_reset(_entry_for_checks, source)
|
||||
|
||||
|
|
@ -2074,7 +2085,16 @@ class SessionStore:
|
|||
# "session_reset" caused by idle/daily expiry).
|
||||
_db_end_reason = auto_reset_reason if auto_reset_reason else "session_reset"
|
||||
try:
|
||||
self._db.end_session(db_end_session_id, _db_end_reason)
|
||||
# promote_to_session_reset, not end_session: the row may
|
||||
# already be ended with a recoverable accidental reason
|
||||
# (agent_close / ws_orphan_reap), which first-reason-wins
|
||||
# end_session would preserve — leaving the reset session
|
||||
# resurrectable by stale-route recovery (#61220, #61993).
|
||||
_promote = getattr(self._db, "promote_to_session_reset", None)
|
||||
if callable(_promote):
|
||||
_promote(db_end_session_id, _db_end_reason)
|
||||
else:
|
||||
self._db.end_session(db_end_session_id, _db_end_reason)
|
||||
except Exception as e:
|
||||
logger.debug("Session DB operation failed: %s", e)
|
||||
|
||||
|
|
@ -2340,7 +2360,15 @@ class SessionStore:
|
|||
|
||||
if self._db and db_end_session_id:
|
||||
try:
|
||||
self._db.end_session(db_end_session_id, "session_reset")
|
||||
# Promote (not plain end_session): an accidental
|
||||
# agent_close/ws_orphan_reap end must not survive an explicit
|
||||
# user reset, or recovery resurrects the reset session
|
||||
# (#61993 — the user's /new was silently undone).
|
||||
_promote = getattr(self._db, "promote_to_session_reset", None)
|
||||
if callable(_promote):
|
||||
_promote(db_end_session_id, "session_reset")
|
||||
else:
|
||||
self._db.end_session(db_end_session_id, "session_reset")
|
||||
except Exception as e:
|
||||
logger.debug("Session DB operation failed: %s", e)
|
||||
|
||||
|
|
@ -2401,7 +2429,15 @@ class SessionStore:
|
|||
|
||||
if self._db and db_end_session_id:
|
||||
try:
|
||||
self._db.end_session(db_end_session_id, "session_switch")
|
||||
# Promote (not plain end_session): a stale agent_close /
|
||||
# ws_orphan_reap end on the outgoing session must be upgraded
|
||||
# to the explicit switch boundary, or recovery can resurrect
|
||||
# it over the user's /resume choice (#61220 bug class).
|
||||
_promote = getattr(self._db, "promote_to_session_reset", None)
|
||||
if callable(_promote):
|
||||
_promote(db_end_session_id, "session_switch")
|
||||
else:
|
||||
self._db.end_session(db_end_session_id, "session_switch")
|
||||
except Exception as e:
|
||||
logger.debug("Session DB end_session failed: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -2286,15 +2286,33 @@ class SessionDB:
|
|||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def promote_to_session_reset(self, session_id: str) -> bool:
|
||||
"""Mark a session as ended by session_reset — but only when safe.
|
||||
def promote_to_session_reset(
|
||||
self, session_id: str, reason: str = "session_reset"
|
||||
) -> bool:
|
||||
"""Durably mark a session as ended by an intentional reset boundary.
|
||||
|
||||
Promotes *only* live rows (``ended_at IS NULL``) or rows ended with
|
||||
``agent_close``. Explicit conversation boundaries such as
|
||||
``compression``, ``session_reset``, ``new_command``, etc. are
|
||||
Promotes *only* live rows (``ended_at IS NULL``) or rows carrying an
|
||||
accidental end_reason that the recovery query
|
||||
(``find_latest_gateway_session_for_peer``) treats as recoverable:
|
||||
``agent_close`` (older gateway cleanup bug) and ``ws_orphan_reap``
|
||||
(mistaken TUI reaper). Explicit conversation boundaries such as
|
||||
``compression``, ``session_reset``, ``session_switch``, etc. are
|
||||
preserved — the first writer wins for those, and a later expiry
|
||||
finalization must not silently overwrite them.
|
||||
|
||||
Plain ``end_session()`` is NOT sufficient for reset boundaries: it
|
||||
no-ops on an already-ended row, so a row that agent cleanup already
|
||||
closed as ``agent_close`` would stay recoverable and stale-route
|
||||
recovery would resurrect the reset session with its full history
|
||||
(#61220, #61993, #63539).
|
||||
|
||||
Keep this promotion set in sync with the recoverable set in
|
||||
``find_latest_gateway_session_for_peer`` — any reason recovery would
|
||||
reopen must be promotable here.
|
||||
|
||||
``reason`` lets reset paths keep their auditable specific reasons
|
||||
(``idle``, ``daily``, ``suspended``, ``resume_pending_expired``).
|
||||
|
||||
Returns ``True`` when the row was promoted, ``False`` when skipped
|
||||
(already has a different explicit end_reason, or row not found).
|
||||
"""
|
||||
|
|
@ -2304,9 +2322,10 @@ class SessionDB:
|
|||
|
||||
def _do(conn):
|
||||
cursor = conn.execute(
|
||||
"UPDATE sessions SET ended_at = ?, end_reason = 'session_reset' "
|
||||
"WHERE id = ? AND (ended_at IS NULL OR end_reason = 'agent_close')",
|
||||
(now, session_id),
|
||||
"UPDATE sessions SET ended_at = ?, end_reason = ? "
|
||||
"WHERE id = ? AND (ended_at IS NULL "
|
||||
"OR end_reason IN ('agent_close', 'ws_orphan_reap'))",
|
||||
(now, reason, session_id),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
|
|
|||
|
|
@ -321,6 +321,8 @@ AUTHOR_MAP = {
|
|||
"dkobi16@gmail.com": "Diyoncrz18",
|
||||
"arnaud@nolimitdevelopment.com": "ali-nld",
|
||||
"sswdarius@gmail.com": "necoweb3",
|
||||
"wei-yujie@qq.com": "DNAlec", # PR #61743 salvage (honor reset policy in #54878 stale-heal recovery)
|
||||
"joelbrilliant1@gmail.com": "joelbrilliant", # PR #58486 salvage (session-expiry cleanup must not end row as agent_close)
|
||||
"bassisho@Mac-mini-bassis.local": "hydracoco7", # PR #61382 salvage (id-less cron job freeze)
|
||||
"AlexFucuson9@users.noreply.github.com": "AlexFucuson9", # PR #61209 salvage (hygiene compression data loss)
|
||||
"email@adambig.gs": "adambiggs", # PR #43819 salvage (holographic shared SQLite connection)
|
||||
|
|
|
|||
|
|
@ -312,7 +312,12 @@ class TestResumePendingFreshnessGate:
|
|||
|
||||
def test_stale_resume_pending_falls_through_to_reset(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "3600")
|
||||
store = _make_store(tmp_path)
|
||||
# The freshness gate only applies when the user has opted into
|
||||
# automatic resets — session_reset.mode: none disables it (#61052).
|
||||
from gateway.config import SessionResetPolicy
|
||||
store = _make_store(
|
||||
tmp_path, policy=SessionResetPolicy(mode="idle", idle_minutes=999999)
|
||||
)
|
||||
source = _make_source()
|
||||
entry = self._mark_resume_pending(store, source)
|
||||
|
||||
|
|
@ -329,6 +334,25 @@ class TestResumePendingFreshnessGate:
|
|||
assert fresh.session_id != entry.session_id
|
||||
assert not fresh.resume_pending
|
||||
|
||||
def test_reset_mode_none_disables_freshness_gate(self, tmp_path, monkeypatch):
|
||||
"""session_reset.mode: none opts out of ALL automatic resets —
|
||||
including the resume_pending freshness gate (#61052)."""
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "3600")
|
||||
from gateway.config import SessionResetPolicy
|
||||
store = _make_store(tmp_path, policy=SessionResetPolicy(mode="none"))
|
||||
source = _make_source()
|
||||
entry = self._mark_resume_pending(store, source)
|
||||
|
||||
with store._lock:
|
||||
entry.last_resume_marked_at = datetime.now() - timedelta(seconds=7200)
|
||||
entry.updated_at = datetime.now()
|
||||
store._save()
|
||||
|
||||
refreshed = store.get_or_create_session(source)
|
||||
# Explicit opt-out honored: same session back, transcript preserved.
|
||||
assert refreshed.session_id == entry.session_id
|
||||
assert refreshed.resume_pending
|
||||
|
||||
def test_freshness_gate_disabled_returns_stale_session(self, tmp_path, monkeypatch):
|
||||
# Opt-out: window <= 0 restores the pre-fix "always fresh" behaviour.
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "0")
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from gateway.run import (
|
|||
_is_fresh_gateway_interruption,
|
||||
_last_transcript_timestamp,
|
||||
_should_clear_resume_pending_after_turn,
|
||||
build_resume_recovery_note,
|
||||
)
|
||||
from gateway.session import SessionEntry, SessionSource, SessionStore
|
||||
from tests.gateway.restart_test_helpers import (
|
||||
|
|
@ -152,32 +153,9 @@ def _simulate_note_injection(
|
|||
|
||||
if is_resume_pending:
|
||||
reason = getattr(resume_entry, "resume_reason", None) or "restart_timeout"
|
||||
reason_phrase = (
|
||||
"a gateway restart"
|
||||
if reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
if message:
|
||||
resume_guidance = (
|
||||
"Address the user's NEW message below FIRST and focus "
|
||||
"on what the user is asking now."
|
||||
)
|
||||
else:
|
||||
resume_guidance = (
|
||||
"Report to the user that the session was restored "
|
||||
"successfully and ask what they would like to do next."
|
||||
)
|
||||
message = (
|
||||
f"[System note: The previous turn was interrupted by "
|
||||
f"{reason_phrase}; the gateway is now back online. "
|
||||
f"Any restart/shutdown command in the history has already "
|
||||
f"run — do NOT re-execute or verify it. {resume_guidance} "
|
||||
f"Do NOT re-execute old tool calls — skip any unfinished "
|
||||
f"work from the conversation history.]"
|
||||
+ (f"\n\n{message}" if message else "")
|
||||
)
|
||||
# Real production note builder — extracted to module scope in
|
||||
# gateway/run.py so tests exercise the actual strings.
|
||||
message = build_resume_recovery_note(reason, message)
|
||||
elif has_fresh_tool_tail:
|
||||
message = (
|
||||
"[System note: A new message has arrived. The conversation "
|
||||
|
|
@ -196,23 +174,7 @@ def _simulate_note_injection(
|
|||
and getattr(resume_entry, "resume_pending", False)
|
||||
):
|
||||
sn_reason = getattr(resume_entry, "resume_reason", None) or "restart_timeout"
|
||||
sn_reason_phrase = (
|
||||
"a gateway restart"
|
||||
if sn_reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if sn_reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
message = (
|
||||
f"[System note: The previous turn was interrupted by "
|
||||
f"{sn_reason_phrase}; the gateway is now back online. "
|
||||
f"Any restart/shutdown command in the history has already "
|
||||
f"run — do NOT re-execute or verify it. Report to the user "
|
||||
f"that the session was restored successfully and ask what "
|
||||
f"they would like to do next. Do NOT re-execute old tool "
|
||||
f"calls — skip any unfinished work from the conversation "
|
||||
f"history.]"
|
||||
)
|
||||
message = build_resume_recovery_note(sn_reason, "")
|
||||
return message
|
||||
|
||||
|
||||
|
|
@ -521,6 +483,34 @@ class TestResumePendingSystemNote:
|
|||
)
|
||||
assert "gateway shutdown" in result
|
||||
|
||||
def test_empty_message_interactive_note_asks_what_next(self):
|
||||
"""Interactive platforms: the startup auto-resume turn reports the
|
||||
restore and asks the (present) human what to do next."""
|
||||
note = build_resume_recovery_note("restart_timeout", "", interactive=True)
|
||||
assert "session was restored" in note
|
||||
assert "ask what they would like to do next" in note
|
||||
assert "skip any unfinished work" in note
|
||||
|
||||
def test_empty_message_noninteractive_note_continues_task(self):
|
||||
"""Non-interactive platforms (webhook, API server): nobody can answer
|
||||
'what next?', so the resumed turn must complete the interrupted work
|
||||
instead of acknowledging (#57056)."""
|
||||
note = build_resume_recovery_note("restart_timeout", "", interactive=False)
|
||||
assert "CONTINUE the interrupted task" in note
|
||||
assert "session was restored" not in note
|
||||
assert "ask what they would like to do next" not in note
|
||||
# Must not tell the model to skip the unfinished work it should finish.
|
||||
assert "skip any unfinished work" not in note
|
||||
# But still guards against re-running already-recorded tool calls.
|
||||
assert "already appear in the history" in note
|
||||
|
||||
def test_new_message_guidance_identical_regardless_of_interactivity(self):
|
||||
"""A real NEW user message always wins — same guidance either way."""
|
||||
a = build_resume_recovery_note("restart_timeout", "do the thing", interactive=True)
|
||||
b = build_resume_recovery_note("restart_timeout", "do the thing", interactive=False)
|
||||
assert a == b
|
||||
assert "NEW message" in a
|
||||
|
||||
def test_resume_pending_fires_without_tool_tail(self):
|
||||
"""Key improvement over PR #9934: the restart-resume note fires
|
||||
even when the transcript's last role is NOT ``tool``."""
|
||||
|
|
|
|||
|
|
@ -376,7 +376,13 @@ class TestResumePendingExpiredAutoReset:
|
|||
"""Stale resume_pending triggers was_auto_reset=True with reason
|
||||
'resume_pending_expired', NOT 'idle'."""
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "3600")
|
||||
store = _make_store_with_db(tmp_path)
|
||||
# The freshness gate requires an opted-in reset policy — mode "none"
|
||||
# disables it entirely (#61052). Use a huge idle window so only the
|
||||
# freshness gate (not the idle policy) can fire.
|
||||
store = _make_store_with_db(
|
||||
tmp_path,
|
||||
policy=SessionResetPolicy(mode="idle", idle_minutes=999999),
|
||||
)
|
||||
source = _make_source()
|
||||
|
||||
old = self._seed_stale_resume_pending(store, source)
|
||||
|
|
@ -392,7 +398,10 @@ class TestResumePendingExpiredAutoReset:
|
|||
):
|
||||
"""reset_had_activity reflects whether the old session was used."""
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "3600")
|
||||
store = _make_store_with_db(tmp_path)
|
||||
store = _make_store_with_db(
|
||||
tmp_path,
|
||||
policy=SessionResetPolicy(mode="idle", idle_minutes=999999),
|
||||
)
|
||||
source = _make_source()
|
||||
|
||||
old = self._seed_stale_resume_pending(store, source)
|
||||
|
|
@ -411,14 +420,19 @@ class TestResumePendingExpiredAutoReset:
|
|||
generic 'session_reset', so the event is auditable (#58933 fix)."""
|
||||
monkeypatch.setenv("HERMES_AUTO_CONTINUE_FRESHNESS", "3600")
|
||||
db = _make_db_mock()
|
||||
store = _make_store_with_db(tmp_path, db)
|
||||
store = _make_store_with_db(
|
||||
tmp_path, db,
|
||||
policy=SessionResetPolicy(mode="idle", idle_minutes=999999),
|
||||
)
|
||||
source = _make_source()
|
||||
|
||||
old = self._seed_stale_resume_pending(store, source)
|
||||
store.get_or_create_session(source)
|
||||
|
||||
db.end_session.assert_called_once()
|
||||
ended_id, ended_reason = db.end_session.call_args.args
|
||||
# Auto-reset now writes through promote_to_session_reset so an
|
||||
# accidental agent_close end can't shadow the reset boundary.
|
||||
db.promote_to_session_reset.assert_called_once()
|
||||
ended_id, ended_reason = db.promote_to_session_reset.call_args.args
|
||||
assert ended_id == old.session_id
|
||||
assert ended_reason == "resume_pending_expired", (
|
||||
f"expected 'resume_pending_expired', got {ended_reason!r} — "
|
||||
|
|
@ -445,8 +459,8 @@ class TestResumePendingExpiredAutoReset:
|
|||
|
||||
store.get_or_create_session(source)
|
||||
|
||||
db.end_session.assert_called_once()
|
||||
_, ended_reason = db.end_session.call_args.args
|
||||
db.promote_to_session_reset.assert_called_once()
|
||||
_, ended_reason = db.promote_to_session_reset.call_args.args
|
||||
assert ended_reason == "idle"
|
||||
|
||||
def test_freshness_disabled_skips_resume_pending_expired(
|
||||
|
|
@ -465,3 +479,4 @@ class TestResumePendingExpiredAutoReset:
|
|||
# Freshness disabled → same session, no DB end_session call.
|
||||
assert refreshed.session_id == old.session_id
|
||||
db.end_session.assert_not_called()
|
||||
db.promote_to_session_reset.assert_not_called()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue