mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
fix(gateway): bound hygiene compression failures
This commit is contained in:
parent
49a8c61cac
commit
ca9c30c7f0
7 changed files with 594 additions and 13 deletions
|
|
@ -191,6 +191,64 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo
|
|||
return True
|
||||
|
||||
|
||||
class CompressionCommitFence:
|
||||
"""Fence timeout cancellation against post-summary session mutation.
|
||||
|
||||
Compression itself is synchronous and may be running in an executor thread.
|
||||
A caller can stop waiting for the summary, but it cannot kill that thread.
|
||||
This fence makes the commit boundary deterministic: cancellation either wins
|
||||
before session mutation starts, or waits until an already-started commit is
|
||||
fully complete before the caller proceeds.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._cancelled = False
|
||||
self._commit_started = False
|
||||
|
||||
def cancel_before_commit(self) -> bool:
|
||||
"""Cancel a pending commit, or wait for an active commit to finish.
|
||||
|
||||
Returns ``True`` when cancellation won before the commit boundary.
|
||||
Returns ``False`` when the worker had already entered the boundary; in
|
||||
that case acquiring this lock waits until all session mutation finishes.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._commit_started:
|
||||
return False
|
||||
self._cancelled = True
|
||||
return True
|
||||
|
||||
def try_cancel_before_commit(self) -> Optional[bool]:
|
||||
"""Non-blocking form of :meth:`cancel_before_commit`.
|
||||
|
||||
Returns ``None`` while an active commit owns the fence, allowing an
|
||||
async caller to yield instead of blocking its event loop.
|
||||
"""
|
||||
if not self._lock.acquire(blocking=False):
|
||||
return None
|
||||
try:
|
||||
if self._commit_started:
|
||||
return False
|
||||
self._cancelled = True
|
||||
return True
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def begin_commit(self) -> bool:
|
||||
"""Enter the commit boundary unless cancellation already won."""
|
||||
self._lock.acquire()
|
||||
if self._cancelled:
|
||||
self._lock.release()
|
||||
return False
|
||||
self._commit_started = True
|
||||
return True
|
||||
|
||||
def finish_commit(self) -> None:
|
||||
"""Leave a commit boundary entered by :meth:`begin_commit`."""
|
||||
self._lock.release()
|
||||
|
||||
|
||||
def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool:
|
||||
"""Whether the live in-memory SessionDB class structurally predates locks.
|
||||
|
||||
|
|
@ -1025,6 +1083,7 @@ def compress_context(
|
|||
focus_topic: Optional[str] = None,
|
||||
force: bool = False,
|
||||
defer_context_engine_notification: bool = False,
|
||||
commit_fence: Optional[CompressionCommitFence] = None,
|
||||
) -> Tuple[list, str]:
|
||||
"""Compress conversation context and split the session in SQLite.
|
||||
|
||||
|
|
@ -1044,6 +1103,9 @@ def compress_context(
|
|||
callers use the default ``False``.
|
||||
defer_context_engine_notification: Delay the existing context-engine
|
||||
hook until a manual host commits its outer history transaction.
|
||||
commit_fence: Optional cooperative fence for executor callers that
|
||||
may time out. It prevents a late worker from mutating session state
|
||||
after its caller has moved on.
|
||||
|
||||
Returns:
|
||||
``(compressed_messages, new_system_prompt)`` tuple. When
|
||||
|
|
@ -1087,14 +1149,26 @@ def compress_context(
|
|||
# the app server does not expose its native summary prompt, so there is no
|
||||
# truthful injection point for ``on_pre_compress()`` return text here.
|
||||
if getattr(agent, "api_mode", None) == "codex_app_server":
|
||||
return _compress_context_via_codex_app_server(
|
||||
agent,
|
||||
messages,
|
||||
system_message,
|
||||
approx_tokens=approx_tokens,
|
||||
task_id=task_id,
|
||||
force=force,
|
||||
)
|
||||
_codex_fence_entered = False
|
||||
if commit_fence is not None:
|
||||
_codex_fence_entered = commit_fence.begin_commit()
|
||||
if not _codex_fence_entered:
|
||||
existing_prompt = getattr(agent, "_cached_system_prompt", None)
|
||||
if not existing_prompt:
|
||||
existing_prompt = agent._build_system_prompt(system_message)
|
||||
return messages, existing_prompt
|
||||
try:
|
||||
return _compress_context_via_codex_app_server(
|
||||
agent,
|
||||
messages,
|
||||
system_message,
|
||||
approx_tokens=approx_tokens,
|
||||
task_id=task_id,
|
||||
force=force,
|
||||
)
|
||||
finally:
|
||||
if _codex_fence_entered:
|
||||
commit_fence.finish_commit()
|
||||
|
||||
# Every automatic entrypoint must honor compressor-owned cooldown and
|
||||
# breaker state. Gateway hygiene constructs a fresh AIAgent, so the
|
||||
|
|
@ -1468,6 +1542,7 @@ def compress_context(
|
|||
if _activity_heartbeat is not None:
|
||||
_activity_heartbeat.stop("context compression completed")
|
||||
|
||||
_commit_fence_entered = False
|
||||
try:
|
||||
# Capture boundary quality before session-rotation callbacks run. Built-in
|
||||
# and plugin lifecycle hooks may reset per-session compressor fields while
|
||||
|
|
@ -1555,6 +1630,28 @@ def compress_context(
|
|||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
if commit_fence is not None:
|
||||
_commit_fence_entered = commit_fence.begin_commit()
|
||||
if not _commit_fence_entered:
|
||||
logger.info(
|
||||
"Compression commit cancelled before session mutation "
|
||||
"(session=%s).",
|
||||
agent.session_id or "none",
|
||||
)
|
||||
agent._last_compaction_in_place = False
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="commit_fence_cancelled",
|
||||
)
|
||||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
summary_error = getattr(agent.context_compressor, "_last_summary_error", None)
|
||||
if summary_error:
|
||||
if getattr(agent, "_last_compression_summary_warning", None) != summary_error:
|
||||
|
|
@ -2006,7 +2103,11 @@ def compress_context(
|
|||
# file dedup) ran. A concurrent path that wakes up the moment we
|
||||
# release will see the NEW session_id in state.db / SessionEntry and
|
||||
# acquire on that — no race against our just-finished work.
|
||||
_release_lock()
|
||||
try:
|
||||
_release_lock()
|
||||
finally:
|
||||
if _commit_fence_entered:
|
||||
commit_fence.finish_commit()
|
||||
|
||||
|
||||
def _compress_context_via_codex_app_server(
|
||||
|
|
|
|||
150
gateway/run.py
150
gateway/run.py
|
|
@ -6644,6 +6644,44 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# loop is never blocked; mirrors the /new reset path's fix (#35994).
|
||||
_CLEANUP_TIMEOUT_S = 30.0
|
||||
|
||||
def _defer_agent_cleanup_until_future_done(
|
||||
self,
|
||||
future: asyncio.Future,
|
||||
agent: Any,
|
||||
*,
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Clean up ``agent`` only after its executor future has finished.
|
||||
|
||||
A timed-out executor call keeps running in its worker thread. Closing
|
||||
the agent before that thread exits can tear down clients or providers
|
||||
it is still using. Keep a strong task reference and wait for the real
|
||||
future before invoking the normal bounded, off-loop cleanup path.
|
||||
"""
|
||||
|
||||
async def _cleanup_when_done() -> None:
|
||||
try:
|
||||
await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
# Loop shutdown can cancel this waiter while the executor still
|
||||
# runs. Never turn that cancellation into premature cleanup.
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Deferred agent worker%s finished with an error: %s",
|
||||
f" ({context})" if context else "",
|
||||
exc,
|
||||
)
|
||||
await self._cleanup_agent_resources_off_loop(agent, context=context)
|
||||
|
||||
task = asyncio.create_task(_cleanup_when_done())
|
||||
tasks = getattr(self, "_deferred_agent_cleanup_tasks", None)
|
||||
if tasks is None:
|
||||
tasks = set()
|
||||
self._deferred_agent_cleanup_tasks = tasks
|
||||
tasks.add(task)
|
||||
task.add_done_callback(tasks.discard)
|
||||
|
||||
async def _cleanup_agent_resources_off_loop(
|
||||
self, agent: Any, *, context: str = ""
|
||||
) -> None:
|
||||
|
|
@ -12694,6 +12732,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_threshold_pct = 0.85
|
||||
_hyg_compression_enabled = True
|
||||
_hyg_hard_msg_limit = 5000
|
||||
_hyg_timeout_seconds = 30.0
|
||||
_hyg_failure_cooldown_seconds = 300.0
|
||||
_hyg_config_context_length = None
|
||||
_hyg_provider = None
|
||||
_hyg_base_url = None
|
||||
|
|
@ -12739,6 +12779,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_hard_msg_limit = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
_raw_timeout = _comp_cfg.get("hygiene_timeout_seconds")
|
||||
if _raw_timeout is not None:
|
||||
try:
|
||||
_parsed = float(_raw_timeout)
|
||||
if _parsed > 0:
|
||||
_hyg_timeout_seconds = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
_raw_cooldown = _comp_cfg.get("hygiene_failure_cooldown_seconds")
|
||||
if _raw_cooldown is not None:
|
||||
try:
|
||||
_parsed = float(_raw_cooldown)
|
||||
if _parsed >= 0:
|
||||
_hyg_failure_cooldown_seconds = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
_hyg_configured_model = _hyg_model
|
||||
_hyg_configured_provider = _hyg_provider
|
||||
|
|
@ -12849,6 +12905,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
or _msg_count >= _HARD_MSG_LIMIT
|
||||
)
|
||||
|
||||
if _needs_compress:
|
||||
_cooldowns = getattr(self, "_hygiene_compression_failure_cooldowns", None)
|
||||
if _cooldowns is None:
|
||||
_cooldowns = {}
|
||||
self._hygiene_compression_failure_cooldowns = _cooldowns
|
||||
_cooldown_key = session_entry.session_id
|
||||
_cooldown_until = float(_cooldowns.get(_cooldown_key) or 0.0)
|
||||
if _cooldown_until > time.time():
|
||||
logger.info(
|
||||
"Session hygiene: skipping compression for %s; "
|
||||
"previous failure cooldown active for %.1fs",
|
||||
_cooldown_key,
|
||||
max(0.0, _cooldown_until - time.time()),
|
||||
)
|
||||
_needs_compress = False
|
||||
|
||||
if _needs_compress:
|
||||
logger.info(
|
||||
"Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing "
|
||||
|
|
@ -12862,6 +12934,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
|
||||
|
||||
try:
|
||||
from agent.conversation_compression import CompressionCommitFence
|
||||
from run_agent import AIAgent
|
||||
|
||||
_hyg_model, _hyg_runtime = self._resolve_session_agent_runtime(
|
||||
|
|
@ -12896,6 +12969,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
session_id=session_entry.session_id,
|
||||
session_db=_hyg_session_db,
|
||||
)
|
||||
_hyg_cleanup_deferred = False
|
||||
try:
|
||||
# Gateway hygiene runs before the user turn
|
||||
# starts and already owns the session binding.
|
||||
|
|
@ -12923,13 +12997,76 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_agent._print_fn = lambda *a, **kw: None
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
_compressed, _ = await loop.run_in_executor(
|
||||
_hyg_commit_fence = CompressionCommitFence()
|
||||
_hyg_future = loop.run_in_executor(
|
||||
None,
|
||||
lambda: _hyg_agent._compress_context(
|
||||
_hyg_msgs, "",
|
||||
approx_tokens=_approx_tokens,
|
||||
commit_fence=_hyg_commit_fence,
|
||||
),
|
||||
)
|
||||
try:
|
||||
_compressed, _ = await asyncio.wait_for(
|
||||
asyncio.shield(_hyg_future),
|
||||
timeout=_hyg_timeout_seconds,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
_cancelled = None
|
||||
while _cancelled is None:
|
||||
_cancelled = (
|
||||
_hyg_commit_fence.try_cancel_before_commit()
|
||||
)
|
||||
if _cancelled is None:
|
||||
await asyncio.sleep(0.001)
|
||||
if not _cancelled:
|
||||
# The worker crossed the commit boundary just
|
||||
# before the timeout. The fence poll waited for
|
||||
# that boundary to finish, so consume the
|
||||
# completed result instead of treating a
|
||||
# successful compaction as a timeout.
|
||||
_compressed, _ = await _hyg_future
|
||||
else:
|
||||
self._defer_agent_cleanup_until_future_done(
|
||||
_hyg_future,
|
||||
_hyg_agent,
|
||||
context="session hygiene timeout",
|
||||
)
|
||||
_hyg_cleanup_deferred = True
|
||||
if _hyg_failure_cooldown_seconds >= 0:
|
||||
self._hygiene_compression_failure_cooldowns[
|
||||
session_entry.session_id
|
||||
] = time.time() + _hyg_failure_cooldown_seconds
|
||||
logger.warning(
|
||||
"Session hygiene compression for session %s "
|
||||
"timed out after %.1fs; continuing without "
|
||||
"compression",
|
||||
session_entry.session_id,
|
||||
_hyg_timeout_seconds,
|
||||
)
|
||||
_timeout_msg = (
|
||||
"⚠️ Context compression timed out "
|
||||
f"after {_hyg_timeout_seconds:.1f}s. "
|
||||
"No messages were dropped — continuing without "
|
||||
"compression. Run /compress to retry, /reset for "
|
||||
"a clean session, or check your "
|
||||
"auxiliary.compression model configuration."
|
||||
)
|
||||
try:
|
||||
_adapter = self.adapters.get(source.platform)
|
||||
if _adapter and source.chat_id:
|
||||
await _adapter.send(
|
||||
source.chat_id,
|
||||
_timeout_msg,
|
||||
metadata=_hyg_meta,
|
||||
)
|
||||
except Exception as _werr:
|
||||
logger.warning(
|
||||
"Failed to deliver compression-timeout "
|
||||
"warning to user: %s",
|
||||
_werr,
|
||||
)
|
||||
raise
|
||||
|
||||
# _compress_context ends the old session and creates
|
||||
# a new session_id. Write compressed messages into
|
||||
|
|
@ -13037,6 +13174,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# fresh.
|
||||
_comp = getattr(_hyg_agent, "context_compressor", None)
|
||||
if _comp is not None and getattr(_comp, "_last_compress_aborted", False):
|
||||
if _hyg_failure_cooldown_seconds >= 0:
|
||||
self._hygiene_compression_failure_cooldowns[
|
||||
session_entry.session_id
|
||||
] = time.time() + _hyg_failure_cooldown_seconds
|
||||
_err = getattr(_comp, "_last_summary_error", None) or "unknown error"
|
||||
# Force-redact: provider exception text
|
||||
# may contain credentials; this message
|
||||
|
|
@ -13089,9 +13230,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# rebuilds its system prompt from current
|
||||
# SOUL.md, memory, and skills.
|
||||
self._evict_cached_agent(session_key)
|
||||
await self._cleanup_agent_resources_off_loop(
|
||||
_hyg_agent, context="session hygiene"
|
||||
)
|
||||
if not _hyg_cleanup_deferred:
|
||||
await self._cleanup_agent_resources_off_loop(
|
||||
_hyg_agent, context="session hygiene"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
|
|
|
|||
|
|
@ -1415,6 +1415,8 @@ DEFAULT_CONFIG = {
|
|||
# rounds cannot clear the request estimate.
|
||||
# Validated >= 1, hard-capped at 10.
|
||||
"hygiene_hard_message_limit": 5000, # gateway session-hygiene force-compress threshold by message count
|
||||
"hygiene_timeout_seconds": 30, # max seconds gateway waits for pre-agent hygiene compression
|
||||
"hygiene_failure_cooldown_seconds": 300, # skip repeated failed hygiene attempts for this session
|
||||
"protect_first_n": 3, # non-system head messages always preserved
|
||||
# verbatim, in ADDITION to the system prompt
|
||||
# (which is always implicitly protected). Set to
|
||||
|
|
|
|||
|
|
@ -6341,6 +6341,7 @@ class AIAgent:
|
|||
focus_topic: str = None,
|
||||
force: bool = False,
|
||||
defer_context_engine_notification: bool = False,
|
||||
commit_fence=None,
|
||||
) -> tuple:
|
||||
"""Forwarder — see ``agent.conversation_compression.compress_context``.
|
||||
|
||||
|
|
@ -6355,6 +6356,7 @@ class AIAgent:
|
|||
approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic,
|
||||
force=force,
|
||||
defer_context_engine_notification=defer_context_engine_notification,
|
||||
commit_fence=commit_fence,
|
||||
)
|
||||
|
||||
def _set_tool_guardrail_halt(self, decision: ToolGuardrailDecision) -> None:
|
||||
|
|
|
|||
|
|
@ -345,6 +345,97 @@ def test_skipped_compression_returns_messages_unchanged(tmp_path: Path) -> None:
|
|||
agent.context_compressor.compress.assert_not_called()
|
||||
|
||||
|
||||
def test_cancelled_commit_fence_blocks_late_session_db_compaction(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A worker cancelled during summarization must not mutate SessionDB later."""
|
||||
from agent.conversation_compression import CompressionCommitFence
|
||||
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
session_id = "HYGIENE_TIMEOUT_SESSION"
|
||||
db.create_session(session_id, source="telegram")
|
||||
|
||||
agent = _build_agent_with_db(db, session_id)
|
||||
agent.compression_in_place = True
|
||||
agent._cached_system_prompt = "sys"
|
||||
agent._last_compaction_in_place = True
|
||||
summary_started = threading.Event()
|
||||
release_summary = threading.Event()
|
||||
|
||||
def _slow_summary(*_args, **_kwargs):
|
||||
summary_started.set()
|
||||
assert release_summary.wait(timeout=5)
|
||||
return [
|
||||
{"role": "user", "content": "[CONTEXT COMPACTION] summary"},
|
||||
{"role": "user", "content": "tail"},
|
||||
]
|
||||
|
||||
agent.context_compressor.compress.side_effect = _slow_summary
|
||||
archive_spy = MagicMock(wraps=db.archive_and_compact)
|
||||
db.archive_and_compact = archive_spy
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
|
||||
fence = CompressionCommitFence()
|
||||
result = {}
|
||||
errors = []
|
||||
|
||||
def _run_compression() -> None:
|
||||
try:
|
||||
result["value"] = agent._compress_context(
|
||||
messages,
|
||||
"sys",
|
||||
approx_tokens=120_000,
|
||||
commit_fence=fence,
|
||||
)
|
||||
except BaseException as exc: # pragma: no cover - surfaced below
|
||||
errors.append(exc)
|
||||
|
||||
worker = threading.Thread(target=_run_compression, name="timed-out-hygiene")
|
||||
worker.start()
|
||||
assert summary_started.wait(timeout=2)
|
||||
|
||||
assert fence.cancel_before_commit() is True
|
||||
release_summary.set()
|
||||
worker.join(timeout=5)
|
||||
|
||||
assert not worker.is_alive()
|
||||
assert errors == []
|
||||
compressed, _prompt = result["value"]
|
||||
assert compressed is messages
|
||||
assert agent.session_id == session_id
|
||||
assert agent._last_compaction_in_place is False
|
||||
archive_spy.assert_not_called()
|
||||
assert db.get_compression_lock_holder(session_id) is None
|
||||
|
||||
|
||||
def test_commit_fence_waits_for_an_active_commit() -> None:
|
||||
"""A timeout that loses the fence race cannot overlap the live turn."""
|
||||
from agent.conversation_compression import CompressionCommitFence
|
||||
|
||||
fence = CompressionCommitFence()
|
||||
assert fence.begin_commit() is True
|
||||
assert fence.try_cancel_before_commit() is None
|
||||
cancel_started = threading.Event()
|
||||
cancel_finished = threading.Event()
|
||||
result = {}
|
||||
|
||||
def _cancel() -> None:
|
||||
cancel_started.set()
|
||||
result["cancelled"] = fence.cancel_before_commit()
|
||||
cancel_finished.set()
|
||||
|
||||
waiter = threading.Thread(target=_cancel, name="hygiene-timeout-fence")
|
||||
waiter.start()
|
||||
try:
|
||||
assert cancel_started.wait(timeout=2)
|
||||
assert not cancel_finished.is_set()
|
||||
finally:
|
||||
fence.finish_commit()
|
||||
waiter.join(timeout=2)
|
||||
|
||||
assert not waiter.is_alive()
|
||||
assert result["cancelled"] is False
|
||||
|
||||
|
||||
def test_compression_restores_user_turn_when_compressor_drops_all_users(tmp_path: Path) -> None:
|
||||
"""Provider chat templates need at least one user message after compaction.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,11 @@ The hygiene system uses the SAME compression config as the agent:
|
|||
so CLI and messaging platforms behave identically.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -593,6 +596,240 @@ async def test_session_hygiene_preserves_transcript_when_in_place_configured_but
|
|||
runner.session_store.rewrite_transcript.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_hygiene_skips_compression_during_failure_cooldown(monkeypatch, tmp_path):
|
||||
"""After a hygiene compression failure, the next message should not block
|
||||
on the same doomed auxiliary compression path again until cooldown expires."""
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_dotenv.load_dotenv = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv)
|
||||
|
||||
class ShouldNotRunCompressAgent:
|
||||
last_instance = None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
type(self).last_instance = self
|
||||
self.session_id = kwargs.get("session_id", "fake-session")
|
||||
self.shutdown_memory_provider = MagicMock()
|
||||
self.close = MagicMock()
|
||||
|
||||
def _compress_context(self, messages, *_args, **_kwargs):
|
||||
raise AssertionError("compression should be skipped during cooldown")
|
||||
|
||||
fake_run_agent = types.ModuleType("run_agent")
|
||||
fake_run_agent.AIAgent = ShouldNotRunCompressAgent
|
||||
monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent)
|
||||
|
||||
gateway_run = importlib.import_module("gateway.run")
|
||||
GatewayRunner = gateway_run.GatewayRunner
|
||||
|
||||
runner = object.__new__(GatewayRunner)
|
||||
runner.config = GatewayConfig(
|
||||
platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token")}
|
||||
)
|
||||
runner.adapters = {Platform.TELEGRAM: HygieneCaptureAdapter()}
|
||||
runner._voice_mode = {}
|
||||
runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False)
|
||||
runner.session_store = MagicMock()
|
||||
runner.session_store.get_or_create_session.return_value = SessionEntry(
|
||||
session_key="agent:main:telegram:dm:12345",
|
||||
session_id="sess-1",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_type="dm",
|
||||
)
|
||||
runner.session_store.load_transcript.return_value = _make_history(6, content_size=400)
|
||||
runner.session_store.has_any_sessions.return_value = True
|
||||
runner.session_store.rewrite_transcript = MagicMock()
|
||||
runner.session_store.append_to_transcript = MagicMock()
|
||||
runner._running_agents = {}
|
||||
runner._pending_messages = {}
|
||||
runner._pending_approvals = {}
|
||||
runner._session_db = None
|
||||
runner._hygiene_compression_failure_cooldowns = {"sess-1": time.time() + 300}
|
||||
runner._is_user_authorized = lambda _source: True
|
||||
runner._set_session_env = lambda _context: None
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"final_response": "ok",
|
||||
"messages": [],
|
||||
"tools": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"})
|
||||
monkeypatch.setattr(
|
||||
"agent.model_metadata.get_model_context_length",
|
||||
lambda *_args, **_kwargs: 100,
|
||||
)
|
||||
|
||||
event = MessageEvent(
|
||||
text="hello",
|
||||
source=SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="12345",
|
||||
chat_type="dm",
|
||||
user_id="12345",
|
||||
),
|
||||
message_id="1",
|
||||
)
|
||||
|
||||
result = await runner._handle_message(event)
|
||||
|
||||
assert result == "ok"
|
||||
assert ShouldNotRunCompressAgent.last_instance is None
|
||||
runner._run_agent.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_hygiene_timeout_continues_to_agent_and_sets_cooldown(monkeypatch, tmp_path):
|
||||
"""A timed-out SessionDB-bound worker cannot compact after the live turn starts.
|
||||
|
||||
The worker remains alive long enough to cross the old race window. The
|
||||
timeout must fence its eventual commit, continue to the live agent, and
|
||||
clean up the temporary agent only after the worker actually returns.
|
||||
"""
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_dotenv.load_dotenv = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv)
|
||||
|
||||
worker_started = threading.Event()
|
||||
release_worker = threading.Event()
|
||||
cleanup_done = threading.Event()
|
||||
fake_db = MagicMock()
|
||||
|
||||
class SlowCompressAgent:
|
||||
last_instance = None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.session_id = kwargs.get("session_id", "fake-session")
|
||||
self._session_db = kwargs.get("session_db")
|
||||
self._last_compaction_in_place = False
|
||||
self.context_compressor = SimpleNamespace(
|
||||
bind_session_state=MagicMock(),
|
||||
_last_compress_aborted=False,
|
||||
_last_aux_model_failure_model=None,
|
||||
)
|
||||
self.shutdown_memory_provider = MagicMock()
|
||||
self.close = MagicMock(side_effect=cleanup_done.set)
|
||||
type(self).last_instance = self
|
||||
|
||||
def _compress_context(
|
||||
self, messages, *_args, commit_fence=None, **_kwargs
|
||||
):
|
||||
worker_started.set()
|
||||
assert release_worker.wait(timeout=2)
|
||||
if commit_fence is not None and not commit_fence.begin_commit():
|
||||
return (messages, None)
|
||||
try:
|
||||
self._session_db.archive_and_compact(
|
||||
self.session_id,
|
||||
[{"role": "assistant", "content": "too late"}],
|
||||
)
|
||||
self._last_compaction_in_place = True
|
||||
return ([{"role": "assistant", "content": "too late"}], None)
|
||||
finally:
|
||||
if commit_fence is not None:
|
||||
commit_fence.finish_commit()
|
||||
|
||||
fake_run_agent = types.ModuleType("run_agent")
|
||||
fake_run_agent.AIAgent = SlowCompressAgent
|
||||
monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent)
|
||||
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(
|
||||
"compression:\n"
|
||||
" enabled: true\n"
|
||||
" hygiene_timeout_seconds: 0.01\n"
|
||||
" hygiene_failure_cooldown_seconds: 120\n"
|
||||
)
|
||||
|
||||
gateway_run = importlib.import_module("gateway.run")
|
||||
GatewayRunner = gateway_run.GatewayRunner
|
||||
|
||||
adapter = HygieneCaptureAdapter()
|
||||
runner = object.__new__(GatewayRunner)
|
||||
runner.config = GatewayConfig(
|
||||
platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token")}
|
||||
)
|
||||
runner.adapters = {Platform.TELEGRAM: adapter}
|
||||
runner._voice_mode = {}
|
||||
runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False)
|
||||
runner.session_store = MagicMock()
|
||||
runner.session_store.get_or_create_session.return_value = SessionEntry(
|
||||
session_key="agent:main:telegram:dm:12345",
|
||||
session_id="sess-timeout",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_type="dm",
|
||||
)
|
||||
runner.session_store.load_transcript.return_value = _make_history(6, content_size=400)
|
||||
runner.session_store.has_any_sessions.return_value = True
|
||||
runner.session_store.rewrite_transcript = MagicMock()
|
||||
runner.session_store.append_to_transcript = MagicMock()
|
||||
runner._running_agents = {}
|
||||
runner._pending_messages = {}
|
||||
runner._pending_approvals = {}
|
||||
runner._session_db = SimpleNamespace(_db=fake_db)
|
||||
runner._is_user_authorized = lambda _source: True
|
||||
runner._set_session_env = lambda _context: None
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"final_response": "ok",
|
||||
"messages": [],
|
||||
"tools": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"})
|
||||
monkeypatch.setattr(
|
||||
"agent.model_metadata.get_model_context_length",
|
||||
lambda *_args, **_kwargs: 100,
|
||||
)
|
||||
|
||||
event = MessageEvent(
|
||||
text="hello",
|
||||
source=SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="12345",
|
||||
chat_type="dm",
|
||||
user_id="12345",
|
||||
),
|
||||
message_id="1",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
result = await runner._handle_message(event)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert result == "ok"
|
||||
assert elapsed < 0.15
|
||||
assert worker_started.is_set()
|
||||
assert runner._run_agent.await_count == 1
|
||||
assert runner._hygiene_compression_failure_cooldowns["sess-timeout"] > time.time()
|
||||
timeout_warnings = [s for s in adapter.sent if "Context compression timed out" in s["content"]]
|
||||
assert len(timeout_warnings) == 1
|
||||
fake_db.archive_and_compact.assert_not_called()
|
||||
SlowCompressAgent.last_instance.close.assert_not_called()
|
||||
|
||||
release_worker.set()
|
||||
await asyncio.wait_for(asyncio.to_thread(cleanup_done.wait), timeout=2)
|
||||
|
||||
# The late worker observed cancellation at the commit fence, so it never
|
||||
# mutated the live session after the new turn began. Cleanup still ran once
|
||||
# it was safe to tear down the helper agent's clients/providers.
|
||||
fake_db.archive_and_compact.assert_not_called()
|
||||
SlowCompressAgent.last_instance.close.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_hygiene_warns_user_when_compression_aborts(monkeypatch, tmp_path):
|
||||
"""When auxiliary compression's summary LLM call fails, the compressor
|
||||
|
|
|
|||
|
|
@ -750,6 +750,8 @@ compression:
|
|||
protect_first_n: 3 # Non-system head messages pinned across compactions (0 = pin nothing)
|
||||
idle_compact_after_seconds: 0 # Opt-in idle compaction (0 = disabled) — see below
|
||||
hygiene_hard_message_limit: 5000 # Gateway safety valve — see below
|
||||
hygiene_timeout_seconds: 30 # Max seconds gateway waits for pre-agent hygiene compression
|
||||
hygiene_failure_cooldown_seconds: 300 # Skip repeated failed hygiene attempts for this session
|
||||
|
||||
# The summarization model/provider is configured under auxiliary:
|
||||
auxiliary:
|
||||
|
|
@ -765,6 +767,10 @@ Older configs with `compression.summary_model`, `compression.summary_provider`,
|
|||
|
||||
`hygiene_hard_message_limit` is a gateway-only **pre-compression safety valve**. It exists to break a death spiral: when API calls keep disconnecting on an oversized session, the gateway never receives token-usage data, so the token-based threshold can't fire, so the transcript keeps growing and disconnects get worse. This count-based floor fires on message count alone (always known, regardless of API failures) to force compression and recover the session. Default `5000` — far above any normal session, including large-context (1M+) models doing thousands of short turns, which compress on the token threshold long before this. Raise it further for unusual platforms, lower it to force more aggressive compression. Editing this value on a running gateway takes effect on the next message (see below).
|
||||
|
||||
`hygiene_timeout_seconds` caps how long the gateway waits for this pre-agent compression pass. If the auxiliary compression backend is down or very slow, the gateway warns the user, continues the incoming message without compression, and records a temporary per-session failure cooldown instead of appearing stuck.
|
||||
|
||||
`hygiene_failure_cooldown_seconds` controls that per-session cooldown after a hygiene compression timeout or abort. During the cooldown, the gateway skips repeated hygiene attempts for the same oversized session so every incoming message does not block on the same broken auxiliary backend. `/compress`, `/reset`, or a healthy later turn can still recover the session.
|
||||
|
||||
`protect_first_n` controls how many **non-system** head messages are pinned across every compaction. Default `3` — the opening user/assistant exchange survives every summarizer pass so the original goal stays visible. On long-running rolling-compaction sessions where the opening turn is no longer relevant, set `protect_first_n: 0` to pin nothing but the system prompt + summary + tail. The system prompt itself is always preserved regardless of this setting.
|
||||
|
||||
`threshold_tokens` sets an optional **absolute token cap** for the compression trigger. When set, compression fires at the lower of the ratio-based `threshold` and this absolute count — so compression never fires later than the user's preferred token number regardless of which model is active. This solves the problem where switching between models with different context windows (e.g. 1M → 400K) shifts the absolute trigger point. The cap is clamped to the model's context length, so setting it higher than the model supports is safe — the ratio-based threshold is used instead. Default `null` (disabled — ratio-based threshold only). The cap survives model switches and fallback activations.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue