Merge origin/main into feat/hermes-relay-shared-metrics

Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
Alex Fournier 2026-07-26 09:21:19 -07:00
commit 45580cc93a
1114 changed files with 96474 additions and 8371 deletions

2
.gitignore vendored
View file

@ -1,6 +1,8 @@
.DS_Store
/venv/
/venv.old/
/venv.stale.runtime-*/
/.hermes-runtime/
/_pycache/
*.pyc*
__pycache__/

View file

@ -32,6 +32,7 @@ else:
import argparse
import asyncio
import logging
import os
import sys
from pathlib import Path
from hermes_constants import get_hermes_home
@ -251,11 +252,13 @@ def main(argv: list[str] | None = None) -> None:
# MCP servers dynamically via asyncio.to_thread inside the event
# loop; that path is unaffected.) Moved from model_tools.py module
# scope to avoid freezing the gateway's loop on lazy import (#16856).
try:
from tools.mcp_tool import discover_mcp_tools
discover_mcp_tools()
except Exception:
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
# Metadata-only hosts can opt out of unrelated global MCP startup.
if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1":
try:
from tools.mcp_tool import discover_mcp_tools
discover_mcp_tools()
except Exception:
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
agent = HermesACPAgent()
try:

View file

@ -85,6 +85,110 @@ from tools.approval import (
logger = logging.getLogger(__name__)
def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]:
"""Return ``(slug, label, [(model_id, description), ...])`` for named endpoints.
Covers both the v12 ``providers:`` mapping and the legacy
``custom_providers:`` list. These endpoints never appear in canonical
provider enumeration, so without this the ACP model selector hides every
named endpoint that the TUI ``/model`` picker already renders (#47039
implemented named-endpoint rows for the TUI surface only).
Model lists come from the entry's declared models (``default_model`` +
``models``), refreshed from the endpoint's live ``/models`` listing when a
credential is available and ``discover_models`` is not disabled. Declared
models are kept even when live discovery fails some OpenAI-compatible
endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at
all yet serve the declared models fine.
Slugs use the ``custom:<name>`` shape that ``parse_model_input`` and
``resolve_runtime_provider`` already resolve, so encoded choice ids
(``custom:<name>:<model>``) round-trip through ``set_session_model``
unchanged.
"""
try:
from hermes_cli.config import (
get_compatible_custom_providers,
is_provider_enabled,
load_config,
)
from hermes_cli.models import fetch_api_models
except ImportError:
return []
try:
cfg = load_config()
entries = get_compatible_custom_providers(cfg)
except Exception:
logger.debug("Could not load named custom providers", exc_info=True)
return []
# ``get_compatible_custom_providers`` drops the ``enabled`` flag during
# normalization, so collect explicitly disabled provider keys from the
# raw config and skip their entries below.
disabled_keys: set[str] = set()
raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None
if isinstance(raw_providers, dict):
for raw_key, raw_entry in raw_providers.items():
if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry):
disabled_keys.add(str(raw_key).strip().lower())
catalogs: list[tuple[str, str, list[tuple[str, str]]]] = []
for entry in entries:
if not isinstance(entry, dict):
continue
provider_key = str(entry.get("provider_key", "") or "").strip()
if provider_key.lower() in disabled_keys:
continue
name = str(entry.get("name", "") or "").strip()
base_url = str(entry.get("base_url", "") or "").strip()
if not name or not base_url:
continue
slug_source = provider_key or name
slug = "custom:" + slug_source.strip().lower().replace(" ", "-")
api_key = str(entry.get("api_key", "") or "").strip()
if not api_key:
key_env = str(entry.get("key_env", "") or "").strip()
api_key = os.environ.get(key_env, "").strip() if key_env else ""
declared: list[str] = []
default_model = str(entry.get("model", "") or "").strip()
if default_model:
declared.append(default_model)
models_cfg = entry.get("models")
if isinstance(models_cfg, dict):
for mid in models_cfg:
mid = str(mid or "").strip()
if mid and mid not in declared:
declared.append(mid)
if not api_key and not declared:
# No credential to discover with and nothing declared:
# not addressable from the selector.
continue
model_ids = list(declared)
discover = entry.get("discover_models", True)
if isinstance(discover, str):
discover = discover.lower() not in {"false", "no", "0"}
if discover and api_key:
try:
live = fetch_api_models(
api_key, base_url, api_mode=entry.get("api_mode")
)
except Exception:
live = None
if live:
model_ids = declared + [m for m in live if m not in declared]
if not model_ids:
continue
catalogs.append((slug, name, [(mid, "") for mid in model_ids]))
return catalogs
try:
from hermes_cli import __version__ as HERMES_VERSION
except Exception:
@ -97,6 +201,13 @@ _executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="acp-agent")
# does not expose a client-side limit, so this is a fixed cap that clients
# paginate against using `cursor` / `next_cursor`.
_LIST_SESSIONS_PAGE_SIZE = 50
# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render
# the whole `availableModels` array in one dropdown, so an unbounded
# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker
# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not
# the total; aggregator providers stay intentionally uncapped inside the shared
# inventory, and the current model is always kept via the fallback insert below.
ACP_MAX_MODELS_PER_PROVIDER = 200
_MAX_ACP_RESOURCE_BYTES = 512 * 1024
_TEXT_RESOURCE_MIME_PREFIXES = ("text/",)
_TEXT_RESOURCE_MIME_TYPES = {
@ -585,46 +696,108 @@ class HermesACPAgent(acp.Agent):
return f"{raw_provider}:{raw_model}"
def _build_model_state(self, state: SessionState) -> SessionModelState | None:
"""Return the ACP model selector payload for editors like Zed."""
"""Return authenticated providers and their models for ACP clients.
The shared Hermes inventory is also used by ``hermes model``, the TUI,
and the dashboard. Keeping ACP on that substrate prevents its selector
from silently collapsing to the current provider's curated list.
"""
model = str(state.model or getattr(state.agent, "model", "") or "").strip()
provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter"
try:
from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label
from hermes_cli.inventory import build_models_payload, load_picker_context
from hermes_cli.models import normalize_provider, provider_label
normalized_provider = normalize_provider(provider)
provider_name = provider_label(normalized_provider)
context = load_picker_context().with_overrides(
current_provider=normalized_provider,
current_model=model,
current_base_url=str(getattr(state.agent, "base_url", "") or ""),
)
payload = build_models_payload(
context,
explicit_only=True,
include_unconfigured=False,
picker_hints=False,
canonical_order=True,
pricing=False,
capabilities=False,
refresh=False,
probe_custom_providers=False,
probe_current_custom_provider=False,
max_models=ACP_MAX_MODELS_PER_PROVIDER,
)
available_models: list[ModelInfo] = []
seen_ids: set[str] = set()
for model_id, description in curated_models_for_provider(normalized_provider):
rendered_model = str(model_id or "").strip()
if not rendered_model:
for row in payload.get("providers") or []:
row_provider = normalize_provider(str(row.get("slug") or "").strip())
if not row_provider:
continue
choice_id = self._encode_model_choice(normalized_provider, rendered_model)
if choice_id in seen_ids:
continue
desc_parts = [f"Provider: {provider_name}"]
if description:
desc_parts.append(str(description).strip())
if rendered_model == model:
desc_parts.append("current")
available_models.append(
ModelInfo(
model_id=choice_id,
name=rendered_model,
description="".join(part for part in desc_parts if part),
)
provider_name = str(row.get("name") or "").strip() or provider_label(
row_provider
)
seen_ids.add(choice_id)
for model_entry in row.get("models") or []:
if isinstance(model_entry, dict):
rendered_model = str(
model_entry.get("id")
or model_entry.get("model")
or model_entry.get("name")
or ""
).strip()
else:
rendered_model = str(model_entry or "").strip()
if not rendered_model:
continue
choice_id = self._encode_model_choice(row_provider, rendered_model)
if choice_id in seen_ids:
continue
is_current = (
row_provider == normalized_provider and rendered_model == model
)
description = f"Provider: {provider_name}"
if is_current:
description += " • current"
available_models.append(
ModelInfo(
model_id=choice_id,
name=f"{provider_name} · {rendered_model}",
description=description,
)
)
seen_ids.add(choice_id)
# Named user-defined endpoints (providers: / custom_providers:)
# are invisible to canonical provider enumeration — append them
# so editor clients can select them like the TUI /model picker.
for named_slug, named_label, named_catalog in _named_custom_provider_catalogs():
for named_model, named_desc in named_catalog:
named_choice = self._encode_model_choice(named_slug, named_model)
if not named_choice or named_choice in seen_ids:
continue
named_parts = [f"Provider: {named_label}"]
if named_desc:
named_parts.append(str(named_desc).strip())
if named_slug == normalized_provider and named_model == model:
named_parts.append("current")
available_models.append(
ModelInfo(
model_id=named_choice,
name=named_model,
description="".join(part for part in named_parts if part),
)
)
seen_ids.add(named_choice)
current_model_id = self._encode_model_choice(normalized_provider, model)
if current_model_id and current_model_id not in seen_ids:
provider_name = provider_label(normalized_provider)
available_models.insert(
0,
ModelInfo(
model_id=current_model_id,
name=model,
name=f"{provider_name} · {model}",
description=f"Provider: {provider_name} • current",
),
)
@ -1588,7 +1761,16 @@ class HermesACPAgent(acp.Agent):
clear_session_vars,
set_session_vars,
)
session_tokens = set_session_vars(session_key=session_id)
# ``cwd`` pins the logical working directory for this context,
# which is what the system prompt's "Current working directory"
# line reports (agent/prompt_builder.py -> resolve_agent_cwd).
# Without it the prompt advertises the global Hermes workspace
# while the tools are rooted at the client's project, so the
# model emits absolute paths under ~/.hermes/workspace and the
# edit silently lands outside the editor's workspace.
session_tokens = set_session_vars(
session_key=session_id, cwd=state.cwd,
)
except Exception:
session_tokens = None
clear_session_vars = None # type: ignore[assignment]
@ -1875,8 +2057,26 @@ class HermesACPAgent(acp.Agent):
if handler is None:
return None # not a known command — let the LLM handle it
try:
# Slash handlers run on the event-loop thread, OUTSIDE the per-turn
# contextvars.copy_context() that pins the session cwd for the agent
# call. ``/compress`` and ``/model`` reach code that REBUILDS the
# system prompt (agent._build_system_prompt -> resolve_agent_cwd), so
# an unpinned handler bakes the Hermes install tree into the session's
# cached prompt — persisted, and therefore poisoning every later turn
# even though the turn itself is pinned. Pin inside a fresh context so
# the write can't leak into other concurrent ACP sessions and needs no
# teardown.
def _dispatch() -> str | None:
try:
from agent.runtime_cwd import set_session_cwd
set_session_cwd(state.cwd)
except Exception:
logger.debug("Could not pin ACP session cwd for slash command", exc_info=True)
return handler(args, state)
try:
return contextvars.copy_context().run(_dispatch)
except Exception as e:
logger.error("Slash command /%s error: %s", cmd, e, exc_info=True)
return f"Error executing /{cmd}: {e}"

View file

@ -69,6 +69,43 @@ def _ra():
return run_agent
def _moa_reference_output_allowed(agent: Any) -> bool:
"""Keep MoA display events off only the machine-readable ``-Q`` surface."""
return not (
getattr(agent, "platform", None) == "cli"
and getattr(agent, "tool_progress_mode", "all") == "off"
)
def _relay_moa_reference_event(agent: Any, event: str, **kwargs: Any) -> None:
"""Relay MoA display events while preserving the ``-Q`` stdout contract."""
if not _moa_reference_output_allowed(agent):
return
cb = getattr(agent, "tool_progress_callback", None)
if cb is None:
return
try:
if event == "moa.reference":
cb(
"moa.reference",
str(kwargs.get("label") or ""),
str(kwargs.get("text") or ""),
None,
moa_index=kwargs.get("index"),
moa_count=kwargs.get("count"),
)
elif event == "moa.aggregating":
cb(
"moa.aggregating",
str(kwargs.get("aggregator") or ""),
None,
None,
moa_ref_count=kwargs.get("ref_count"),
)
except Exception:
pass
def _normalize_route_base_url(base_url: Any) -> str:
"""Canonicalize an endpoint URL for model-route identity comparisons."""
return normalize_route_base_url(base_url)
@ -786,9 +823,10 @@ def init_agent(
# Anthropic prompt caching: auto-enabled for Claude models on native
# Anthropic, OpenRouter, and third-party gateways that speak the
# Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces
# input costs by ~75% on multi-turn conversations. Uses system_and_3
# strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy``
# for the layout-vs-transport decision.
# input costs by ~75% on multi-turn conversations. Uses four breakpoints:
# the static system prefix, full system prompt, and last two messages
# (falling back to system-and-3 when no static prefix is available). See
# ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision.
agent._use_prompt_caching, agent._use_native_cache_layout = (
agent._anthropic_prompt_cache_policy()
)
@ -1025,49 +1063,20 @@ def init_agent(
elif isinstance(effective_key, str) and len(effective_key) > 12:
print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}")
elif agent.provider == "moa":
from agent.moa_loop import MoAClient
from agent.moa_loop import build_moa_facade
agent.api_mode = "chat_completions"
# Route reference-model outputs to the agent's tool_progress_callback so
# build_moa_facade wires the reference relay that routes
# reference-model outputs to the agent's tool_progress_callback so
# every surface that already consumes it (CLI spinner/scrollback, TUI,
# desktop, gateway) can show each reference's answer as a labelled block
# before the aggregator acts. The facade emits "moa.reference" and
# "moa.aggregating" events; we forward them through the same callback
# the tool lifecycle uses. Best-effort and cache-safe — these are
# display-only events, they never touch the message history.
def _moa_reference_relay(event: str, **kwargs: Any) -> None:
cb = getattr(agent, "tool_progress_callback", None)
if cb is None:
return
try:
if event == "moa.reference":
label = str(kwargs.get("label") or "")
text = str(kwargs.get("text") or "")
idx = kwargs.get("index")
count = kwargs.get("count")
cb(
"moa.reference",
label,
text,
None,
moa_index=idx,
moa_count=count,
)
elif event == "moa.aggregating":
cb(
"moa.aggregating",
str(kwargs.get("aggregator") or ""),
None,
None,
moa_ref_count=kwargs.get("ref_count"),
)
except Exception:
pass
agent.client = MoAClient(
agent.model or "default",
reference_callback=_moa_reference_relay,
)
# desktop, gateway) can show each reference's answer as a labelled
# block before the aggregator acts. The facade emits "moa.reference",
# "moa.progress", "moa.phase", and "moa.aggregating" events, forwarded
# through the same callback the tool lifecycle uses. Best-effort and
# cache-safe — display-only events, they never touch the message
# history. The factory is shared with the fallback-restore/recovery
# paths so a restored facade keeps emitting these events (#53802).
agent.client = build_moa_facade(agent, agent.model)
agent._client_kwargs = {}
agent.api_key = api_key or "moa-virtual-provider"
agent.base_url = "moa://local"
@ -1333,6 +1342,13 @@ def init_agent(
print("⚠️ Warning: API key appears invalid or missing")
except Exception as e:
raise RuntimeError(f"Failed to initialize OpenAI client: {e}")
# Keep a stable identity for the pool entry that supplied this runtime.
# OAuth refreshes can replace the runtime token before a failed request is
# recovered, so the mutable API-key value alone cannot reliably attribute
# the failure to its source entry.
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
sync_credential_pool_entry_id(agent)
# Provider fallback chain — ordered list of backup providers tried
# when the primary is exhausted (rate-limit, overload, connection
@ -1479,6 +1495,9 @@ def init_agent(
# Cached system prompt -- built once per session, only rebuilt on compression
agent._cached_system_prompt: Optional[str] = None
# Cross-session-stable prefix of the cached prompt. It remains separate
# from the persisted string and is used only to place an early cache marker.
agent._cached_system_prompt_static: Optional[str] = None
# Filesystem checkpoint manager (transparent — not a tool)
from tools.checkpoint_manager import CheckpointManager
@ -1810,6 +1829,28 @@ def init_agent(
compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"}
compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20))
compression_protect_last = int(_compression_cfg.get("protect_last_n", 20))
# Minimum REAL (actionable) user messages guaranteed to survive in the
# uncompressed tail (compression.min_tail_user_messages). Default 1
# preserves current behavior exactly — the existing single-user tail
# anchor. Values > 1 extend the guarantee to the last N actionable
# user turns. Booleans rejected (bool subclasses int), non-int-like
# values fall back to 1, floor at 1.
_raw_min_tail_users = _compression_cfg.get("min_tail_user_messages", 1)
if isinstance(_raw_min_tail_users, bool):
compression_min_tail_users = 1
elif isinstance(_raw_min_tail_users, int):
compression_min_tail_users = _raw_min_tail_users
elif isinstance(_raw_min_tail_users, float):
compression_min_tail_users = (
int(_raw_min_tail_users) if _raw_min_tail_users.is_integer() else 1
)
else:
try:
compression_min_tail_users = int(str(_raw_min_tail_users).strip())
except (TypeError, ValueError):
compression_min_tail_users = 1
if compression_min_tail_users < 1:
compression_min_tail_users = 1
# Cap on compression retry rounds before a turn gives up with "max
# compression attempts reached" (compression.max_attempts). Hardcoding 3
# strands sessions that legitimately need more rounds — e.g. a restart
@ -1838,6 +1879,39 @@ def init_agent(
if compression_max_attempts < 1:
compression_max_attempts = 3
compression_max_attempts = min(compression_max_attempts, 10)
def _parse_prune_int(raw, default):
# Same parser semantics as compression.max_attempts above: reject
# booleans (bool subclasses int — YAML `true` would coerce to 1),
# reject fractional floats rather than truncating them, accept
# integral floats and numeric strings, fall back to the default on
# anything else.
if isinstance(raw, bool):
return default
if isinstance(raw, int):
return raw
if isinstance(raw, float):
return int(raw) if raw.is_integer() else default
try:
return int(str(raw).strip())
except (TypeError, ValueError):
return default
# Opt-in proactive tool-result prune trigger (0 = disabled — the
# default, so an unset key is behavior-neutral). Negative values are
# treated as disabled rather than erroring.
compression_proactive_prune_tokens = max(
0, _parse_prune_int(_compression_cfg.get("proactive_prune_tokens", 0), 0)
)
compression_proactive_prune_min_chars = _parse_prune_int(
_compression_cfg.get("proactive_prune_min_result_chars", 8000), 8000
)
compression_proactive_prune_min_reclaim = max(
0,
_parse_prune_int(
_compression_cfg.get("proactive_prune_min_reclaim_tokens", 4096), 4096
),
)
# protect_first_n is the number of non-system messages to protect at
# the head, in addition to the system prompt (which is always
# implicitly protected by the compressor). Floor at 0 — a value of
@ -2312,6 +2386,10 @@ def init_agent(
max_tokens=agent.max_tokens,
model_thresholds=compression_model_thresholds,
threshold_tokens_cap=compression_threshold_tokens,
proactive_prune_tokens=compression_proactive_prune_tokens,
proactive_prune_min_result_chars=compression_proactive_prune_min_chars,
proactive_prune_min_reclaim_tokens=compression_proactive_prune_min_reclaim,
min_tail_user_messages=compression_min_tail_users,
)
_bind_session_state = getattr(agent.context_compressor, "bind_session_state", None)
if callable(_bind_session_state):

View file

@ -850,6 +850,25 @@ def strip_think_blocks(agent, content: str) -> str:
def sync_credential_pool_entry_id(agent) -> None:
"""Rebind ``agent._credential_pool_entry_id`` from the current pool + key.
OAuth refreshes can replace the runtime token before a failed request is
recovered, so the mutable API-key value alone cannot reliably attribute
the failure to its source entry. This resolves the stable pool-entry ID
for the agent's current ``api_key`` and clears it when no pool is bound.
"""
pool = getattr(agent, "_credential_pool", None)
try:
agent._credential_pool_entry_id = (
pool.entry_id_for_api_key(getattr(agent, "api_key", None))
if pool is not None
else None
)
except Exception:
agent._credential_pool_entry_id = None
def recover_with_credential_pool(
agent,
*,
@ -934,10 +953,30 @@ def recover_with_credential_pool(
# failing entry exactly; fall back to current()'s key only when the agent
# carries no key at all.
_api_key_hint = getattr(agent, "api_key", None) or None
_raw_credential_id = getattr(agent, "_credential_pool_entry_id", None)
_credential_id = (
_raw_credential_id
if isinstance(_raw_credential_id, str) and _raw_credential_id
else None
)
if not _api_key_hint:
_cur = pool.current()
if _cur:
_api_key_hint = getattr(_cur, "runtime_api_key", None)
if not _credential_id:
_current_id = getattr(_cur, "id", None)
if isinstance(_current_id, str) and _current_id:
_credential_id = _current_id
def _rotate_failed_credential(rotate_status: int):
kwargs = {
"status_code": rotate_status,
"error_context": error_context,
"api_key_hint": _api_key_hint,
}
if _credential_id:
kwargs["credential_id"] = _credential_id
return pool.mark_exhausted_and_rotate(**kwargs)
effective_reason = classified_reason
if effective_reason is None:
@ -972,11 +1011,7 @@ def recover_with_credential_pool(
# Runtime credentials can be resolved by a separate pool instance,
# leaving this recovery pool without ``current_id``. Match the key
# that actually failed instead of quarantining a different account.
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
next_entry = _rotate_failed_credential(rotate_status)
if next_entry is not None:
_ra().logger.info(
"Credential %s (billing) — rotated to pool entry %s",
@ -995,8 +1030,13 @@ def recover_with_credential_pool(
# Prefer the entry matching the failing key over the shared current()
# pointer, for the same attribution reason as above.
current_entry = None
if _api_key_hint:
if _credential_id:
current_entry = next(
(e for e in pool.entries() if e.id == _credential_id),
None,
)
if _api_key_hint:
current_entry = current_entry or next(
(e for e in pool.entries() if e.runtime_api_key == _api_key_hint),
None,
)
@ -1009,11 +1049,7 @@ def recover_with_credential_pool(
current_last_status,
)
rotate_status = status_code if status_code is not None else 429
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
next_entry = _rotate_failed_credential(rotate_status)
if next_entry is not None:
_ra().logger.info(
"Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s",
@ -1037,11 +1073,7 @@ def recover_with_credential_pool(
if not has_retried_429 and not usage_limit_reached:
return False, True
rotate_status = status_code if status_code is not None else 429
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
next_entry = _rotate_failed_credential(rotate_status)
if next_entry is not None:
_ra().logger.info(
"Credential %s (rate limit) — rotated to pool entry %s",
@ -1113,7 +1145,10 @@ def recover_with_credential_pool(
# the shared pointer can reference a different, healthy entry, and
# refreshing it would consume that entry's single-use refresh token
# (or mark it exhausted on failure) for a failure it never had.
refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint)
refresh_kwargs = {"api_key_hint": _api_key_hint}
if _credential_id:
refresh_kwargs["credential_id"] = _credential_id
refreshed = pool.try_refresh_matching(**refresh_kwargs)
if refreshed is not None:
# ``try_refresh_matching()`` re-mints a fresh OAuth token and reports
# success even when the upstream keeps rejecting it — a single-entry
@ -1145,11 +1180,7 @@ def recover_with_credential_pool(
# Refresh failed — rotate to next credential instead of giving up.
# The failed entry is already marked exhausted by the refresh attempt.
rotate_status = status_code if status_code is not None else 401
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
next_entry = _rotate_failed_credential(rotate_status)
if next_entry is not None:
_ra().logger.info(
"Credential %s (auth refresh failed) — rotated to pool entry %s",
@ -1194,11 +1225,17 @@ def try_recover_primary_transport(
return False
try:
# Close existing client to release stale connections
# Retire the existing client to release stale connections. #70773:
# never hard-close the shared client here — this runs on the
# conversation-loop thread while workers from stale-killed streaming
# attempts may still be unwinding their SSL BIOs on the old pool.
# ``_retire_shared_openai_client`` shuts the sockets down (FD-safe
# from any thread) and defers the FD release to GC, which cannot
# complete until every borrowing thread has unwound.
if getattr(agent, "client", None) is not None:
try:
agent._close_openai_client(
agent.client, reason="primary_recovery", shared=True,
agent._retire_shared_openai_client(
agent.client, reason="primary_recovery",
)
except Exception:
pass
@ -1225,6 +1262,14 @@ def try_recover_primary_transport(
)
agent._is_anthropic_oauth = rt["is_anthropic_oauth"]
agent.client = None
elif (agent.provider or "").strip().lower() == "moa":
# MoA is a virtual provider with empty client_kwargs — rebuilding
# via _create_openai_client would raise "api_key client option
# must be set". Recreate the facade through the shared factory so
# the reference_callback relay survives recovery (#53802).
from agent.moa_loop import build_moa_facade
agent.client = build_moa_facade(agent, agent.model)
else:
agent.client = agent._create_openai_client(
dict(rt["client_kwargs"]),
@ -1388,7 +1433,18 @@ def restore_primary_runtime(agent) -> bool:
)
# ── Rebuild client for the primary provider ──
if agent.api_mode == "anthropic_messages":
if agent.provider == "moa":
# MoA is a virtual chat-completions provider. It never has real
# OpenAI client kwargs; restoring it after a fallback must recreate
# the facade, not call OpenAI() with an empty api_key. Use the
# shared factory so the restored facade keeps the reference_callback
# relay wired at init — a bare MoAClient() would silently stop
# emitting moa.reference/moa.aggregating display events (#53802).
from agent.moa_loop import build_moa_facade
agent.client = build_moa_facade(agent, agent.model)
agent._anthropic_client = None
elif agent.api_mode == "anthropic_messages":
from agent.anthropic_adapter import build_anthropic_client
agent._anthropic_api_key = rt["anthropic_api_key"]
agent._anthropic_base_url = rt["anthropic_base_url"]
@ -1441,6 +1497,7 @@ def restore_primary_runtime(agent) -> bool:
pool_matches_primary = False
if pool is not None and pool_provider and not pool_matches_primary:
agent._credential_pool = None
agent._credential_pool_entry_id = None
try:
from agent.credential_pool import load_pool
@ -1460,6 +1517,7 @@ def restore_primary_runtime(agent) -> bool:
# the pool for its current best entry and swap the live credential in.
# When the pool is absent, empty, or the entry has no usable key, we
# keep the snapshot key (the existing behavior). Fixes #25205.
agent._credential_pool_entry_id = None
pool = getattr(agent, "_credential_pool", None)
if pool is not None and pool.has_available():
entry = pool.select()
@ -2056,6 +2114,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
# restore the original pool (issue #52727: pool reload is part of this
# switch and must be reversible on rollback).
_snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING)
_snapshot["_credential_pool_entry_id"] = getattr(
agent, "_credential_pool_entry_id", _MISSING
)
try:
# Clear the per-config context_length override so the new model's
@ -2112,6 +2173,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
# A pool bound to the old provider is worse than no pool: the
# recovery guard rejects it and every later 401/429 skips rotation.
agent._credential_pool = None
agent._credential_pool_entry_id = None
try:
from agent.credential_pool import load_pool
agent._credential_pool = load_pool(new_provider)
@ -2121,10 +2183,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
"continuing without pool rotation this turn",
new_provider, _pool_exc,
)
# ── Build new client ──
if (new_provider or "").strip().lower() == "moa":
from agent.moa_loop import MoAClient
from agent.moa_loop import build_moa_facade
# The MoA virtual provider speaks only chat.completions via the
# MoAClient facade — the aggregator's real transport
@ -2141,7 +2202,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
agent.api_key = api_key or "moa-virtual-provider"
agent.base_url = "moa://local"
agent._client_kwargs = {}
agent.client = MoAClient(agent.model or "default")
agent.client = build_moa_facade(agent, agent.model)
elif api_mode == "anthropic_messages":
from agent.anthropic_adapter import (
build_anthropic_client,
@ -2217,6 +2278,8 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
reason="switch_model",
shared=True,
)
sync_credential_pool_entry_id(agent)
except Exception:
# Rollback every mutated field to the pre-swap snapshot so the agent
# is left consistent (old model + old provider + old client) and the

View file

@ -368,7 +368,7 @@ def _detect_claude_code_version() -> str:
try:
result = _sp.run(
[cmd, "--version"],
capture_output=True, text=True, timeout=5,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5,
)
if result.returncode == 0 and result.stdout.strip():
# Output is like "2.1.74 (Claude Code)" or just "2.1.74"
@ -914,7 +914,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]:
"-s", "Claude Code-credentials",
"-w"],
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=5,
stdin=subprocess.DEVNULL,
)
@ -1920,10 +1920,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
return None
btype = b.get("type")
if btype == "text":
# Coerce empty/whitespace-only text to a non-whitespace placeholder;
# the Messages input schema rejects blank text blocks (#69512), and a
# blank block stored in history replays on every turn → permanent 400.
out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))}
text_val = b.get("text", "")
# Bedrock and strict Anthropic-compatible endpoints reject text
# blocks where "text" is empty or whitespace-only (#69512). Drop the
# blank block (the caller relocates any cache_control it carried and
# falls back to a non-whitespace placeholder when nothing survives)
# rather than coercing in place — a coerced "(empty)" block would be
# model-visible noise next to surviving thinking/tool_use blocks.
# Type-safe: captured blocks can carry text=None from an invalid
# upstream payload, which a bare .strip() would crash on.
if not isinstance(text_val, str) or not text_val.strip():
return None
out: Dict[str, Any] = {"type": "text", "text": text_val}
# citations is input-valid ONLY when it's a non-empty list; the SDK
# emits citations=None on responses, which the input schema rejects.
cits = b.get("citations")
@ -2011,9 +2019,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
parsed_args = {}
redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args
replayed: List[Dict[str, Any]] = []
_relocated_replay_cache_control = None
_dropped_blank_text = False
for b in ordered_blocks:
clean = _sanitize_replay_block(b)
if clean is None:
if isinstance(b, dict) and b.get("type") == "text":
_dropped_blank_text = True
if isinstance(b, dict) and isinstance(b.get("cache_control"), dict):
# A dropped blank text block can still carry the cache
# breakpoint marker -- relocate it rather than losing it.
_relocated_replay_cache_control = b["cache_control"]
continue
if clean.get("type") == "tool_use":
# Override raw (un-redacted) input with the redacted copy when
@ -2023,20 +2039,68 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
if redacted is not None:
clean["input"] = redacted
replayed.append(clean)
# When every text block was blank and nothing cacheable survived
# (e.g. signed thinking + a blank text block, or a SOLE blank
# cache-marked block), emit the non-whitespace placeholder so the
# replayed message stays schema-valid (#69512) and a relocated cache
# marker still has a carrier instead of being silently lost.
_has_cacheable_replay = any(
isinstance(b, dict) and b.get("type") in {"text", "tool_use"}
for b in replayed
)
if not _has_cacheable_replay and (
_dropped_blank_text or _relocated_replay_cache_control is not None
):
replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER})
if replayed:
if _relocated_replay_cache_control is not None:
_apply_assistant_cache_control_to_last_cacheable_block(
replayed, _relocated_replay_cache_control
)
_apply_assistant_cache_control_to_last_cacheable_block(
replayed, m.get("cache_control")
)
return {"role": "assistant", "content": replayed}
blocks = _extract_preserved_thinking_blocks(m)
# Cache markers dropped along with a blank block are relocated onto the
# last surviving cacheable block below (via
# _apply_assistant_cache_control_to_last_cacheable_block), rather than
# lost -- prompt_caching.py's _apply_cache_marker() sets cache_control
# directly on content[-1] for list content, so if that last part happens
# to be blank text, dropping it silently would lose the breakpoint.
_relocated_cache_control = None
if content:
if isinstance(content, list):
converted_content = _convert_content_to_anthropic(content)
if isinstance(converted_content, list):
blocks.extend(converted_content)
# Bedrock and strict Anthropic-compatible endpoints reject
# text blocks where "text" is empty or whitespace-only. The
# ordered-replay path enforces the same invariant via
# _sanitize_replay_block(). Type-safe against ANY invalid
# "text" value from an upstream payload -- None, or a
# truthy non-string like an int -- not just None: checking
# isinstance() first (rather than `blk.get("text") or ""`)
# means a non-string value is treated as blank/invalid
# instead of reaching .strip() and raising AttributeError.
for blk in converted_content:
_blk_text = blk.get("text") if isinstance(blk, dict) else None
if (
isinstance(blk, dict)
and blk.get("type") == "text"
and (not isinstance(_blk_text, str) or not _blk_text.strip())
):
if isinstance(blk.get("cache_control"), dict):
_relocated_cache_control = blk["cache_control"]
continue
blocks.append(blk)
else:
blocks.append({"type": "text", "text": str(content)})
# Scalar (non-list) content: a whitespace-only string is the
# same invalid-payload case as an empty list block -- drop it
# rather than emitting a blank text block.
text_str = str(content)
if text_str.strip():
blocks.append({"type": "text", "text": text_str})
for tc in m.get("tool_calls", []):
if not tc or not isinstance(tc, dict):
continue
@ -2052,9 +2116,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
"name": fn.get("name", ""),
"input": parsed_args,
})
_apply_assistant_cache_control_to_last_cacheable_block(
blocks, m.get("cache_control")
)
# Kimi's /coding endpoint (Anthropic protocol) requires assistant
# tool-call messages to carry reasoning_content when thinking is
# enabled server-side. Preserve it as a thinking block so Kimi
@ -2080,19 +2141,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
)
if isinstance(reasoning_content, str) and not _already_has_thinking:
blocks.insert(0, {"type": "thinking", "thinking": reasoning_content})
# Anthropic rejects empty assistant content
effective = blocks or content
if not effective or effective == "":
effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
elif isinstance(effective, list):
# The all-empty guard above misses a list that still contains a
# whitespace-only text block (e.g. from a content array of blank parts,
# or compression). Those also trip "text content blocks must contain
# non-whitespace text" (#69512). Coerce text blocks in place; other
# block types (thinking/tool_use/image) are left untouched.
for blk in effective:
if isinstance(blk, dict) and blk.get("type") == "text":
blk["text"] = _safe_text(blk.get("text", ""))
# Anthropic rejects empty assistant content. IMPORTANT: fall back only
# to the placeholder, never to the raw `content` variable -- `content`
# is the UNFILTERED original message content, and can itself be exactly
# the blank/whitespace-only payload the filtering above just removed
# (a sole blank text block, or scalar whitespace with no tool_calls).
# `blocks or content` there would silently restore the invalid provider
# payload this function exists to prevent (#69512).
effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
# Applied here (after the empty-fallback resolution) rather than
# earlier against `blocks` directly, so a cache_control relocated from
# a dropped blank block that was the ONLY block still lands on the
# (empty) placeholder instead of being silently lost when blocks was
# empty at the point the marker would otherwise have been applied.
if _relocated_cache_control is not None:
_apply_assistant_cache_control_to_last_cacheable_block(
effective, _relocated_cache_control
)
_apply_assistant_cache_control_to_last_cacheable_block(
effective, m.get("cache_control")
)
return {"role": "assistant", "content": effective}
@ -2825,6 +2893,7 @@ def create_anthropic_message(
*,
log_prefix: str = "",
prefer_stream: bool = True,
on_stream_event=None,
) -> Any:
"""Create an Anthropic message, aggregating via stream when available.
@ -2834,6 +2903,13 @@ def create_anthropic_message(
crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to
match the main turn path, falling back to ``create()`` only for providers
that explicitly do not support streaming, such as restricted Bedrock roles.
``on_stream_event``: optional callable invoked once per streamed event
(best-effort, exceptions swallowed). Lets callers report forward progress
to liveness watchdogs e.g. the auxiliary compression path ticking its
progress hook so a slow-but-generating summary model isn't treated as
hung. Only fires on the streaming path; the ``create()`` fallback has no
events to report.
"""
sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix)
@ -2844,6 +2920,18 @@ def create_anthropic_message(
stream_kwargs.pop("stream", None)
try:
with stream_fn(**stream_kwargs) as stream:
if callable(on_stream_event):
# Consume the event stream manually so each event can
# tick the caller's progress callback; get_final_message
# then returns the accumulated snapshot.
for _event in stream:
try:
on_stream_event(_event)
except Exception:
logger.debug(
"%son_stream_event callback failed",
log_prefix, exc_info=True,
)
return stream.get_final_message()
except Exception as exc:
if not _is_stream_unavailable_error(exc):

View file

@ -54,7 +54,7 @@ import time
import uuid
from pathlib import Path # noqa: F401 — used by test mocks
from types import SimpleNamespace
from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING
from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
from urllib.parse import urlparse, parse_qs, urlunparse
# NOTE: `from openai import OpenAI` is deliberately NOT at module top — the
@ -249,6 +249,50 @@ def aux_interrupt_protection(active: bool = True):
_aux_interrupt_protection.active = prev
# ── Forward-progress hook for streamed auxiliary calls ───────────────────
# Long auxiliary calls (context compression is the prime case) are watched by
# wall-clock deadlines in their hosts (gateway session hygiene). A fixed
# deadline punishes SLOW summary models exactly as hard as HUNG ones: a
# reasoning model happily streaming a large summary is killed mid-generation.
# This thread-local hook lets the host observe liveness instead: the wire
# consumers below tick it on every streamed token/SSE event, and the host
# extends its deadline while tokens are moving (see gateway/run.py session
# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the
# call topology — the aux call and its stream consumption run synchronously
# on the thread that installed the hook.
_aux_progress = threading.local()
def _notify_aux_progress() -> None:
"""Tick the installed forward-progress hook, if any. Never raises."""
hook = getattr(_aux_progress, "hook", None)
if hook is None:
return
try:
hook()
except Exception:
logger.debug("aux progress hook failed", exc_info=True)
def _aux_progress_active() -> bool:
return getattr(_aux_progress, "hook", None) is not None
@contextlib.contextmanager
def aux_progress_hook(hook):
"""Install *hook* as the current thread's aux forward-progress callback.
``hook=None`` is a no-op passthrough so callers can wire it
unconditionally. Re-entrant-safe: restores the previous hook on exit.
"""
prev = getattr(_aux_progress, "hook", None)
_aux_progress.hook = hook if callable(hook) else prev
try:
yield
finally:
_aux_progress.hook = prev
def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
"""Return False instead of raising when a patched symbol is not a type."""
try:
@ -1060,16 +1104,29 @@ class _CodexCompletionsAdapter:
# key in extra_body (not top-level) and GitHub/Copilot Responses opts
# out of cache-key routing entirely — for those hosts, skip it here.
try:
from agent.transports.codex import _content_cache_key
from agent.transports.codex import (
_content_cache_key,
_default_prompt_cache_retention_for_request,
)
from utils import base_url_host_matches
_host_src = str(getattr(self._client, "base_url", "") or "")
_is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai")
_is_github = base_url_host_matches(_host_src, "githubcopilot.com")
_is_github = (
base_url_host_matches(_host_src, "githubcopilot.com")
or base_url_host_matches(_host_src, "models.github.ai")
)
if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs:
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"))
if _cache_key:
resp_kwargs["prompt_cache_key"] = _cache_key
if "prompt_cache_retention" not in resp_kwargs:
_cache_retention = _default_prompt_cache_retention_for_request(
model,
_host_src,
)
if _cache_retention:
resp_kwargs["prompt_cache_retention"] = _cache_retention
except Exception:
logger.debug(
"Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True
@ -1151,6 +1208,10 @@ class _CodexCompletionsAdapter:
def _on_each_event(_event: Any) -> None:
# Re-check timeout/cancellation per event, matching the
# cadence the old in-line ``_check_cancelled()`` used.
# Each SSE event is also forward progress for hosts watching
# a progress hook (gateway session hygiene): a reasoning
# model streaming a long summary must not look hung.
_notify_aux_progress()
_check_cancelled()
event_stream = self._client.responses.create(**stream_kwargs)
@ -1392,7 +1453,18 @@ class _AnthropicCompletionsAdapter:
existing = {}
anthropic_kwargs["extra_body"] = {**existing, **passthrough}
response = create_anthropic_message(self._client, anthropic_kwargs)
response = create_anthropic_message(
self._client,
anthropic_kwargs,
# Tick the aux forward-progress hook per streamed event so hosts
# watching liveness (gateway session hygiene) don't kill a
# slow-but-generating summary model. No-op when no hook is
# installed (None keeps the fast get_final_message path).
on_stream_event=(
(lambda _event: _notify_aux_progress())
if _aux_progress_active() else None
),
)
_transport = get_transport("anthropic_messages")
_nr = _transport.normalize_response(
response, strip_tool_prefix=self._is_oauth
@ -1705,7 +1777,7 @@ def _read_nous_auth() -> Optional[dict]:
try:
if not _AUTH_JSON_PATH.is_file():
return None
data = json.loads(_AUTH_JSON_PATH.read_text())
data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8"))
if data.get("active_provider") != "nous":
return None
provider = data.get("providers", {}).get("nous", {})
@ -2489,16 +2561,18 @@ def _relay_sync_completion(
*,
provider: str | None = None,
api_mode: str | None = None,
create: Callable[[dict[str, Any]], Any] | None = None,
) -> Any:
callback = create or (lambda request: client.chat.completions.create(**request))
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
if route is None:
return client.chat.completions.create(**kwargs)
return callback(kwargs)
provider_name, fallback_model, metadata = route
from agent import relay_llm
return relay_llm.execute_current(
kwargs,
lambda request: client.chat.completions.create(**request),
callback,
name=provider_name,
model_name=str(kwargs.get("model") or fallback_model),
metadata=metadata,
@ -2512,16 +2586,18 @@ async def _relay_async_completion(
*,
provider: str | None = None,
api_mode: str | None = None,
create: Callable[[dict[str, Any]], Any] | None = None,
) -> Any:
callback = create or (lambda request: client.chat.completions.create(**request))
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
if route is None:
return await client.chat.completions.create(**kwargs)
return await callback(kwargs)
provider_name, fallback_model, metadata = route
from agent import relay_llm
return await relay_llm.execute_current_async(
kwargs,
lambda request: client.chat.completions.create(**request),
callback,
name=provider_name,
model_name=str(kwargs.get("model") or fallback_model),
metadata=metadata,
@ -3807,6 +3883,7 @@ def _retry_same_provider_sync(
effective_timeout: float,
effective_extra_body: dict,
reasoning_config: Optional[dict],
extra_headers: Optional[Dict[str, str]] = None,
) -> Any:
if task == "vision":
_, retry_client, retry_model = resolve_vision_provider_client(
@ -3842,7 +3919,13 @@ def _retry_same_provider_sync(
extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=retry_base or resolved_base_url,
task=task,
)
# Preserve per-request attribution headers (e.g. Copilot's
# ``x-initiator: user``) across the rebuilt-client retry — dropping them
# here would let a recovery retry silently lose capability gating (#60293).
if extra_headers:
retry_kwargs["extra_headers"] = dict(extra_headers)
if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
return _validate_llm_response(
@ -3872,6 +3955,7 @@ async def _retry_same_provider_async(
effective_timeout: float,
effective_extra_body: dict,
reasoning_config: Optional[dict],
extra_headers: Optional[Dict[str, str]] = None,
) -> Any:
if task == "vision":
_, retry_client, retry_model = resolve_vision_provider_client(
@ -3907,7 +3991,12 @@ async def _retry_same_provider_async(
extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=retry_base or resolved_base_url,
task=task,
)
# Preserve per-request attribution headers across the rebuilt-client
# retry — see the sync variant above (#60293).
if extra_headers:
retry_kwargs["extra_headers"] = dict(extra_headers)
if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
return _validate_llm_response(
@ -3988,6 +4077,24 @@ def _refresh_provider_credentials(provider: str) -> bool:
return False
_evict_cached_clients(normalized)
return True
if normalized == "vertex":
# Mirrors run_agent.py's _try_refresh_vertex_client_credentials
# for the main conversation loop. Without this branch, an
# auxiliary Vertex client (vision, title generation, reflection,
# context compression, ...) that 401s on its ~1h token expiry
# falls through to the final `return False` below: the stale
# client is never evicted from _client_cache (whose cache key
# ignores the rotating bearer token), so every subsequent
# auxiliary Vertex call keeps 401ing until process restart.
from agent.vertex_adapter import get_vertex_config
token, base_url = get_vertex_config()
if not isinstance(token, str) or not token.strip():
return False
if not isinstance(base_url, str) or not base_url.strip():
return False
_evict_cached_clients(normalized)
return True
except Exception as exc:
logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc)
return False
@ -4101,7 +4208,7 @@ def _call_fallback_candidate_sync(
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body, reasoning_config=reasoning_config,
base_url=fb_base)
base_url=fb_base, task=task)
try:
return _validate_llm_response(
_relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task)
@ -4118,7 +4225,7 @@ def _call_fallback_candidate_sync(
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task)
try:
return _validate_llm_response(
_relay_sync_completion(
@ -4173,7 +4280,7 @@ async def _call_fallback_candidate_async(
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body, reasoning_config=reasoning_config,
base_url=fb_base)
base_url=fb_base, task=task)
try:
return _validate_llm_response(
await _relay_async_completion(
@ -4197,7 +4304,7 @@ async def _call_fallback_candidate_async(
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task)
try:
return _validate_llm_response(
await _relay_async_completion(
@ -6965,6 +7072,7 @@ def _build_call_kwargs(
extra_body: Optional[dict] = None,
reasoning_config: Optional[dict] = None,
base_url: Optional[str] = None,
task: Optional[str] = None,
) -> dict:
"""Build kwargs for .chat.completions.create() with model/provider adjustments."""
kwargs: Dict[str, Any] = {
@ -7019,11 +7127,32 @@ def _build_call_kwargs(
_provider_norm in {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"}
or base_url_host_matches(_effective_base, "integrate.api.nvidia.com")
)
_is_moa = bool(task) and str(task) == "moa_reference"
# Gemini's native generateContent maps max_tokens → maxOutputTokens and,
# when it is omitted, applies a fixed 65,535-token ceiling rather than
# "the model's full budget" (see gemini_native_adapter.build_gemini_request).
# So an explicit cap is both safe and the ONLY way to honor it here —
# dropping max_tokens silently makes MoA's reference_max_tokens a no-op
# for gemini advisors (they run effectively uncapped).
_is_gemini_native = _provider_norm in {
"gemini", "google", "google-gemini", "google-ai-studio",
}
if not _is_gemini_native and _effective_base:
try:
from agent.gemini_native_adapter import is_native_gemini_base_url
_is_gemini_native = is_native_gemini_base_url(_effective_base)
except Exception:
pass
if (
_is_anthropic_compat_endpoint(provider, _effective_base)
or _is_nvidia_nim
or _is_moa
or _is_gemini_native
):
kwargs["max_tokens"] = max_tokens
# Use auxiliary_max_tokens_param() so models that require
# max_completion_tokens (GPT-5 family, Copilot) get the right
# parameter name instead of a hardcoded max_tokens that 400s.
kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model))
if tools:
# Defensive dedup: providers like Google Vertex, Azure, and Bedrock
@ -7266,6 +7395,346 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any:
return value
# ── Streamed aggregation for progress-hooked auxiliary calls ─────────────
# When a forward-progress hook is installed (aux_progress_hook — today only
# by context compression), the primary chat.completions attempt is upgraded
# to a streamed request that is aggregated back into a complete response.
# Two effects, both deliberate:
# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead
# of a total budget (httpx applies the read timeout per stream read), so
# a slow-but-generating summary model is never killed mid-generation
# while tokens are moving — only a genuinely silent connection dies.
# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs
# (gateway session hygiene) extend their deadlines on liveness instead
# of guessing with a fixed wall clock.
# A total ceiling still bounds the pathological 1-token-per-idle-window
# stream; see _aux_stream_total_ceiling().
_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
_AUX_STREAM_CEILING_MULTIPLIER = 4.0
def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
"""Absolute wall-clock bound for a progress-hooked streamed aux call.
Generous by design the idle timeout is the real guard; this only stops
a degenerate stream that trickles one token per idle window forever.
"""
try:
timeout = float(effective_timeout) if effective_timeout is not None else 0.0
except (TypeError, ValueError):
timeout = 0.0
return max(_AUX_STREAM_CEILING_FLOOR_SECONDS,
_AUX_STREAM_CEILING_MULTIPLIER * timeout)
def _client_streams_internally(client: Any) -> bool:
"""Wire adapters that consume a stream inside .create() already tick the
progress hook themselves (Codex per SSE event, Anthropic per stream
event); Bedrock's Converse shim cannot stream at all. None of them
accept chat-completions ``stream=True`` semantics from us."""
return isinstance(client, (
CodexAuxiliaryClient,
AnthropicAuxiliaryClient,
BedrockAuxiliaryClient,
))
def _is_streaming_rejected_error(exc: Exception) -> bool:
"""Provider explicitly refused a streamed chat.completions request."""
err = str(exc).lower()
if "stream_options" in err:
return True
return "stream" in err and (
"not supported" in err
or "unsupported" in err
or "not allowed" in err
or "disabled" in err
)
def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
"""Detect providers that only accept streaming (non-stream = HTTP 400).
Some OpenAI-compatible endpoints reject non-streaming chat requests
outright e.g. Tencent Copilot returns
``{"code": 11101, "msg": "Non-stream chat request is currently not
supported"}``. The main conversation loop already streams, so interactive
chat works; auxiliary tasks (title generation, compression, web extract)
used the non-streaming path and failed on every call. When this returns
True the auxiliary client sends ``stream=True`` and aggregates the chunks
itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686).
Beyond the known-host list, users can mark ANY custom endpoint as
stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml
(list of substrings matched against the endpoint URL).
"""
_url = str(base_url or "").lower()
if not _url:
return False
# Tencent Copilot — "Non-stream chat request is currently not supported"
if base_url_host_matches(_url, "copilot.tencent.com"):
return True
try:
from hermes_cli.config import load_config
aux_cfg = (load_config() or {}).get("auxiliary", {})
markers = aux_cfg.get("stream_only_base_urls") or []
if isinstance(markers, (list, tuple)):
for marker in markers:
if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url:
return True
except Exception:
# Config read is best-effort; never break an aux call over it.
pass
return False
def _create_with_progress(
client: Any,
kwargs: Dict[str, Any],
task: Optional[str] = None,
*,
force_stream: bool = False,
) -> Any:
"""chat.completions.create() that streams when a progress hook is active
or the provider only accepts streamed requests.
Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when
neither trigger applies (every existing caller/task) or when the client's
wire adapter streams internally. With a hook + a chunk-capable client,
the request is sent with ``stream=True`` and aggregated, ticking the hook
per chunk so the configured ``timeout`` acts per stream read (idle)
rather than as a total budget, and outer liveness watchdogs see tokens
moving. ``force_stream=True`` (stream-only providers such as Tencent
Copilot credit @kudi88, PR #60686) takes the same streamed path even
without a hook. Providers that reject the streamed request fall back to
the plain non-streaming call except under ``force_stream``, where a
stream-only provider rejects the plain call by definition, so the
original error is surfaced to the normal recovery chains instead.
"""
_notify_aux_progress() # request dispatched counts as progress
if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
return client.chat.completions.create(**kwargs)
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
stream_kwargs = dict(kwargs)
stream_kwargs["stream"] = True
stream_kwargs["stream_options"] = {"include_usage": True}
try:
chunks = client.chat.completions.create(**stream_kwargs)
except Exception as exc:
# Genuine provider failures (auth, credit, rate limit, network) are
# not streaming's fault — surface them unchanged so the existing
# recovery chains (credential refresh, pool rotation, provider
# fallback) see the same error they would on a plain call.
if (
force_stream
or _is_transient_transport_error(exc)
or _is_auth_error(exc)
or _is_payment_error(exc)
or _is_rate_limit_error(exc)
):
raise
# Anything else may be a streaming-specific rejection (explicit
# "stream not supported", stream_options 400, or an idiosyncratic
# 4xx). Retry non-streaming once; if the request itself is bad the
# plain call reproduces the real error for the normal except-chains.
logger.debug(
"Auxiliary %s: streamed request failed (%s); retrying "
"non-streaming", task or "call", exc,
)
return client.chat.completions.create(**kwargs)
# Some shims (MoA virtual provider under quiet mode, defensive adapters)
# return a complete response even when stream=True was requested.
if hasattr(chunks, "choices"):
_notify_aux_progress()
return chunks
return _aggregate_chat_stream(
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
)
def _aggregate_chat_stream(
chunks: Any,
*,
model: str = "",
total_ceiling: Optional[float] = None,
) -> Any:
"""Consume a chat.completions chunk stream into a complete response.
Ticks the thread-local aux progress hook on every chunk. Raises
TimeoutError when *total_ceiling* seconds elapse before the stream
finishes phrased with "timed out" so existing timeout classification
(``_is_timeout_error``) treats it exactly like a request timeout.
Accumulation is shared with the async mirror via
:class:`_ChatStreamAccumulator`.
"""
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
try:
for chunk in chunks:
acc.feed(chunk)
finally:
close_fn = getattr(chunks, "close", None)
if callable(close_fn):
try:
close_fn()
except Exception:
pass
return acc.finish()
class _ChatStreamAccumulator:
"""Shared per-chunk accumulation for sync and async stream aggregation.
Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async
consumer below cannot drift from the sync one (same content/reasoning/
tool-call delta reassembly, same "timed out" ceiling phrasing).
"""
def __init__(self, model: str = "", total_ceiling: Optional[float] = None):
self._started = time.monotonic()
self._total_ceiling = total_ceiling
self.content_parts: List[str] = []
self.reasoning_parts: List[str] = []
self.tool_calls_acc: Dict[int, Dict[str, Any]] = {}
self.finish_reason = None
self.usage = None
self.resp_id = ""
self.resp_model = model or ""
def feed(self, chunk: Any) -> None:
_notify_aux_progress()
if (
self._total_ceiling is not None
and (time.monotonic() - self._started) >= self._total_ceiling
):
raise TimeoutError(
f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
"total ceiling (stream still open but over budget)"
)
self.resp_id = getattr(chunk, "id", None) or self.resp_id
self.resp_model = getattr(chunk, "model", None) or self.resp_model
chunk_usage = getattr(chunk, "usage", None)
if chunk_usage:
self.usage = chunk_usage
choices = getattr(chunk, "choices", None) or []
if not choices:
return
choice = choices[0]
self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
delta = getattr(choice, "delta", None)
if delta is None:
return
piece = getattr(delta, "content", None)
if piece:
self.content_parts.append(piece)
reasoning_piece = (
getattr(delta, "reasoning", None)
or getattr(delta, "reasoning_content", None)
)
if reasoning_piece and isinstance(reasoning_piece, str):
self.reasoning_parts.append(reasoning_piece)
for tc in (getattr(delta, "tool_calls", None) or []):
idx = getattr(tc, "index", 0) or 0
acc = self.tool_calls_acc.setdefault(
idx, {"id": "", "name": "", "arguments": []}
)
if getattr(tc, "id", None):
acc["id"] = tc.id
fn = getattr(tc, "function", None)
if fn is not None:
if getattr(fn, "name", None):
acc["name"] = fn.name
if getattr(fn, "arguments", None):
acc["arguments"].append(fn.arguments)
def finish(self) -> Any:
tool_calls = None
if self.tool_calls_acc:
tool_calls = [
SimpleNamespace(
id=acc["id"],
type="function",
function=SimpleNamespace(
name=acc["name"],
arguments="".join(acc["arguments"]),
),
)
for _idx, acc in sorted(self.tool_calls_acc.items())
]
message = SimpleNamespace(
role="assistant",
content="".join(self.content_parts),
tool_calls=tool_calls,
reasoning="".join(self.reasoning_parts) or None,
)
choice = SimpleNamespace(
index=0,
message=message,
finish_reason=self.finish_reason or "stop",
)
return SimpleNamespace(
id=self.resp_id,
model=self.resp_model,
object="chat.completion",
choices=[choice],
usage=self.usage,
)
async def _aggregate_chat_stream_async(
chunks: Any,
*,
model: str = "",
total_ceiling: Optional[float] = None,
) -> Any:
"""Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer).
The AsyncOpenAI stream contract is an async iterator consuming it with
the sync helper raises. Same accumulation and ceiling semantics via
:class:`_ChatStreamAccumulator`.
"""
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
try:
async for chunk in chunks:
acc.feed(chunk)
finally:
close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None)
if callable(close_fn):
try:
result = close_fn()
if inspect.isawaitable(result):
await result
except Exception:
pass
return acc.finish()
async def _acreate_with_stream(
client: Any,
kwargs: Dict[str, Any],
task: Optional[str] = None,
) -> Any:
"""Async chat.completions.create() for stream-only providers.
Sends ``stream=True`` and aggregates the async chunk stream into a
complete response (credit @kudi88, PR #60686 — async contract fixed to
``async for`` and tool-call deltas preserved per sweeper review).
"""
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
stream_kwargs = dict(kwargs)
stream_kwargs["stream"] = True
stream_kwargs["stream_options"] = {"include_usage": True}
chunks = await client.chat.completions.create(**stream_kwargs)
# Defensive: shims may hand back a complete response despite stream=True.
if hasattr(chunks, "choices"):
return chunks
return await _aggregate_chat_stream_async(
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
)
@_relay_auxiliary_call
def call_llm(
task: str = None,
@ -7282,6 +7751,7 @@ def call_llm(
timeout: float = None,
extra_body: dict = None,
reasoning_config: Optional[dict] = None,
extra_headers: Optional[Dict[str, str]] = None,
api_mode: str = None,
stream: bool = False,
stream_options: dict = None,
@ -7307,6 +7777,9 @@ def call_llm(
extra_body: Additional request body fields.
reasoning_config: Optional Hermes reasoning config for direct model calls
such as MoA reference/aggregator slots.
extra_headers: Additional per-request HTTP headers. These override
client-level defaults for providers that gate capabilities on
request attribution (for example Copilot's ``x-initiator``).
stream: When True, return the raw SDK streaming iterator instead of a
validated complete response. The caller is responsible for consuming
chunks (and for any fallback). Used by the MoA aggregator so its
@ -7424,7 +7897,9 @@ def call_llm(
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=_base_info or resolved_base_url)
base_url=_base_info or resolved_base_url, task=task)
if extra_headers:
kwargs["extra_headers"] = dict(extra_headers)
# Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
_client_base = str(getattr(client, "base_url", "") or "")
@ -7476,7 +7951,16 @@ def call_llm(
kwargs,
provider=resolved_provider,
api_mode=resolved_api_mode,
), task,
create=lambda request: _create_with_progress(
client,
request,
task,
force_stream=_provider_requires_stream(
resolved_provider, _base_info or resolved_base_url,
),
),
),
task,
provider=resolved_provider, base_url=_base_info)
except Exception as transient_err:
if not _is_transient_transport_error(transient_err):
@ -7514,7 +7998,17 @@ def call_llm(
kwargs,
provider=resolved_provider,
api_mode=resolved_api_mode,
), task)
create=lambda request: _create_with_progress(
client,
request,
task,
force_stream=_provider_requires_stream(
resolved_provider,
_base_info or resolved_base_url,
),
),
),
task)
except Exception as retry_transient:
if not _is_transient_transport_error(retry_transient):
raise
@ -7718,6 +8212,7 @@ def call_llm(
effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body,
reasoning_config=reasoning_config,
extra_headers=extra_headers,
)
# ── Same-provider credential-pool recovery ─────────────────────
@ -7766,6 +8261,7 @@ def call_llm(
effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body,
reasoning_config=reasoning_config,
extra_headers=extra_headers,
)
except Exception as retry2_err:
# The rotated key also hit a quota/auth wall. Mark it
@ -8091,7 +8587,7 @@ async def async_call_llm(
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
reasoning_config=reasoning_config,
base_url=_client_base or resolved_base_url)
base_url=_client_base or resolved_base_url, task=task)
# Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
if _is_anthropic_compat_endpoint(resolved_provider, _client_base):
@ -8101,6 +8597,22 @@ async def async_call_llm(
# Retry ONCE on the same provider for a transient transport blip
# before the except-chain escalates to fallback — see call_llm()
# for the rationale. (PR #16587)
_force_stream_async = (
_provider_requires_stream(
resolved_provider, _client_base or resolved_base_url,
)
and not isinstance(client, (
AsyncCodexAuxiliaryClient,
AsyncAnthropicAuxiliaryClient,
AsyncBedrockAuxiliaryClient,
))
)
async def _acreate(_kwargs: Dict[str, Any]) -> Any:
if _force_stream_async:
return await _acreate_with_stream(client, _kwargs, task)
return await client.chat.completions.create(**_kwargs)
try:
return _validate_llm_response(
await _relay_async_completion(
@ -8108,7 +8620,9 @@ async def async_call_llm(
kwargs,
provider=resolved_provider,
api_mode=resolved_api_mode,
), task,
create=_acreate,
),
task,
provider=resolved_provider, base_url=_client_base)
except Exception as transient_err:
if not _is_transient_transport_error(transient_err):
@ -8134,7 +8648,9 @@ async def async_call_llm(
kwargs,
provider=resolved_provider,
api_mode=resolved_api_mode,
), task)
create=_acreate,
),
task)
except Exception as first_err:
if "temperature" in kwargs and _is_unsupported_temperature_error(first_err):
retry_kwargs = dict(kwargs)

View file

@ -209,7 +209,10 @@ _SKILL_REVIEW_PROMPT = (
"conversation for skills the user loaded via /skill-name or you "
"read via skill_view. If any of them covers the territory of the "
"new learning, PATCH that one first. It is the skill that was in "
"play, so it's the right one to extend.\n"
"play, so it's the right one to extend — but only if it is "
"curator-managed. Bundled, hub, pinned, and user-owned skills are "
"off-limits to you no matter how relevant (see Protected skills "
"below); for those, fall through to the next option.\n"
" 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). "
"If no loaded skill fits but an existing class-level skill does, "
"patch it. Add a subsection, a pitfall, or broaden a trigger.\n"
@ -251,10 +254,18 @@ _SKILL_REVIEW_PROMPT = (
"Protected skills (DO NOT edit these):\n"
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
" • Hub-installed skills (installed via 'hermes skills install').\n"
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
"pin only blocks deletion/archive/consolidation by the curator, not "
"content updates. Patch them when a pitfall or missing step turns up, "
"same as any other agent-created skill.\n"
" • Skills in skills.external_dirs (externally owned).\n"
" • PINNED skills (marked via 'hermes curator pin'). You are an "
"autonomous no-user-present actor, so pin blocks your writes too — "
"content updates included. Only the user, in a foreground session, "
"can change a pinned skill.\n"
" • USER-OWNED skills — anything not curator-managed. A skill the "
"user hand-wrote, installed by URL, or asked a foreground agent to "
"create is theirs, not yours; your writes to it WILL be refused. "
"This includes skills that were loaded or consulted this session: "
"being in play does not make one yours to edit. If such a skill is "
"wrong or outdated, say so in your reply and recommend "
"'hermes curator adopt <name>' — do not try to patch it.\n"
"If the only skills that need updating are protected, say\n"
"'Nothing to save.' and stop.\n\n"
"Do NOT capture (these become persistent self-imposed constraints "
@ -309,7 +320,9 @@ _COMBINED_REVIEW_PROMPT = (
" 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were "
"loaded via /skill-name or skill_view in the conversation. If one "
"of them covers the learning, PATCH it first. It was in play; "
"it's the right place.\n"
"it's the right place — provided it is curator-managed. Protected "
"and user-owned skills are off-limits however relevant; fall "
"through when one of those is the best fit.\n"
" 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to "
"find the right one). Patch it.\n"
" 3. ADD A SUPPORT FILE under an existing umbrella via "
@ -337,10 +350,15 @@ _COMBINED_REVIEW_PROMPT = (
"Protected skills (DO NOT edit these):\n"
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
" • Hub-installed skills (installed via 'hermes skills install').\n"
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
"pin only blocks deletion/archive/consolidation by the curator, not "
"content updates. Patch them when a pitfall or missing step turns up, "
"same as any other agent-created skill.\n"
" • Skills in skills.external_dirs (externally owned).\n"
" • PINNED skills (marked via 'hermes curator pin'). Pin blocks "
"autonomous writes entirely — content updates included — because no "
"user is present to consent. Only a foreground session can change one.\n"
" • USER-OWNED skills — anything not curator-managed (hand-written, "
"URL-installed, or created by a foreground agent at the user's "
"request). Your writes to these WILL be refused, including to skills "
"loaded or consulted this session. If one is wrong, say so in your "
"reply and recommend 'hermes curator adopt <name>' instead.\n"
"If the only skills that need updating are protected, say\n"
"'Nothing to save.' and stop.\n\n"
"Do NOT capture as skills (these become persistent self-imposed "

View file

@ -1083,6 +1083,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
tools=tools_for_api,
reasoning_config=agent.reasoning_config,
session_id=getattr(agent, "session_id", None),
base_url=agent.base_url,
max_tokens=agent.max_tokens,
timeout=agent._resolved_api_call_timeout(),
request_overrides=agent.request_overrides,
@ -1520,6 +1521,45 @@ def _fallback_entry_key(fb: dict) -> tuple[str, str, str]:
)
def _fallback_entry_is_same_backend_by_base_url(
*,
current_provider: str,
fb_provider: str,
current_base_url: str,
fb_base_url: str,
current_model: str,
fb_model: str,
) -> bool:
"""True when base_url+model identity means the fallback is the same backend.
Issue #22548: two ``custom_providers`` aliases that point at the same shim
URL with the same model must be skipped, or failover loops on the dead
backend. First-class providers that share a host while using different
auth (``xai-oauth`` vs ``xai``, ``openai-codex`` vs ``openai-api``) are
distinct credential surfaces skipping them strands configured failover
when primary and fallback reuse the same model slug on that host.
"""
if not (
fb_base_url
and current_base_url
and fb_base_url == current_base_url
and fb_model == current_model
):
return False
if fb_provider == current_provider:
return True
try:
from hermes_cli.auth import PROVIDER_REGISTRY
# Both sides are registered first-class providers → different auth
# identities even when the inference host matches. Allow failover.
if current_provider in PROVIDER_REGISTRY and fb_provider in PROVIDER_REGISTRY:
return False
except Exception:
pass
return True
def _fallback_entry_unavailable_without_network(agent, fb: dict) -> Optional[str]:
"""Return a skip reason for fallback entries known to be unusable locally."""
fb_provider = (fb.get("provider") or "").strip().lower()
@ -1608,7 +1648,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
# Skip entries that resolve to the current (provider, model) — falling
# back to the same backend that just failed loops the failure. Compare
# base_url too so two distinct custom_providers entries pointing at the
# same shim/proxy URL also dedup. See issue #22548.
# same shim/proxy URL also dedup. See issue #22548. Do NOT treat
# first-class providers that share a host (xai-oauth vs xai) as the same
# backend — they use different credentials.
current_provider = (getattr(agent, "provider", "") or "").strip().lower()
current_model = (getattr(agent, "model", "") or "").strip()
current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower()
@ -1619,11 +1661,13 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
fb_provider, fb_model,
)
return agent._try_activate_fallback(reason)
if (
fb_base_url_for_dedup
and current_base_url
and fb_base_url_for_dedup == current_base_url
and fb_model == current_model
if _fallback_entry_is_same_backend_by_base_url(
current_provider=current_provider,
fb_provider=fb_provider,
current_base_url=current_base_url,
fb_base_url=fb_base_url_for_dedup,
current_model=current_model,
fb_model=fb_model,
):
logger.warning(
"Fallback skip: chain entry base_url %s matches current backend",
@ -1746,6 +1790,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
fb_provider, fb_model, _pool_provider,
)
agent._credential_pool = None
agent._credential_pool_entry_id = None
if getattr(agent, "_credential_pool", None) is None:
try:
from agent.credential_pool import load_pool
@ -1808,6 +1853,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
# not only after a later credential-rotation rebuild.
agent._replace_primary_openai_client(reason="fallback_timeout_apply")
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
sync_credential_pool_entry_id(agent)
# Re-evaluate prompt caching for the new provider/model
agent._use_prompt_caching, agent._use_native_cache_layout = (
agent._anthropic_prompt_cache_policy(
@ -1975,7 +2023,17 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str:
for internal_key in [k for k in api_msg if isinstance(k, str) and k.startswith("_")]:
api_msg.pop(internal_key, None)
if _needs_sanitize:
agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model)
# In MoA mode, agent.model is the virtual preset name,
# not the actual aggregator model. Resolve the real
# aggregator model so Gemini preserves thought_signature.
_sanitize_model = agent.model
if agent.provider == "moa":
_moa_client = getattr(agent, "client", None)
if _moa_client is not None:
_agg_slot = getattr(_moa_client, "last_aggregator_slot", None)
if _agg_slot and _agg_slot.get("model"):
_sanitize_model = _agg_slot["model"]
agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model)
api_messages.append(api_msg)
effective_system = agent._cached_system_prompt or ""
@ -2568,7 +2626,11 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
request_client_holder = {"client": None, "diag": None, "owner_tid": None}
# Transport kind of the registered request client — see the non-streaming
# variant. Routes _close_request_client_once to anthropic vs openai abort/
# close helpers (#67142).
# close helpers (#67142). ``kind="stream"`` registers a per-request
# *stream handle* instead of a client — used under the MoA facade, whose
# singleton client has no per-request sockets to abort
# (_abort_request_openai_client is a no-op on it), so interrupts must
# close the stream object itself (#57354).
request_client_kind = {"value": "openai"}
request_client_lock = threading.Lock()
# Request-local cancellation flag — see interruptible_api_call for the full
@ -2588,6 +2650,44 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
request_client_holder["owner_tid"] = threading.get_ident()
return client
def _stream_close_callable(stream):
close = getattr(stream, "close", None)
if callable(close):
return close
response = getattr(stream, "response", None)
close = getattr(response, "close", None)
if callable(close):
return close
return None
def _set_request_stream_handle(stream):
# Register the per-request *stream* under kind="stream" so an
# interrupt closes the stream handle itself. Under the MoA facade the
# registered "client" is the shared facade singleton whose
# per-request abort helpers are no-ops, leaving the underlying HTTP
# stream open until the provider drained it (#57354).
if _stream_close_callable(stream) is None:
return stream
with request_client_lock:
request_client_holder["client"] = stream
request_client_kind["value"] = "stream"
request_client_holder["owner_tid"] = threading.get_ident()
return stream
def _close_request_stream_handle(stream, reason: str) -> None:
close = _stream_close_callable(stream)
if close is None:
return
try:
close()
logger.info("Streaming response handle closed (%s)", reason)
except Exception as exc:
logger.debug(
"Streaming response handle close failed (%s): %s",
reason,
exc,
)
def _close_request_client_once(reason: str) -> None:
# See #29507 explanation in the non-streaming variant above. A
# stranger thread (the interrupt-check / stale-stream detector loop)
@ -2595,9 +2695,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
# so the worker thread retains ownership of the FD release.
with request_client_lock:
request_client = request_client_holder.get("client")
request_kind = request_client_kind.get("value", "openai")
owner_tid = request_client_holder.get("owner_tid")
# A registered stream handle (kind="stream", MoA facade path) is
# safe to close from any thread — closing IS the abort — so the
# stranger-thread ownership carve-out only applies to real
# per-request clients (#57354).
stranger_thread = (
request_client is not None
request_kind != "stream"
and request_client is not None
and owner_tid is not None
and owner_tid != threading.get_ident()
)
@ -2606,8 +2712,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
request_client_holder["owner_tid"] = None
if request_client is None:
return
kind = request_client_kind.get("value", "openai")
if kind == "anthropic_messages":
if request_kind == "stream":
_close_request_stream_handle(request_client, reason)
elif request_kind == "anthropic_messages":
if stranger_thread:
agent._abort_request_anthropic_client(request_client, reason=reason)
else:
@ -2897,6 +3004,10 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
defer_logical_completion=True,
)
)
if agent.provider == "moa":
# Hermes interrupts the managed stream; Relay retains sole
# ownership of closing the underlying provider stream.
_set_request_stream_handle(stream)
for chunk in stream:
last_chunk_time["t"] = time.time()
agent._touch_activity("receiving stream response")
@ -3567,13 +3678,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
# already worker-owned-closed by _close_request_client_once
# above; the next attempt builds a fresh one. The shared
# _anthropic_client is never closed from inside a request.
if agent.api_mode != "anthropic_messages":
try:
agent._replace_primary_openai_client(
reason="stream_mid_tool_retry_pool_cleanup"
)
except Exception:
pass
# #70773: same FD-recycle corruption vector for OpenAI.
# The shared client will be replaced lazily by
# _ensure_primary_openai_client on the next attempt.
continue
# SSE error events from proxies (e.g. OpenRouter sends
@ -3632,13 +3739,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
# above; next attempt builds fresh), so the shared
# _anthropic_client is never closed from inside a
# request — only the OpenAI-wire primary is refreshed.
if agent.api_mode != "anthropic_messages":
try:
agent._replace_primary_openai_client(
reason="stream_retry_pool_cleanup"
)
except Exception:
pass
# #70773: same FD-recycle corruption vector for OpenAI.
# The shared client will be replaced lazily by
# _ensure_primary_openai_client on the next attempt.
continue
# Retries exhausted. Log the final failure with
# full diagnostic detail (chain, headers,
@ -3882,10 +3985,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
# FD-recycle corruption vector. Nothing further is needed.
pass
else:
try:
agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup")
except Exception:
pass
# #70773: same FD-recycle corruption vector as #67142.
# The shared OpenAI client's connection pool must NOT be
# closed from this watchdog/poll thread — worker threads
# from previous stale-killed attempts may still be
# unwinding their SSL BIOs. The request-local client is
# already closed above via _close_request_client_once.
# The shared client will be replaced lazily by
# _ensure_primary_openai_client on the next request.
pass
# Reset the timer so we don't kill repeatedly while
# the inner thread processes the closure.
last_chunk_time["t"] = time.time()

View file

@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs(
allowed_keys = {
"model", "instructions", "input", "tools", "store",
"reasoning", "include", "max_output_tokens", "temperature",
"tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier",
"tool_choice", "parallel_tool_calls", "prompt_cache_key",
"prompt_cache_retention", "service_tier",
"extra_headers", "extra_body", "timeout",
}
normalized: Dict[str, Any] = {
@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs(
if isinstance(temperature, (int, float)):
normalized["temperature"] = float(temperature)
# Pass through tool_choice, parallel_tool_calls, prompt_cache_key
for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"):
# Pass through cache routing/retention and tool-dispatch hints.
for passthrough_key in (
"tool_choice",
"parallel_tool_calls",
"prompt_cache_key",
"prompt_cache_retention",
):
val = api_kwargs.get(passthrough_key)
if val is not None:
normalized[passthrough_key] = val

View file

@ -520,30 +520,46 @@ class RuntimeMode:
return None
return [self.profile.toolset, *_enabled_mcp_servers(config)]
def system_blocks(self) -> list[str]:
"""Stable system-prompt blocks for this posture (brief + workspace).
def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]:
"""Return prefix, workspace, and trailing posture blocks separately.
The operating brief carries a model-family edit-format nudge appended
to it (one cached string, not a separate block) so the model is steered
toward the `patch` mode it handles best see ``_edit_format_line``.
The three lists preserve the historical flat prompt order: the brief,
the live workspace snapshot, then configured operator instructions.
Prompt assembly can therefore put a cache boundary before the snapshot
without changing the persisted system-prompt bytes.
"""
if not self.is_coding:
return []
blocks: list[str] = []
return [], [], []
prefix: list[str] = []
workspace_parts: list[str] = []
trailing: list[str] = []
if self.profile.guidance:
brief = self.profile.guidance
edit_line = _edit_format_line(self.model)
if edit_line:
brief = f"{brief}\n{edit_line}"
blocks.append(brief)
prefix.append(brief)
workspace = build_coding_workspace_block(self.cwd)
if workspace:
blocks.append(workspace)
workspace_parts.append(workspace)
# Operator instructions ride their own block so the brief (block 0) stays
# byte-stable and cache-keyed independently of user config.
if self.instructions:
blocks.append(f"Operator instructions (from config):\n{self.instructions}")
return blocks
trailing.append(f"Operator instructions (from config):\n{self.instructions}")
return prefix, workspace_parts, trailing
def system_blocks(self) -> list[str]:
"""Return posture blocks in their historical display order.
``system_prompt_parts`` is the cache-aware API. This compatibility
helper retains the public flat list for callers outside prompt assembly.
"""
prefix, workspace, trailing = self.system_prompt_parts()
return [*prefix, *workspace, *trailing]
def compact_skill_categories(self) -> frozenset[str]:
"""Skill categories to demote to names-only in the prompt's skill index.
@ -644,6 +660,19 @@ def coding_system_blocks(
).system_blocks()
def coding_system_prompt_parts(
*,
platform: Optional[str] = None,
cwd: Optional[str | Path] = None,
config: Optional[dict[str, Any]] = None,
model: Optional[str] = None,
) -> tuple[list[str], list[str], list[str]]:
"""Return coding prefix, workspace snapshot, and trailing guidance."""
return resolve_runtime_mode(
platform=platform, cwd=cwd, config=config, model=model
).system_prompt_parts()
def coding_compact_skill_categories(
*,
platform: Optional[str] = None,

View file

@ -90,9 +90,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool:
HISTORICAL_TASK_HEADING = "## Historical Task Snapshot"
HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State"
HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks"
HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work"
SUMMARY_PREFIX = (
@ -107,9 +104,7 @@ SUMMARY_PREFIX = (
"Topic overlap with the summary does NOT mean you should resume its "
"task: even on similar topics, the latest user message WINS. Treat ONLY "
"the latest message as the active task and discard stale items from "
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or "
"'finish' work described there unless the latest message explicitly "
"asks for it. "
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
@ -219,8 +214,45 @@ _MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW]
# stale directive it carried (e.g. "resume exactly from Active Task") survives
# embedded in the body and keeps hijacking replies. Keep newest-first; entries
# are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes.
# NEVER mutate or reorder an existing entry — each one is the exact wire text a
# shipped build persisted, so editing it silently un-normalizes every summary
# written by that build generation; prepend only. tests/agent/
# test_summary_prefix_semantics.py byte-pins every entry to enforce this.
_HISTORICAL_SUMMARY_PREFIXES = (
# Jul 2026 (#65848 class): identical to the current prefix except it
# Pre-#69619: identical to the current prefix except the stale-item
# discard clause named all four historical headings (the three
# section headers removed by #69619 were still in the template).
# Summaries persisted by builds immediately before #69619 carry this
# exact text and must remain detectable/strippable on resume.
"[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted "
"into the summary below. This is a handoff from a previous context "
"window — treat it as background reference, NOT as active instructions. "
"Do NOT answer questions or fulfill requests mentioned in this summary; "
"they were already addressed. "
"Respond ONLY to the latest user message that appears AFTER this "
"summary — that message is the single source of truth for what to do "
"right now. "
"Topic overlap with the summary does NOT mean you should resume its "
"task: even on similar topics, the latest user message WINS. Treat ONLY "
"the latest message as the active task and discard stale items from "
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
"'## Historical Pending User Asks' / "
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
"'finish' work described there unless the latest message explicitly "
"asks for it. "
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
"back', 'just verify', 'don't do that anymore', 'never mind', a new "
"topic) must immediately end any in-flight work described in the "
"summary; do not re-surface it in later turns. "
"IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system "
"prompt is ALWAYS authoritative and active — never ignore or deprioritize "
"memory content due to this compaction note. "
"None of the above restricts HOW you work: your tools remain fully "
"active — keep calling them normally for the active task (edit files, "
"run commands, search) instead of merely narrating what you would do. "
"The current session state (files, config, etc.) may reflect work "
"described here — avoid repeating it:",
# Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it
# lacked the explicit "tools remain fully active" clause — the strong
# REFERENCE ONLY framing bled into general tool-use suppression
# (observed: 7 consecutive narration-only turns immediately after a
@ -236,9 +268,9 @@ _HISTORICAL_SUMMARY_PREFIXES = (
"Topic overlap with the summary does NOT mean you should resume its "
"task: even on similar topics, the latest user message WINS. Treat ONLY "
"the latest message as the active task and discard stale items from "
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
"'## Historical Pending User Asks' / "
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
"'finish' work described there unless the latest message explicitly "
"asks for it. "
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
@ -303,9 +335,238 @@ _SUMMARY_RATIO = 0.20
# itself a context-pressure source and slows every compaction.
_SUMMARY_TOKENS_CEILING = 10_000
# Aggregate cap on the serialized turn block fed to the summarizer prompt
# (chars). Per-message truncation (_CONTENT_MAX / _TOOL_ARGS_MAX) alone is
# not enough: a compression window with hundreds of already-truncated turns
# can still produce a multi-hundred-KB prompt that blows past slow auxiliary
# backends' context limits or timeouts (Codex Responses fallback paths
# especially). 160K chars ≈ 40K tokens — comfortably inside every supported
# aux model's window while leaving room for the template + previous summary.
# Applied AFTER per-message truncation, with head+tail retention and an
# explicit omitted-middle marker (see _bound_summary_input). This is a
# prompt-side bound only — NEVER add a max_tokens wire cap on the summary
# call (see the no-wire-cap contract test in
# test_compression_small_ctx_threshold_floor.py).
_SUMMARY_INPUT_MAX_CHARS = 160_000
# Placeholder used when pruning old tool results
_PRUNED_TOOL_PLACEHOLDER = "[Old tool output cleared to save context space]"
# Ghost-skill defense (#32106): when compaction reduces an old ``skill_view``
# result to a 1-line metadata summary, the model still believes the skill is
# loaded even though its instructions are gone. The marker below is the ONE
# canonical prune signal — ``_skill_pruned_marker()`` builds it and every
# presence check matches against the same string, so the emit side and the
# check side can never drift apart (the original PR #44166 emitted
# ``[SKILL_PRUNED:`` but presence-checked ``[SKILL_PRUNED]``, making
# re-injection fire even when the marker had survived).
SKILL_PRUNED_MARKER_PREFIX = "[SKILL_PRUNED:"
# skill_view results at or below this size stay verbatim in pruned
# summaries — small skills are cheap to keep and their loss is unlikely to
# ghost the model. Shared by the emit site and the summarizer-input scan.
_SKILL_VIEW_PRUNE_MIN_CHARS = 5000
# Cap for the deterministic marker re-injection list — keeps a very long
# session from growing an unbounded "## Pruned Skills" block in every
# iterative summary update. Newest-referenced skills win.
_MAX_PRUNED_SKILL_MARKERS = 20
def _skill_pruned_marker(skill_name: str) -> str:
"""Return the canonical prune marker for *skill_name*.
Used verbatim by BOTH the emit sites (tool-result summarization,
summary re-injection) and the survival check in
``_reinject_pruned_skill_markers`` one string, no drift.
"""
return (
f"{SKILL_PRUNED_MARKER_PREFIX} content lost in compression; "
f"reload with skill_view(name='{skill_name}')]"
)
# Matches the canonical marker and captures the skill name. Anchored on the
# shared prefix constant so a wording change to the marker body updates the
# emit helper and this extractor together.
_SKILL_PRUNED_MARKER_RE = re.compile(
re.escape(SKILL_PRUNED_MARKER_PREFIX)
+ r"[^\]]*?reload with skill_view\(name='([^']+)'\)"
)
def _extract_pruned_skill_names(text: str) -> list[str]:
"""Return skill names referenced by prune markers in *text*, in order."""
names: list[str] = []
for match in _SKILL_PRUNED_MARKER_RE.finditer(text or ""):
name = match.group(1)
if name not in names:
names.append(name)
return names
def _collect_ghosted_skill_names(turns: List[Dict[str, Any]]) -> list[str]:
"""Skill names whose instructions are about to be lost in compaction.
Covers BOTH shapes a compacted middle window can carry:
- a ``skill_view`` result already demoted by Phase-1 pruning the
canonical ``[SKILL_PRUNED: ...]`` marker is in the row content;
- a RAW ``skill_view`` body that was never demoted (it sat inside the
protected tail of an earlier prune, then aged into the compression
window). The summarizer will paraphrase the instructions away, which
is exactly the ghost-skill failure so it needs a marker too.
"""
names: list[str] = []
def _add(name: str) -> None:
if name and name not in names:
names.append(name)
call_id_to_skill: dict[str, str] = {}
for idx, skill in _skill_view_call_sites(turns):
msg = turns[idx]
for tc in msg.get("tool_calls") or []:
tc_fn = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", None)
tc_name = tc_fn.get("name", "") if isinstance(tc_fn, dict) else getattr(tc_fn, "name", "")
if tc_name != "skill_view":
continue
cid = tc.get("id", "") if isinstance(tc, dict) else (getattr(tc, "id", "") or "")
if cid:
call_id_to_skill[cid] = skill
for msg in turns:
content = msg.get("content")
text = content if isinstance(content, str) else _content_text_for_contains(content)
for name in _extract_pruned_skill_names(text):
_add(name)
if (
msg.get("role") == "tool"
and isinstance(content, str)
and len(content) > _SKILL_VIEW_PRUNE_MIN_CHARS
):
skill = call_id_to_skill.get(str(msg.get("tool_call_id") or ""))
if skill:
_add(skill)
return names
_PRUNED_SKILLS_SECTION_HEADING = "## Pruned Skills"
def _reinject_pruned_skill_markers(summary: str, skill_names: list[str]) -> str:
"""Deterministically restore prune markers the summarizer dropped.
``skill_names`` was extracted from the summarizer INPUT before the LLM
call. For every skill whose canonical marker (``_skill_pruned_marker``)
is absent from the model's output, append it under a ``## Pruned
Skills`` section. Presence is checked against the SAME canonical string
the emit sites produce a paraphrased or renamed marker counts as
dropped and is restored (the original PR checked the literal
``[SKILL_PRUNED]``, which never matches the emitted ``[SKILL_PRUNED:``
form, so it duplicated markers that HAD survived).
The appended block is plain body text: it never carries a handoff
prefix, the merged-summary delimiter, or a start-of-content scaffolding
marker, so ``classify_summary_content`` / todo-snapshot flag handling
are unaffected. The block is routed through ``_redact_compaction_text``
like every other compaction-boundary text.
"""
if not skill_names:
return summary
missing = [
name for name in skill_names
if _skill_pruned_marker(name) not in summary
]
if not missing:
return summary
lines = [_skill_pruned_marker(name) for name in missing]
block = (
"\n\n" + _PRUNED_SKILLS_SECTION_HEADING + "\n"
+ "\n".join(lines)
+ "\n(The listed skills' instructions were pruned during context "
"compression. Reload with the skill_view call in each marker before "
"relying on that skill; one reload per skill is enough — ignore any "
"older markers for the same skill.)"
)
return summary + _redact_compaction_text(block)
# A skill_view call within this many trailing messages counts as "just
# loaded": its full instruction body must survive the Phase-1 prune even when
# the token-budget boundary would otherwise demote it (#32106). Distinct from
# the protected-tail boundary, which is token-based and can land immediately
# after a bulky just-loaded skill body.
_SKILL_PRUNE_RECENT_WINDOW = 10
def _skill_view_call_sites(
messages: List[Dict[str, Any]],
) -> list[tuple[int, str]]:
"""Yield ``(message_index, skill_name)`` for every skill_view tool call."""
sites: list[tuple[int, str]] = []
for i, msg in enumerate(messages):
if msg.get("role") != "assistant":
continue
for tc in msg.get("tool_calls") or []:
if isinstance(tc, dict):
fn = tc.get("function", {})
name = fn.get("name", "") if isinstance(fn, dict) else ""
args_str = fn.get("arguments", "") if isinstance(fn, dict) else ""
else:
fn = getattr(tc, "function", None)
name = getattr(fn, "name", "") if fn else ""
args_str = getattr(fn, "arguments", "") if fn else ""
if name != "skill_view" or not isinstance(args_str, str) or not args_str:
continue
try:
args = json.loads(args_str)
except (json.JSONDecodeError, TypeError):
continue
if isinstance(args, dict):
skill = args.get("name", "")
if isinstance(skill, str) and skill:
sites.append((i, skill))
return sites
def _collect_protected_skill_names(
messages: List[Dict[str, Any]], prune_boundary: int,
) -> set[str]:
"""Skill names whose skill_view bodies must survive Phase-1 demotion.
A skill is protected (lower-cased set) when any of these hold:
- its most recent ``skill_view`` call sits within the last
``_SKILL_PRUNE_RECENT_WINDOW`` messages (just loaded / just reloaded);
- its most recent ``skill_view`` call sits inside the protected tail
(at or after *prune_boundary*);
- its name is mentioned in a user message inside the protected tail
(the user is actively steering work that depends on it).
Protection applies to the ordinary Phase-1/2 prune only. The Pass-4
pressure demotion deliberately ignores it: when the protected region
itself exceeds the soft budget, exempting skill bodies would recreate
the #61932 dead-end shape.
"""
total = len(messages)
if not total:
return set()
recent_start = max(0, total - _SKILL_PRUNE_RECENT_WINDOW)
tail_start = max(0, prune_boundary)
tail_user_texts: list[str] = []
for msg in messages[tail_start:]:
if msg.get("role") != "user":
continue
content = msg.get("content")
if isinstance(content, str) and content:
tail_user_texts.append(content.lower())
protected: set[str] = set()
for idx, skill in _skill_view_call_sites(messages):
key = skill.lower()
if idx >= recent_start or idx >= tail_start:
protected.add(key)
elif any(key in text for text in tail_user_texts):
protected.add(key)
return protected
# Chars per token rough estimate
_CHARS_PER_TOKEN = 4
# Flat token cost per attached image part. Real cost varies by provider and
@ -877,7 +1138,19 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten
code_preview += "..."
return f"[execute_code] `{code_preview}` ({line_count} lines output)"
if tool_name in {"skill_view", "skills_list", "skill_manage"}:
if tool_name == "skill_view":
name = args.get("name", "?")
if content_len > _SKILL_VIEW_PRUNE_MIN_CHARS:
# Ghost-skill defense (#32106): a metadata-only summary makes the
# model believe the skill is still loaded. The canonical marker
# tells it the instructions are gone AND how to get them back.
return (
f"[skill_view] name={name} ({content_len:,} chars) "
+ _skill_pruned_marker(str(name))
)
return f"[skill_view] name={name} ({content_len:,} chars)"
if tool_name in {"skills_list", "skill_manage"}:
name = args.get("name", "?")
return f"[{tool_name}] name={name} ({content_len:,} chars)"
@ -972,6 +1245,7 @@ class ContextCompressor(ContextEngine):
self._last_aux_model_failure_model = None
self._last_compression_savings_pct = 100.0
self._ineffective_compression_count = 0
self._anti_thrash_recovery_deadline = 0.0
self._fallback_compression_streak = 0
self._verify_compaction_cleared_threshold = False
self._last_compression_made_progress = False
@ -1109,6 +1383,7 @@ class ContextCompressor(ContextEngine):
self._last_aux_model_failure_model = None
self._last_compression_savings_pct = 100.0
self._ineffective_compression_count = 0
self._anti_thrash_recovery_deadline = 0.0
self._fallback_compression_streak = 0
self._verify_compaction_cleared_threshold = False
self._last_compression_made_progress = False
@ -1135,6 +1410,7 @@ class ContextCompressor(ContextEngine):
self._consecutive_timeout_failures = 0
self._fallback_compression_streak = 0
self._ineffective_compression_count = 0
self._anti_thrash_recovery_deadline = 0.0
self.get_active_compression_failure_cooldown()
self._load_fallback_compression_streak()
self._load_ineffective_compression_count()
@ -1506,6 +1782,15 @@ class ContextCompressor(ContextEngine):
# rationale as the gpt-5.5/Codex 85% autoraise.
_MIN_CTX_TRIGGER_RATIO = 0.85
# Anti-thrash recovery window (#14694): once the ineffective/fallback
# breaker trips, automatic compaction stays blocked for this long, then
# ONE probe attempt is allowed (counters drop to 1 strike, so another
# ineffective pass re-trips immediately). Long enough that a genuinely
# incompressible session isn't compacting in a loop; short enough that a
# session which has since grown real compressible material recovers well
# before it rides into the provider's hard context limit.
_ANTI_THRASH_RECOVERY_SECONDS = 300.0
@staticmethod
def _coerce_max_tokens(value: Any) -> int | None:
"""Normalize a max_tokens value to a positive int or None.
@ -1629,6 +1914,10 @@ class ContextCompressor(ContextEngine):
max_tokens: int | None = None,
model_thresholds: dict[str, float] | None = None,
threshold_tokens_cap: Any = None,
proactive_prune_tokens: int = 0,
proactive_prune_min_result_chars: int = 8000,
proactive_prune_min_reclaim_tokens: int = 4096,
min_tail_user_messages: int = 1,
):
self.model = model
self.base_url = base_url
@ -1658,6 +1947,32 @@ class ContextCompressor(ContextEngine):
)
self.protect_first_n = protect_first_n
self.protect_last_n = protect_last_n
# Proactive tool-result pruning (cost-oriented; runs INDEPENDENTLY of the
# full-compression trigger, via prune_tool_results_only()). 0 = disabled.
self.proactive_prune_tokens = int(proactive_prune_tokens or 0)
# Floor the summarize threshold at 200 chars (matching
# _prune_old_tool_results' dedup floor). Below ~200 a generated summary
# can be longer than the floor it replaces, so Pass 2 would re-summarize
# its own output every turn (corrupting it and never converging); a
# negative value would strip every non-tail tool result outright. A
# configured 0 keeps the 8000 default via `or`. Keep the floor well above
# typical summary length (default 8000) to stay idempotent.
self.proactive_prune_min_result_chars = max(
200, int(proactive_prune_min_result_chars or 8000)
)
# Minimum estimated token reclaim before a proactive prune COMMITS.
# Every commit rewrites messages the provider has already seen, which
# invalidates the prompt-cache prefix from the earliest rewritten
# message forward. Without this gate a busy tool loop would re-fire
# the prune nearly every iteration (each new tool pair ages an old one
# out of the protected tail), breaking the cache per turn. Requiring a
# meaningful batch of reclaimable tokens makes fires episodic and
# amortized — the same way full compression is the one sanctioned
# cache break. 0 disables the gate (commit any non-zero prune).
self.proactive_prune_min_reclaim_tokens = max(
0, int(proactive_prune_min_reclaim_tokens or 0)
)
self.min_tail_user_messages = min_tail_user_messages
self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80))
self.quiet_mode = quiet_mode
# Output-token reservation: the provider carves max_tokens out of the
@ -1745,6 +2060,12 @@ class ContextCompressor(ContextEngine):
# Anti-thrashing: track whether last compression was effective
self._last_compression_savings_pct: float = 100.0
self._ineffective_compression_count: int = 0
# Monotonic deadline after which a tripped anti-thrash guard grants
# one probation probe (#14694). 0.0 = clock not armed. Armed lazily on
# the first blocked evaluation; deliberately NOT durable, so a process
# restart with a persisted tripped counter (#69872) waits a full fresh
# window before probing (#54923: restart must never disarm a guard).
self._anti_thrash_recovery_deadline: float = 0.0
# Consecutive completed deterministic-fallback boundaries. Unlike the
# real-usage effectiveness counter, ordinary fitting responses must not
# reset this breaker; only a healthy completed summary does.
@ -1856,6 +2177,16 @@ class ContextCompressor(ContextEngine):
self._verify_compaction_cleared_threshold = False
self.awaiting_real_usage_after_compression = False
def snapshot_preflight_display_tokens(self) -> int:
"""Capture the display token count before a speculative preflight seed."""
return self.last_prompt_tokens
def rollback_interrupted_preflight_display_tokens(self, snapshot: int) -> None:
"""Restore a speculative display seed without touching compaction state."""
if self.awaiting_real_usage_after_compression and self.last_prompt_tokens == -1:
return
self.last_prompt_tokens = snapshot
def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool:
"""Return True when a high rough preflight estimate is known-noisy.
@ -2025,21 +2356,66 @@ class ContextCompressor(ContextEngine):
_cooldown_remaining,
)
return True
# Anti-thrashing: back off if recent compressions were ineffective
# Anti-thrashing: back off if recent compressions were ineffective.
# The back-off must not be permanent (#14694): the tripped state was
# judged against the transcript as it existed THEN (e.g. a middle
# region too small to matter), but the conversation keeps growing and
# can accumulate plenty of compressible material later. Without a
# recovery path the session never auto-compacts again and rides into
# the provider's hard context limit. Recovery is a probation probe:
# after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE
# attempt by dropping the tripped counter(s) to 1 strike (persisted,
# so sibling agents on the same session row unblock too). If the probe
# is ineffective again the very next verdict re-trips the guard, so
# the worst case in the truly-incompressible state is one compaction
# attempt per recovery window — bounded, not thrash.
#
# The clock is armed lazily on the first BLOCKED evaluation rather
# than persisted at trip time: a fresh process that loads a durable
# tripped counter (#69872) therefore starts a full window blocked,
# preserving the restart-must-not-disarm contract (#54923).
if (
self._ineffective_compression_count >= 2
or self._fallback_compression_streak >= 2
):
_now = time.monotonic()
if self._anti_thrash_recovery_deadline <= 0.0:
self._anti_thrash_recovery_deadline = (
_now + self._ANTI_THRASH_RECOVERY_SECONDS
)
elif _now >= self._anti_thrash_recovery_deadline:
self._anti_thrash_recovery_deadline = 0.0
if self._ineffective_compression_count >= 2:
self._record_ineffective_compression_verdict(1)
if self._fallback_compression_streak >= 2:
self._fallback_compression_streak = 1
self._persist_fallback_compression_streak()
if not self.quiet_mode:
logger.info(
"Anti-thrashing recovery: %.0fs elapsed since the "
"guard tripped — allowing one compaction probe "
"(ineffective=%d fallback=%d).",
self._ANTI_THRASH_RECOVERY_SECONDS,
self._ineffective_compression_count,
self._fallback_compression_streak,
)
return False
if not self.quiet_mode:
logger.warning(
"Compression skipped — repeated compaction attempts did not "
"restore healthy context. ineffective=%d fallback=%d. "
"Consider /new to start fresh, or /compress <topic> for "
"focused compression.",
"Auto-compaction will retry once in %.0fs. Consider /new "
"to start fresh, or /compress <topic> for focused "
"compression.",
self._ineffective_compression_count,
self._fallback_compression_streak,
max(0.0, self._anti_thrash_recovery_deadline - _now),
)
return True
# Guard not tripped (counters were cleared by an effective compaction
# or a fitting real-usage reading) — disarm any pending recovery clock
# so a LATER trip starts its own full window.
self._anti_thrash_recovery_deadline = 0.0
return False
# ------------------------------------------------------------------
@ -2049,6 +2425,7 @@ class ContextCompressor(ContextEngine):
def _prune_old_tool_results(
self, messages: List[Dict[str, Any]], protect_tail_count: int,
protect_tail_tokens: int | None = None,
min_prune_chars: int = 200,
) -> tuple[List[Dict[str, Any]], int]:
"""Replace old tool result contents with informative 1-line summaries.
@ -2158,7 +2535,14 @@ class ContextCompressor(ContextEngine):
else:
content_hashes[h] = (i, msg.get("tool_call_id", "?"))
def _demote_tool_result_at(idx: int) -> bool:
# Ghost-skill defense (#32106): skills just loaded (or actively
# referenced in the protected tail) keep their full skill_view
# bodies through the ordinary prune passes. Without this, a skill
# loaded moments before a compaction can be demoted to metadata
# while the model still believes its instructions are in context.
protected_skills = _collect_protected_skill_names(result, prune_boundary)
def _demote_tool_result_at(idx: int, *, spare_protected_skills: bool = True) -> bool:
"""Replace a bulky tool result at ``idx`` with a 1-line summary.
Returns True when the message was modified.
@ -2191,10 +2575,22 @@ class ContextCompressor(ContextEngine):
return False
if content.startswith("[screenshot removed"):
return False
if len(content) <= 200:
# Only prune if the content is substantial (default >200 chars; the
# proactive path raises this floor via min_prune_chars).
if len(content) <= min_prune_chars:
return False
call_id = msg.get("tool_call_id", "")
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
if spare_protected_skills and tool_name == "skill_view" and protected_skills:
# Just-loaded / actively-referenced skills survive verbatim
# (#32106). Pass-4 pressure demotion overrides this.
try:
_args = json.loads(tool_args) if tool_args else {}
except (json.JSONDecodeError, TypeError):
_args = {}
_skill = _args.get("name", "") if isinstance(_args, dict) else ""
if isinstance(_skill, str) and _skill.lower() in protected_skills:
return False
summary = _summarize_tool_result(tool_name, tool_args, content)
result[idx] = {**msg, "content": summary}
pruned += 1
@ -2259,7 +2655,10 @@ class ContextCompressor(ContextEngine):
if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling:
pressure_hits = 0
for i in range(max(0, prune_boundary), demote_end):
if _demote_tool_result_at(i):
# Pressure passes override the just-loaded-skill guard:
# when the protected region itself blows the soft budget,
# sparing skill bodies would recreate the #61932 dead-end.
if _demote_tool_result_at(i, spare_protected_skills=False):
pressure_hits += 1
if _truncate_tool_call_args_at(i):
pressure_hits += 1
@ -2279,7 +2678,7 @@ class ContextCompressor(ContextEngine):
if last_tool_idx is not None and i == last_tool_idx:
continue
if result[i].get("role") == "tool":
if _demote_tool_result_at(i):
if _demote_tool_result_at(i, spare_protected_skills=False):
pressure_hits += 1
elif result[i].get("role") == "assistant":
if _truncate_tool_call_args_at(i):
@ -2293,7 +2692,9 @@ class ContextCompressor(ContextEngine):
and last_tool_idx >= prune_boundary
and _protected_region_tokens() > soft_ceiling
):
if _demote_tool_result_at(last_tool_idx):
if _demote_tool_result_at(
last_tool_idx, spare_protected_skills=False
):
pressure_hits += 1
if pressure_hits and not self.quiet_mode:
logger.info(
@ -2307,6 +2708,75 @@ class ContextCompressor(ContextEngine):
return result, pruned
def prune_tool_results_only(
self, messages: List[Dict[str, Any]], current_tokens: int | None = None,
) -> tuple[List[Dict[str, Any]], int]:
"""Deterministic, no-LLM tool-result prune for the cost-oriented path.
Runs the Phase-1 prune (``_prune_old_tool_results``) WITHOUT the
compression summary phase, gated on ``proactive_prune_tokens`` rather
than the (much higher) full-compression threshold. On large-window
models ``should_compress()`` (50% of the window) rarely fires, so old
tool outputs otherwise ride in history and are re-sent verbatim on every
subsequent turn; this reclaims them early with no quality-risky LLM
summarization.
Protects the recent tail by message COUNT (``protect_last_n``), never by
``tail_token_budget`` the latter is derived from the 50% compression
threshold (100K tokens on a 1M window) and would protect the entire
session, pruning nothing.
``_prune_old_tool_results`` runs all three deterministic passes:
(1) dedup byte-identical tool results keeps the newest full copy and
back-references older exact duplicates ANYWHERE in the list (including
the protected tail), so no unique content is ever lost; (2) summarize
non-tail tool results larger than ``min_prune_chars``; (3) truncate
oversized tool_call arguments on non-tail assistant messages. Only
pass (2)'s floor is raised by ``proactive_prune_min_result_chars``;
passes (1) and (3) keep their own fixed floors. The recent-tail
protection applies to passes (2) and (3); pass (1) is tail-agnostic by
design because dedup is lossless.
PROMPT-CACHE CONTRACT: a committed prune rewrites message bodies the
provider has already seen, invalidating the cached prefix from the
earliest rewritten message forward exactly like a compression
boundary. To keep that break episodic rather than per-turn, the prune
only COMMITS when the estimated reclaim meets
``proactive_prune_min_reclaim_tokens`` (measured on the actual pruned
output, not guessed up front). Below the gate the INPUT list object is
returned unchanged the standard no-op caller contract (callers gate
bookkeeping on ``result is not input``).
Returns ``(messages, 0)`` the input object when disabled, below
the trigger, or when the reclaim gate rejects the commit.
"""
if self.proactive_prune_tokens <= 0:
return messages, 0
if current_tokens is not None and current_tokens < self.proactive_prune_tokens:
return messages, 0
# Nothing to reclaim until there are messages outside the protected tail.
if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1:
return messages, 0
pruned_msgs, pruned_count = self._prune_old_tool_results(
messages,
protect_tail_count=self.protect_last_n,
protect_tail_tokens=None,
min_prune_chars=self.proactive_prune_min_result_chars,
)
if not pruned_count:
# Standard no-op contract: hand back the INPUT object so callers
# can gate bookkeeping on `result is not input`.
return messages, 0
# Measured-savings gate (prompt-cache hysteresis): only commit when
# the prune reclaims a meaningful batch of tokens. Estimated on the
# real before/after messages so dedup + arg truncation count too.
if self.proactive_prune_min_reclaim_tokens > 0:
before = sum(_estimate_msg_budget_tokens(m) for m in messages)
after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs)
if (before - after) < self.proactive_prune_min_reclaim_tokens:
return messages, 0
return pruned_msgs, pruned_count
# ------------------------------------------------------------------
# Summarization
# ------------------------------------------------------------------
@ -2330,6 +2800,10 @@ class ContextCompressor(ContextEngine):
_CONTENT_TAIL = 1500 # chars kept from the end
_TOOL_ARGS_MAX = 1500 # tool call argument chars
_TOOL_ARGS_HEAD = 1200 # kept from the start of tool args
# Aggregate cap over the whole serialized block, applied AFTER the
# per-message limits above. Alias of the module-level constant (which
# carries the full rationale) so subclasses/tests can override per-class.
_SUMMARY_INPUT_MAX_CHARS = _SUMMARY_INPUT_MAX_CHARS
def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str:
"""Serialize conversation turns into labeled text for the summarizer.
@ -2589,12 +3063,6 @@ Recovered from a deterministic fallback because the LLM context summarizer was u
## Active State
Unknown from deterministic fallback. Inspect current repository/session state if needed.
{HISTORICAL_IN_PROGRESS_HEADING}
Unknown from deterministic fallback the latest user ask is recorded once under
"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an
unfulfilled instruction to re-answer; verify current state and continue from the
protected recent messages after this summary.
## Blocked
{_bullets(blockers, limit=5)}
@ -2604,27 +3072,62 @@ None recoverable from deterministic fallback.
## Resolved Questions
None recoverable from deterministic fallback.
{HISTORICAL_PENDING_ASKS_HEADING}
None recoverable from deterministic fallback. (The latest user ask is preserved once
under "{HISTORICAL_TASK_HEADING}" as historical context it is NOT necessarily
outstanding.)
## Relevant Files
{_bullets(relevant_files, limit=12)}
{HISTORICAL_REMAINING_WORK_HEADING}
Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims.
## Last Dropped Turns
{_bullets(last_dropped_turns, limit=8)}
## Critical Context
Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}"""
# Ghost-skill defense (#32106): the fallback's per-turn truncation
# (``_FALLBACK_TURN_MAX_CHARS``) routinely cuts [SKILL_PRUNED: ...]
# markers out of the compacted turns. Re-derive the ghosted skills
# from the raw turn contents and re-inject deterministically,
# exactly like the LLM-summary path.
_pruned_names = _collect_ghosted_skill_names(turns_to_summarize)
del _pruned_names[_MAX_PRUNED_SKILL_MARKERS:]
summary = self._with_summary_prefix(_redact_compaction_text(body.strip()))
if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS:
summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]"
# Re-inject AFTER the size cap: the markers live at the end of the
# body, exactly where the truncation above cuts.
summary = _reinject_pruned_skill_markers(summary, _pruned_names)
return summary
@classmethod
def _bound_summary_input(cls, content: str) -> str:
"""Cap total summarizer input while preserving beginning and recent tail.
Per-message truncation alone is not enough for very long sessions: a
compression window with hundreds of messages can still produce a huge
single prompt that slow auxiliary backends time out on. Keep both edges
because the beginning often has task setup and the tail has the most
recent state; explicitly mark the omitted middle so the summarizer knows
context was intentionally compressed before it saw the prompt.
"""
if len(content) <= cls._SUMMARY_INPUT_MAX_CHARS:
return content
marker_template = (
"\n\n...[summary input truncated: omitted "
"{omitted:,} chars from the middle to keep compression prompt bounded]...\n\n"
)
# Estimate once, then rebuild with the exact omitted span after the
# head/tail split is known. The second marker can differ by a few chars
# if the comma-formatted number changes width, so recompute once.
marker = marker_template.format(omitted=len(content))
remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0)
head_chars = int(remaining * 0.45)
tail_chars = remaining - head_chars
omitted = max(len(content) - head_chars - tail_chars, 0)
marker = marker_template.format(omitted=omitted)
remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0)
head_chars = int(remaining * 0.45)
tail_chars = remaining - head_chars
tail = content[-tail_chars:].lstrip() if tail_chars else ""
return content[:head_chars].rstrip() + marker + tail
def _fallback_to_main_for_compression(self, e: Exception, reason: str) -> None:
"""Switch from a separate ``summary_model`` back to the main model.
@ -2698,6 +3201,23 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
summary_budget = self._compute_summary_budget(turns_to_summarize)
content_to_summarize = self._serialize_for_summary(turns_to_summarize)
# P2 ghost-skill defense (#32106): [SKILL_PRUNED: ...] markers entering
# the summarizer are prompt INPUT only — LLMs routinely paraphrase them
# into vague prose ("some skills were loaded"), which erases the reload
# instruction. Collect the ghosted skills deterministically BEFORE the
# call (both already-pruned marker rows AND raw skill_view bodies whose
# instructions are about to be summarized away);
# ``_reinject_pruned_skill_markers`` restores any marker the model
# dropped AFTER the call. Markers already carried by the previous
# summary must survive iterative rewrites the same way. Collection
# walks the turn LIST, so the serialized input bound below cannot
# hide a marker in its omitted middle.
_pruned_skill_names = _collect_ghosted_skill_names(turns_to_summarize)
for _name in _extract_pruned_skill_names(self._previous_summary or ""):
if _name not in _pruned_skill_names:
_pruned_skill_names.append(_name)
del _pruned_skill_names[_MAX_PRUNED_SKILL_MARKERS:]
content_to_summarize = self._bound_summary_input(content_to_summarize)
_sanitized_memory_context = sanitize_memory_context(memory_context)
_serialized_memory_context = json.dumps(
_sanitized_memory_context,
@ -2865,9 +3385,6 @@ Be specific with file paths, commands, line numbers, and results.]
- Any running processes or servers
- Environment details that matter]
{HISTORICAL_IN_PROGRESS_HEADING}
[Work currently underway what was being done when compaction fired]
## Blocked
[Any blockers, errors, or issues not yet resolved. Include exact error messages.]
@ -2877,30 +3394,39 @@ Be specific with file paths, commands, line numbers, and results.]
## Resolved Questions
{_resolved_questions_instructions}
{HISTORICAL_PENDING_ASKS_HEADING}
{_pending_asks_instructions}
## Relevant Files
[Files read, modified, or created with brief note on each]
{HISTORICAL_REMAINING_WORK_HEADING}
[What remains to be done framed as STALE context for reference only. The agent must NOT resume this work unless the latest user message explicitly asks for it.]
## Critical Context
[Any specific values, error messages, configuration details, or data that would be lost without explicit preservation. NEVER include API keys, tokens, passwords, or credentials write [REDACTED] instead.]
{_PRUNED_SKILLS_SECTION_HEADING}
[If any [SKILL_PRUNED: ...reload with skill_view(...)] markers appear in the input,
repeat each one verbatim here copy the exact text, do NOT paraphrase, summarize,
or describe them. These markers tell the agent which skills must be reloaded before
use. If none appear, omit this section entirely.]
Target ~{summary_budget} tokens. Be CONCRETE include file paths, command outputs, error messages, line numbers, and specific values. Avoid vague descriptions like "made some changes" say exactly what changed.
{_temporal_anchoring_rule}
Write only the summary body. Do not include any preamble or prefix."""
if self._previous_summary:
# Iterative update: preserve existing info, add new progress
# Iterative update: preserve existing info, add new progress.
# Bound the previous-summary block with the same aggregate cap as
# the serialized new turns: a normal summary is far below the cap
# (the output side is held to a ~10K-token ceiling), but a
# pathological handoff rehydrated from a persisted session can be
# arbitrarily large — the iterative prompt (previous summary +
# new turns) must stay bounded too.
_bounded_previous_summary = self._bound_summary_input(
self._previous_summary
)
prompt = f"""{_summarizer_preamble}
You are updating a context compaction summary. A previous compaction produced the summary below. New conversation turns have occurred since then and need to be incorporated.
PREVIOUS SUMMARY:
{self._previous_summary}
{_bounded_previous_summary}
NEW TURNS TO INCORPORATE:
{content_to_summarize}{_memory_section}
@ -3031,6 +3557,9 @@ This compaction should PRIORITISE preserving all information related to the focu
# Redact the summary output as well — the summarizer LLM may
# ignore prompt instructions and echo back secrets verbatim.
summary = _redact_compaction_text(content.strip())
# P2 ghost-skill defense (#32106): deterministically restore any
# [SKILL_PRUNED: ...] marker the summarizer paraphrased away.
summary = _reinject_pruned_skill_markers(summary, _pruned_skill_names)
summary = self._ground_historical_task_snapshot(summary, turns_to_summarize)
self._validate_summary_user_provenance(summary, has_user_turn)
# Store for iterative updates on next compaction
@ -4091,6 +4620,69 @@ This compaction should PRIORITISE preserving all information related to the focu
return max(pair_end, head_end + 1)
return adjusted
def _ensure_last_n_user_messages_in_tail(
self,
messages: List[Dict[str, Any]],
cut_idx: int,
head_end: int,
n: int,
) -> int:
"""Guarantee the last N actionable user messages are in the protected tail.
Generalizes ``_ensure_last_user_message_in_tail`` to preserve an
arbitrary number of recent user messages. This prevents the token-
budget-based tail cut from consuming recent conversation turns
when large tool outputs fill the budget.
When *n* <= 1, delegates directly to the existing single-message
method for byte-identical regression safety.
If the conversation has fewer than *n* user messages, the earliest
available user message is used without error.
Only REAL actionable user turns count toward N the collector uses
the same ``_is_actionable_user_turn`` /
``_is_synthetic_compression_user_turn`` pair as
``_find_last_user_message_idx``, so blank platform echoes, compaction
handoffs, continuation markers, and todo-snapshot rows never consume
a slot (#69291 bug class).
A user message is already a clean boundary there is no
tool_call/result group that spans across it, so
``_align_boundary_backward`` is intentionally NOT called.
Calling it can pull the cut past the user message into the
preceding assistant(tool_calls)tool group and split it (#22566).
"""
if n <= 1:
return self._ensure_last_user_message_in_tail(messages, cut_idx, head_end)
# Collect real user message indices walking backward from end.
# Mirror _find_last_user_message_idx's filters: compaction handoffs,
# blank platform echoes, and synthetic continuation/todo rows are
# continuity artifacts, not real user turns.
user_indices = []
for i in range(len(messages) - 1, head_end - 1, -1):
msg = messages[i]
if (
self._is_actionable_user_turn(msg)
and not self._is_synthetic_compression_user_turn(msg)
):
user_indices.append(i)
if len(user_indices) == 0:
return cut_idx
if len(user_indices) < n:
target_idx = user_indices[-1]
else:
target_idx = user_indices[n - 1]
if target_idx >= cut_idx:
return cut_idx
cut_idx = target_idx
return max(cut_idx, head_end + 1)
def _find_turn_pair_end(
self,
messages: List[Dict[str, Any]],
@ -4220,6 +4812,26 @@ This compaction should PRIORITISE preserving all information related to the focu
# monotonic — the tail can only grow, never shrink.
cut_idx = self._ensure_last_assistant_message_in_tail(messages, cut_idx, head_end)
# Extend to the last N actionable user messages when configured
# (compression.min_tail_user_messages > 1). This prevents the
# token-budget tail from consuming recent turns when large tool
# outputs fill the budget. The anchor only walks ``cut_idx``
# backward (monotonic — the tail can only grow, never shrink), and
# a user message is a clean boundary, so the forward re-alignment
# below remains a no-op for the anchored index. Gated at the call
# site so the default (1) path is byte-identical to the historical
# single-anchor pipeline — the single-user anchor already ran above,
# and re-invoking it here could re-trigger the causal-coupling
# forward push (#22523) after the assistant anchor adjusted the cut.
# getattr-guarded: bare ``ContextCompressor.__new__`` test doubles
# (and plugin engines) skip __init__, so the attribute may be absent
# (see the compression-path test-double pitfall).
_min_tail_users = getattr(self, "min_tail_user_messages", 1)
if isinstance(_min_tail_users, int) and not isinstance(_min_tail_users, bool) and _min_tail_users > 1:
cut_idx = self._ensure_last_n_user_messages_in_tail(
messages, cut_idx, head_end, _min_tail_users,
)
# The floor guarantees forward progress — compression must always claim
# at least one message or the caller's compress_start >= compress_end
# guard turns the pass into a no-op that re-runs forever (the same loop
@ -4521,6 +5133,39 @@ This compaction should PRIORITISE preserving all information related to the focu
)
telemetry["chunk_count"] = 1 if turns_to_summarize else 0
if not turns_to_summarize:
# The newest handoff summary consumed the entire compressible
# window (every window row was a standalone handoff that strips
# to None, and nothing follows it before compress_end) — there
# is nothing new to summarize. Skip the summary call entirely:
# without this guard the empty window still reached
# _generate_summary, wasting an aux LLM call that aborts
# noisily on empty input (#59496). Mirrors the sibling
# "no compressable window" guard above (#40803): record an
# ineffective strike through the durable write-through helper
# so the anti-thrash breaker in should_compress() can stop the
# loop — this shape cannot shrink, so every subsequent turn
# would otherwise re-fire the same no-op. The rehydrated
# _previous_summary is deliberately KEPT (not rolled back as
# the summary-abort path does for #57835): it came from a
# handoff genuinely present in this transcript, which is
# returned unchanged.
telemetry["failure_class"] = "empty_post_handoff_window"
self._record_ineffective_compression_verdict(
self._ineffective_compression_count + 1,
)
self._last_compression_savings_pct = 0.0
if not self.quiet_mode:
logger.warning(
"Compression skipped: latest context summary leaves no "
"new turns to summarize in window %d-%d. "
"ineffective_compression_count=%d",
compress_start,
compress_end,
self._ineffective_compression_count,
)
return messages
if not self.quiet_mode:
logger.info(
"Context compression triggered (%d tokens >= %d threshold)",

View file

@ -189,6 +189,144 @@ class ContextEngine(ABC):
host filters unsupported optional arguments by signature.
"""
# -- Optional: proactive tool-result prune -----------------------------
def prune_tool_results_only(
self,
messages: List[Dict[str, Any]],
current_tokens: int | None = None,
) -> tuple[List[Dict[str, Any]], int]:
"""Deterministically trim old tool-result payloads without an LLM call.
Runs on a low, cost-oriented trigger independent of ``should_compress``
so large-window engines can reclaim re-sent tool output long before full
compaction would fire. Returns ``(messages, n_pruned)``.
Default is a safe no-op: the list is returned unchanged with ``0``
pruned. Engines that don't implement a cheap prune — and any engine that
predates this hook inherit this default, so the agent loop's
post-tool-call prune path never raises ``AttributeError`` on them. The
built-in ContextCompressor overrides this with the real implementation.
"""
return messages, 0
# -- Optional: per-turn context selection (distinct from compression) --
def select_context(
self,
request_messages: List[Dict[str, Any]],
*,
conversation_messages: List[Dict[str, Any]] = None,
incoming_message: Dict[str, Any] = None,
budget_tokens: int = 0,
) -> List[Dict[str, Any]]:
"""Optionally choose/replace the context for THIS request, pre-generation.
Called every turn after the request message list is assembled and
before it is dispatched to the provider independent of
``should_compress()``. This lets an engine *select* which context
enters the prompt (retrieval, topic routing, role/branch switching)
rather than *shrink* context that is already there. The two verbs are
orthogonal:
- ``compress()`` : context is too long -> make it shorter.
- ``select_context()``: this turn belongs to a different context
-> use that one instead.
Without this hook, engines that need per-turn access to the message
list have to force ``should_compress()`` to return ``True`` so that
``compress()`` is invoked every turn purely as a callback which
conflates selection with compression and degrades behaviour when the
engine's backend is unavailable. ``select_context()`` removes the need
for that workaround.
The returned list is request-only: it replaces the messages sent to
the provider for this single call and MUST NOT be treated as persisted
transcript state. The conversation history in the session DB is left
untouched, so nothing leaks across turns. Return ``None`` to leave the
request unchanged.
Unlike the ``pre_llm_call`` plugin hook (which appends to the user
message and intentionally never rewrites the list, to preserve the
cache prefix), ``select_context()`` may *replace* the message list.
Ordering / cache contract: the host runs this hook **before** prompt
cache-control and **before** every request sanitizer (orphaned-tool
cleanup, thinking-only/role normalization, whitespace/JSON
normalization). So (a) whatever the hook returns still passes through
the same validation as any request a malformed replacement cannot
reach the provider and (b) prompt-cache stability (an AGENTS.md
invariant) is preserved: the default no-op leaves the request
byte-identical, so cache behaviour is unchanged for the built-in
compressor and any non-implementing engine. An engine that *does*
replace the list changes its own cache prefix by definition; that is
the engine's concern, and cache-control breakpoints are re-derived on
the selected list. The hook is evaluated per provider request (so it
re-runs on retries within a turn), consistent with "select the context
for THIS request".
Args:
request_messages: The assembled request message list (system
prompt + history + any ephemeral prefill), in OpenAI format.
conversation_messages: The unmodified persisted conversation
history, for reference only (do not mutate).
incoming_message: The current turn's user message, if available.
budget_tokens: The active model's context length, or 0 if unknown.
Default returns ``None`` (no-op) zero impact on the built-in
compressor or any existing engine.
"""
return None
def on_turn_complete(
self,
messages: List[Dict[str, Any]],
usage: Dict[str, Any] = None,
**kwargs: Any,
) -> None:
"""Observe a finished user turn (post-turn ingestion / observation).
Called from the standard turn-finalization path once the assistant/tool
loop completes, with the finalized in-memory transcript snapshot. This
is the complement to ``select_context()``: selection happens *before*
the request, while observation happens *after* the turn. It lets an
engine ingest, index, summarize, or update routing / topic / session
state from what actually happened so the next ``select_context()``
can act on it.
Coverage: this fires from the normal finalization seam. Some abnormal
early-return paths in the loop (e.g. a content-policy block or a
provider terminal failure) persist and return without routing through
finalization, and therefore do not currently emit this hook. Treat it
as a best-effort post-turn observation for completed turns, not a
guaranteed callback for every possible early exit; unifying all
terminal paths behind one finalization seam is a separate follow-up.
Together the two hooks remove the need to abuse ``should_compress()`` /
``compress()`` as a generic per-turn callback just to observe history,
and they cover the case where a turn finishes and there may be no next
request from which to infer the previous turn.
``messages`` is a shallow copy and should be treated as read-only:
return values are ignored and this hook must not rely on transcript
mutation for persistence. ``kwargs`` may include ``turn_id``,
``task_id``, ``api_call_count``, ``interrupted``, ``failed``, and
``turn_exit_reason``.
``usage`` carries the completed turn's canonical token usage (the same
dict shape passed to ``update_from_response`` ``prompt_tokens`` /
``completion_tokens`` / ``total_tokens`` plus the canonical
``input_tokens`` / ``output_tokens`` / ``cache_read_tokens`` /
``cache_write_tokens`` / ``reasoning_tokens`` buckets) so an engine can
weigh how large/expensive the selected context actually was when
deciding the next ``select_context()``. It is ``None`` on finalized
turns that never reached a provider response (e.g. interrupt); engines
must treat it as optional.
Default is a no-op.
"""
return None
# -- Optional: pre-flight check ----------------------------------------
def should_compress_preflight(self, messages: List[Dict[str, Any]]) -> bool:

View file

@ -19,6 +19,7 @@ REFERENCE_PATTERN = re.compile(
rf"(?<![\w/])@(?:(?P<simple>diff|staged)\b|(?P<kind>file|folder|git|url):(?P<value>{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))"
)
TRAILING_PUNCTUATION = ",.;!?"
_NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""")
_SENSITIVE_HOME_DIRS = (".ssh", ".aws", ".gnupg", ".kube", ".docker", ".azure", ".config/gh")
_SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",)
_SENSITIVE_HOME_FILES = (
@ -60,6 +61,21 @@ class ContextReferenceResult:
blocked: bool = False
def format_reference_value(value: str) -> str:
"""Quote a reference value so ``REFERENCE_PATTERN`` reads it back whole.
The unquoted alternative in the pattern is ``\\S+``, so a path containing a
space parses as a truncated ref with the tail left behind as loose text.
Mirrors ``formatRefValue`` in the desktop's directive-text.tsx.
"""
if not _NEEDS_QUOTING.search(value):
return value
for quote in ("`", '"', "'"):
if quote not in value:
return f"{quote}{value}{quote}"
return value
def parse_context_references(message: str) -> list[ContextReference]:
refs: list[ContextReference] = []
if not message:
@ -308,7 +324,7 @@ def _expand_git_reference(
["git", *args],
cwd=cwd,
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=30,
stdin=subprocess.DEVNULL,
**_popen_kwargs,
@ -534,7 +550,7 @@ def _rg_files(path: Path, cwd: Path, limit: int) -> list[Path] | None:
["rg", "--files", str(path.relative_to(cwd))],
cwd=cwd,
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=10,
stdin=subprocess.DEVNULL,
**_popen_kwargs,

View file

@ -40,7 +40,7 @@ import uuid
import threading
from datetime import datetime
from pathlib import Path
from typing import Any, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple
from agent.context_engine import (
automatic_compaction_status_message,
@ -221,6 +221,25 @@ class CompressionCommitFence:
self._lock = threading.Lock()
self._cancelled = False
self._commit_started = False
# Forward-progress telemetry: the compression worker touches this
# whenever the streamed summary call produces a token (see
# ContextCompressor._call_summary_llm). Waiters use it to distinguish
# a SLOW-but-alive summary model from a HUNG one, so slow models are
# not killed by a fixed wall-clock deadline while tokens are moving.
self._last_progress = time.monotonic()
def touch_progress(self) -> None:
"""Record forward progress (e.g. a streamed summary token arriving).
Called from the compression worker thread; read by async waiters via
:meth:`seconds_since_progress`. A bare float store is atomic in
CPython, so no lock is needed.
"""
self._last_progress = time.monotonic()
def seconds_since_progress(self) -> float:
"""Seconds since the worker last reported forward progress."""
return max(0.0, time.monotonic() - self._last_progress)
def cancel_before_commit(self) -> bool:
"""Cancel a pending commit, or wait for an active commit to finish.
@ -355,6 +374,143 @@ def _emit_compression_attempt_telemetry(
logger.debug("failed to emit compression attempt telemetry: %s", exc)
def compression_skipped_due_to_lock(agent: Any) -> bool:
"""Type-pinned read of the #69870 lock-skip signal.
``agent._compression_skipped_due_to_lock`` is set by ``compress_context``
when a compression pass no-ops because another path holds the per-session
compression lock (holder string when the holder was confirmed, ``True``
otherwise) and cleared to ``None`` at the entry of every call.
The read MUST be type-pinned (``is True or isinstance(x, str)``), never
bare truthiness: MagicMock test-double agents auto-create truthy
attributes, and a bare ``if getattr(agent, ...)`` would hijack every
mocked agent in sibling suites into the lock-skip branch (the
#69870 × #69840 type-ahead incident).
"""
_sig = getattr(agent, "_compression_skipped_due_to_lock", None)
return _sig is True or isinstance(_sig, str)
def _adopt_live_compression_child(
agent: Any,
session_db: Any,
parent_session_id: str,
) -> Optional[List[Dict[str, Any]]]:
"""Move a stale compression contender onto the unique durable child.
Resolve and load first, then mutate the live agent. This ordering keeps the
stale contender fail-closed when lineage is ambiguous or the compacted
handoff cannot be read.
"""
finder = getattr(type(session_db), "find_live_compression_child", None)
loader = getattr(type(session_db), "get_messages_as_conversation", None)
if not callable(finder) or not callable(loader):
return None
child = finder(session_db, parent_session_id)
if not child or not child.get("id"):
return None
child_session_id = str(child["id"])
recovered = loader(session_db, child_session_id)
if not isinstance(recovered, list) or not recovered:
return None
# Revalidate after loading: the child may have rotated or a competing
# continuation may have appeared between the two DB reads.
confirmed = finder(session_db, parent_session_id)
if not confirmed or str(confirmed.get("id") or "") != child_session_id:
return None
agent.session_id = child_session_id
try:
from gateway.session_context import set_current_session_id
set_current_session_id(child_session_id)
except Exception:
os.environ["HERMES_SESSION_ID"] = child_session_id
try:
from hermes_logging import set_session_context
set_session_context(child_session_id)
except Exception:
pass
agent._session_db_created = True
if child.get("system_prompt"):
agent._cached_system_prompt = child["system_prompt"]
agent._last_flushed_db_idx = len(recovered)
agent._flushed_db_message_session_id = child_session_id
agent._flushed_db_message_ids = {
id(message) for message in recovered if isinstance(message, dict)
}
on_session_start = getattr(agent.context_compressor, "on_session_start", None)
if callable(on_session_start):
try:
on_session_start(
child_session_id,
boundary_reason="compression",
old_session_id=parent_session_id,
session_db=session_db,
platform=getattr(agent, "platform", None) or "cli",
conversation_id=getattr(agent, "_gateway_session_key", None),
)
except Exception as exc:
logger.debug("context engine compression-child adoption failed: %s", exc)
else:
bind_state = getattr(agent.context_compressor, "bind_session_state", None)
if callable(bind_state):
try:
bind_state(session_db=session_db, session_id=child_session_id)
except Exception:
pass
try:
if agent._memory_manager:
agent._memory_manager.on_session_switch(
child_session_id,
parent_session_id=parent_session_id,
reset=False,
reason="compression",
)
except Exception as exc:
logger.debug("memory manager compression-child adoption failed: %s", exc)
return recovered
def recover_rotated_compression_session(
agent: Any,
) -> Optional[List[Dict[str, Any]]]:
"""Recover a stale live agent before a new turn writes to its old parent."""
session_db = getattr(agent, "_session_db", None)
session_id = getattr(agent, "session_id", None) or ""
if session_db is None or not session_id:
return None
try:
if not _session_was_rotated_by_compression(session_db, session_id):
return None
# Rotation publication holds the parent compression lease until the
# child handoff is durable. A concurrent turn waits briefly rather than
# observing the intentional parent-ended/child-empty intermediate state.
holder_getter = getattr(session_db, "get_compression_lock_holder", None)
for attempt in range(21):
recovered = _adopt_live_compression_child(agent, session_db, session_id)
if recovered is not None:
return recovered
holder = holder_getter(session_id) if callable(holder_getter) else None
if not holder or attempt == 20:
return None
time.sleep(0.05)
return None
except Exception as exc:
logger.warning(
"compression session recovery failed for session=%s (%s: %s)",
session_id,
type(exc).__name__,
exc,
)
return None
def _compression_lock_holder(agent: Any) -> str:
"""Build a unique holder id for the lock: pid:tid:agent-instance:uuid.
@ -515,7 +671,17 @@ class _CompressionLockLeaseRefresher:
# by the TTL the acquirer set — the lock can never be held past its TTL
# by a stuck refresher.
consecutive_failures = 0
while not self._stop.wait(self._refresh_interval_seconds):
# First refresh happens immediately, not one interval late. Everything
# between try_acquire() and start() (the rotation-ownership lookup, the
# durable-breaker re-read, thread startup) is charged against the very
# first lease, so on a short TTL under load the lock could already be
# expired — and reclaimable by a competing path — before tick #1.
first = True
while first or not self._stop.wait(self._refresh_interval_seconds):
if first:
first = False
if self._stop.is_set():
break
try:
refreshed = self._db.refresh_compression_lock(
self._session_id,
@ -874,6 +1040,7 @@ _SYNTHETIC_USER_FLAGS = (
"_empty_recovery_synthetic",
"_verification_stop_synthetic",
"_pre_verify_synthetic",
"_dropped_toolcall_nudge",
)
@ -1143,6 +1310,14 @@ def compress_context(
# boundary, so the previous flush baseline remains authoritative.
agent._last_compression_attempt_recorded = True
agent._last_compression_attempt_in_place = None
# Clear the lock-skip signal at the VERY TOP, before the codex route and
# the breaker gates below can early-return (per-attempt state rule,
# #58630/#69853). A stale ``True``/holder value from a prior lock-skip
# must never make a later breaker/codex no-op look like lock contention
# to the automatic-path consumers (compression_deferred, #49874) — the
# second clear before lock acquisition below stays for the same reason
# it was added in #69870 and is simply idempotent now.
agent._compression_skipped_due_to_lock = None
_attempt_started_at = time.monotonic()
_attempt_id = uuid.uuid4().hex
@ -1226,7 +1401,7 @@ def compress_context(
# parent_session_id child, no
# `name #N` renumber, no contextvar/env/logging re-sync, no memory/context-
# engine session-switch. The conversation keeps one durable id for life,
# eliminating the session-rotation bug cluster. Default False during rollout.
# eliminating the session-rotation bug cluster. Default True (2107b86024).
in_place = bool(getattr(agent, "compression_in_place", False))
# Set True once the in-place DB write actually completes (the DB block can
# raise and skip it). Surfaced to the gateway via agent._last_compaction_in_place.
@ -1434,6 +1609,8 @@ def compress_context(
if _lock_released:
return
_lock_released = True
if getattr(agent, "_active_compression_lock_holder", None) == _lock_holder:
agent._active_compression_lock_holder = None
if _lock_refresher is not None:
try:
_lock_refresher.stop()
@ -1445,6 +1622,9 @@ def compress_context(
except Exception as _rel_err:
logger.debug("compression lock release failed: %s", _rel_err)
if _lock_holder is not None:
agent._active_compression_lock_holder = _lock_holder
# A delayed contender can acquire the parent lock after the winning path
# has released it and completed rotation. The lock serializes work but does
# not by itself prove that this stale agent still owns a live parent.
@ -1467,15 +1647,25 @@ def compress_context(
_existing_sp = agent._build_system_prompt(system_message)
return messages, _existing_sp
if _parent_already_rotated:
logger.info(
"compression skipped: session=%s was already rotated by "
"another compression path",
_lock_sid,
recovered_messages = _adopt_live_compression_child(
agent, _lock_db, _lock_sid
)
_release_lock()
_existing_sp = getattr(agent, "_cached_system_prompt", None)
if not _existing_sp:
_existing_sp = agent._build_system_prompt(system_message)
if recovered_messages is not None:
logger.warning(
"compression recovery: stale session=%s adopted live child=%s",
_lock_sid,
agent.session_id,
)
return recovered_messages, _existing_sp
logger.warning(
"compression skipped: session=%s was already rotated by "
"another compression path, but no unique live child could be adopted",
_lock_sid,
)
return messages, _existing_sp
# The agent may have been constructed before another path completed an
@ -1509,6 +1699,37 @@ def compress_context(
)
_lock_refresher.start()
# The caller's history snapshot predates lease acquisition. Reload the
# durable parent after the lease is live; MORE durable rows than the
# snapshot carries means a frontend/background writer committed a turn
# in that window, so publishing from this snapshot would omit it.
# Deliberately a LENGTH check, not content equality: in-memory
# mutation of past turns is legal (multimodal compression, retry
# history replacement, think-tag stripping), and a content-equality
# abort would permanently wedge compression on such sessions — the
# #14694 failure shape.
# Rotation-only: in-place compaction (archive_and_compact) is
# non-destructive — pre-compaction rows are soft-archived (active=0,
# compacted=1), stay searchable and recoverable, so snapshot/durable
# drift cannot lose data there and must not abort compaction.
if not in_place and _lock_db is not None and _lock_sid:
durable_loader = getattr(
type(_lock_db), "get_messages_as_conversation", None
)
if callable(durable_loader):
durable_parent = durable_loader(_lock_db, _lock_sid)
if isinstance(durable_parent, list) and len(durable_parent) > len(messages):
logger.warning(
"compression aborted: session=%s changed before lease "
"acquisition; preserving newer durable messages",
_lock_sid,
)
_release_lock()
existing_prompt = getattr(agent, "_cached_system_prompt", None)
if not existing_prompt:
existing_prompt = agent._build_system_prompt(system_message)
return messages, existing_prompt
# Notify external memory provider before compression discards context.
# The provider's on_pre_compress() may return a string of insights it
# wants surfaced inside the compression summary; capture and forward it
@ -1549,7 +1770,29 @@ def compress_context(
messages_before_compression = copy.deepcopy(messages)
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
compressed = compress_fn(messages, **compress_kwargs)
# Publish forward progress to the commit fence while the summary LLM
# call streams. Async hosts (gateway session hygiene) poll
# ``commit_fence.seconds_since_progress()`` to extend their deadline
# while tokens are moving — so a SLOW summary model is only killed
# when it is actually silent, not merely thorough. The hook is
# thread-local and the compress call is synchronous on this thread,
# so it cannot leak into unrelated auxiliary calls.
#
# Fenceless callers (CLI /compress, in-loop auto-compress) install a
# no-op hook: nobody polls their progress, but an ACTIVE hook is what
# switches the summary call onto the streamed path — giving every
# compression path the same two guarantees: the configured timeout
# acts on inactivity (slow models finish), and a byte-trickling
# provider that keeps the connection alive forever is cut off at the
# streamed total ceiling (see _aux_stream_total_ceiling) instead of
# outliving the SDK's inactivity timeout indefinitely.
from agent.auxiliary_client import aux_progress_hook
_progress_hook = (
commit_fence.touch_progress if commit_fence is not None
else (lambda: None)
)
with aux_progress_hook(_progress_hook):
compressed = compress_fn(messages, **compress_kwargs)
except BaseException as _compress_exc:
# ANY exception after lock acquisition — memory hook, capability
# inspection, engine lookup, or compress() — must release the lock so
@ -1777,6 +2020,21 @@ def compress_context(
):
new_system_prompt = cached_system_prompt
agent._cached_system_prompt = cached_system_prompt
# _invalidate_system_prompt() above also cleared the
# cross-session-stable prefix marker boundary. The kept prompt
# is byte-identical, so reconstruct the stable tier and reuse
# it ONLY when the kept prompt still literally starts with it
# (same startswith gate as the restore path); otherwise the
# request layer falls back to the legacy single-breakpoint
# layout with the prompt bytes untouched.
try:
from agent.system_prompt import build_system_prompt_parts as _build_parts
_static = _build_parts(agent, system_message=system_message)["stable"]
if _static and cached_system_prompt.startswith(_static):
agent._cached_system_prompt_static = _static
except Exception:
pass
else:
new_system_prompt = agent._build_system_prompt(system_message)
agent._cached_system_prompt = new_system_prompt
@ -1853,81 +2111,54 @@ def compress_context(
)
except Exception:
pass # best-effort — don't block compression on a flush error
# Propagate title to the new session with auto-numbering
# Publish parent closure + child row + compacted handoff in
# one transaction. No reader can observe a missing/empty child.
# The rotation child must stay on the parent's profile —
# mirror _ensure_db_session's stamp ("default" persists as
# NULL). publish_compression_child additionally COALESCEs
# from the parent row, covering app-global remote sessions
# whose thread lacks the HERMES_HOME context.
try:
from hermes_cli.profiles import get_active_profile_name
_profile_for_child = get_active_profile_name()
if _profile_for_child == "default":
_profile_for_child = None
except Exception:
_profile_for_child = None
old_title = agent._session_db.get_session_title(agent.session_id)
agent._session_db.end_session(agent.session_id, "compression")
old_session_id = agent.session_id
agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
# Ordering contract: the agent thread updates the contextvar here;
# the gateway propagates to SessionEntry after run_in_executor returns.
new_session_id = (
f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_"
f"{uuid.uuid4().hex[:6]}"
)
agent._session_db.publish_compression_child(
parent_session_id=old_session_id,
child_session_id=new_session_id,
source=agent.platform
or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
model=agent.model,
model_config=agent._session_init_model_config,
system_prompt=new_system_prompt,
messages=compressed,
cwd=getattr(agent, "working_directory", None),
profile_name=_profile_for_child,
compression_lock_holder=_lock_holder,
require_compression_lease=_lock_holder is not None,
)
agent.session_id = new_session_id
try:
from gateway.session_context import set_current_session_id
set_current_session_id(agent.session_id)
except Exception:
os.environ["HERMES_SESSION_ID"] = agent.session_id
# The gateway/tools session context (ContextVar + env) and the
# logging session context are SEPARATE mechanisms. The call above
# moves the former; the ``[session_id]`` tag on log lines comes
# from ``hermes_logging._session_context`` (set once per turn in
# conversation_loop.py). Without this, post-rotation log lines in
# the same turn keep the STALE old id while the message/DB/gateway
# state carry the new one — breaking log correlation exactly at the
# compaction boundary (see #34089). Guarded separately so a logging
# failure can never regress the routing update above.
try:
from hermes_logging import set_session_context
set_session_context(agent.session_id)
except Exception:
pass
agent._session_db_created = False
try:
agent._session_db.create_session(
session_id=agent.session_id,
source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
model=agent.model,
model_config=agent._session_init_model_config,
parent_session_id=old_session_id,
)
except Exception as _cs_err:
# The child row could not be created (e.g. FK constraint,
# contended write). Previously the outer handler simply
# warned and let the agent continue on the NEW id — which
# has no row in state.db, producing an orphan: the parent
# is ended, the child is never indexed, and every
# subsequent message is attributed to a session that
# doesn't exist (#33906/#33907). Roll the live id back to
# the parent so the conversation stays attached to a real,
# indexed session instead of a phantom.
logger.warning(
"Compression child session create failed (%s) — "
"rolling back to parent session %s to avoid an orphan.",
_cs_err, old_session_id,
)
agent.session_id = old_session_id
try:
from gateway.session_context import set_current_session_id
set_current_session_id(agent.session_id)
except Exception:
os.environ["HERMES_SESSION_ID"] = agent.session_id
try:
from hermes_logging import set_session_context
set_session_context(agent.session_id)
except Exception:
pass
# Re-open the parent: it was ended above, but we're
# continuing on it, so it must not stay closed.
try:
agent._session_db.reopen_session(old_session_id)
except Exception:
pass
old_session_id = None # no rotation happened
# The parent row already exists in state.db, so mark the
# session as created — _ensure_db_session would otherwise
# retry a (harmless INSERT OR IGNORE) create next turn.
agent._session_db_created = True
raise
agent._session_db_created = True
split_status = "rotated_committed"
# Carry a persistent /goal onto the continuation session.
@ -1947,18 +2178,14 @@ def compress_context(
except (ValueError, Exception) as e:
logger.debug("Could not propagate title on compression: %s", e)
# Shared post-write steps (both modes target agent.session_id, which
# in-place keeps and rotation has already reassigned to the new id):
# refresh the stored system prompt and reset the flush cursor so the
# next turn re-bases its append diff.
agent._session_db.update_system_prompt(agent.session_id, new_system_prompt)
# In-place mode still updates/replaces the current row here.
# Rotation already published prompt + compacted handoff atomically.
if in_place:
agent._session_db.update_system_prompt(
agent.session_id, new_system_prompt
)
agent._last_flushed_db_idx = 0
else:
# A headless turn can be killed before its finalizer. Persist
# the rotated child's compacted handoff at the boundary so
# the new session is immediately resumable.
agent._session_db.replace_messages(agent.session_id, compressed)
agent._last_flushed_db_idx = len(compressed)
agent._flushed_db_message_session_id = agent.session_id
agent._flushed_db_message_ids = {
@ -1968,7 +2195,22 @@ def compress_context(
}
_session_commit_succeeded = True
except Exception as e:
split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed"
if (
not in_place
and locals().get("old_session_id")
and agent.session_id == old_session_id
):
# Atomic publication failed (including lease loss): keep the
# parent live and discard the stale compacted snapshot.
old_session_id = None
messages[:] = copy.deepcopy(messages_before_compression)
compressed = messages
_compression_made_progress = False
split_status = (
"aborted"
if locals().get("old_session_id") is None and not in_place
else "failed_not_indexed"
)
# If the rotation rolled back to the parent (orphan-avoidance
# above), agent.session_id is the still-indexed parent and
# old_session_id was cleared — so this is recovery, not an

View file

@ -34,6 +34,7 @@ from agent.conversation_compression import (
COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE,
COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE,
PRE_API_COMPRESSION_STATUS_TEMPLATE,
compression_skipped_due_to_lock,
conversation_history_after_compression,
)
from agent.context_engine import automatic_compaction_status_message
@ -47,6 +48,7 @@ from agent.turn_context import (
reanchor_current_turn_user_idx,
)
from agent.turn_retry_state import TurnRetryState
from agent.runtime_cwd import resolve_agent_cwd
from agent.message_sanitization import (
close_interrupted_tool_sequence,
_repair_tool_call_arguments,
@ -435,6 +437,37 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
# Continuing session — reuse the exact system prompt from the
# previous turn so the Anthropic cache prefix matches.
agent._cached_system_prompt = stored_prompt
# Reconstruct the cross-session-stable prefix for the early cache
# breakpoint. The static prefix is not persisted (only the full
# prompt is), so gateway surfaces that build a fresh AIAgent per
# turn would otherwise lose the two-block system layout after the
# first turn — flip-flopping the wire shape mid-conversation and
# silently degrading to the legacy single-breakpoint layout.
#
# Safety: the rebuilt stable tier is used ONLY when the restored
# prompt literally starts with it (checked here AND re-checked by
# ``_apply_system_cache_markers``'s ``startswith`` gate). If any
# stable-tier input changed since the prompt was persisted (skills
# edited, identity changed), the prefix mismatches, ``_static``
# stays None, and the request falls back to the legacy layout with
# the restored prompt bytes untouched — never a rewritten prompt.
#
# Gated on ``_use_prompt_caching`` so non-Anthropic routes skip the
# rebuild entirely (the static prefix is only consumed by
# ``apply_anthropic_cache_control``).
if getattr(agent, "_use_prompt_caching", False):
try:
from agent.system_prompt import build_system_prompt_parts as _build_parts
_static = _build_parts(agent, system_message=system_message)["stable"]
if _static and stored_prompt.startswith(_static):
agent._cached_system_prompt_static = _static
except Exception:
# Fail-open: restore continues with the legacy cache layout.
logger.debug(
"static system-prefix reconstruction failed on restore",
exc_info=True,
)
return
if stored_prompt:
stored_state = "stale_runtime"
@ -507,9 +540,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
"""Return False when the persisted Model/Provider lines are stale."""
"""Return False when the persisted runtime-identity lines are stale."""
def line_value(label: str) -> str:
"""Last matching line wins.
Safe ONLY for fields emitted in the volatile tier at the very END of
the prompt (Model / Provider / Platform). User-supplied project
context (AGENTS.md / CLAUDE.md / .cursorrules) is embedded in the
middle context tier, so a last-match scan lets project prose shadow
any field emitted EARLIER see ``host_info_value``.
"""
prefix = f"{label}:"
value = ""
for line in prompt.splitlines():
@ -517,6 +558,32 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
value = line[len(prefix):].strip()
return value
def host_info_value(label: str) -> str:
"""Read a field from the prompt's own host-info block.
The host-info block (``build_environment_hints``) sits in the STABLE
tier, ahead of the embedded project context files. A bare scan of the
whole prompt would therefore match a user's ``AGENTS.md`` that merely
contains a line starting with the same label, comparing runtime state
against project prose. That mismatch never clears, so the check would
reject the stored prompt on EVERY turn rebuilding the system prompt
each message and destroying the prefix cache for the whole session,
which is far worse than the staleness this function guards against.
Anchor on the ``User home directory:`` line that immediately precedes
the working-directory line in that block, and take the FIRST such
occurrence, so only Hermes' own emitted block can satisfy the read.
"""
prefix = f"{label}:"
lines = prompt.splitlines()
for idx, line in enumerate(lines):
if not line.startswith("User home directory:"):
continue
for candidate in lines[idx + 1: idx + 4]:
if candidate.startswith(prefix):
return candidate[len(prefix):].strip()
return ""
stored_model = line_value("Model")
current_model = str(getattr(agent, "model", "") or "").strip()
if stored_model and current_model and stored_model != current_model:
@ -527,6 +594,24 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
if stored_provider and current_provider and stored_provider != current_provider:
return False
# Detect cwd drift: if the stored prompt was built in a different working
# directory, reuse would silently inject a stale path into the prefix cache.
# Compare against resolve_agent_cwd() — the SAME resolver used to build the
# prompt — so gateway/TUI sessions that set TERMINAL_CWD are not falsely
# rejected (they would always differ from the launch dir's os.getcwd()).
stored_cwd = host_info_value("Current working directory")
if stored_cwd:
if stored_cwd != str(resolve_agent_cwd()):
return False
# Detect runtime-surface drift: the stored prompt records which platform it
# was built for (e.g. "desktop" vs "cli"). Reusing a desktop-built prompt on
# a terminal session (or vice versa) would inject the wrong runtime hints.
stored_platform = line_value("Platform")
current_platform = str(getattr(agent, "platform", "") or "").strip()
if stored_platform and current_platform and stored_platform != current_platform:
return False
return True
@ -640,6 +725,55 @@ def _content_policy_blocked_result(
}
def _compression_deferred_result(
agent,
messages: List[Dict],
api_call_count: int,
) -> Dict[str, Any]:
"""Build the soft turn result for a lock-contended compression defer.
Another path (a sibling turn, a background review fork, a manual
``/compress``) holds this session's compression lock, so every
compression pass this turn no-oped and the request still does not fit.
This is a TEMPORARY condition the lock winner is actively shrinking
the same session so the turn must end as a soft defer
(``compression_deferred``), never as ``compression_exhausted``: the
gateway auto-resets (wipes) the session on exhaustion (#9893/#35809),
which would destroy a session that the concurrent compressor is about
to make healthy again.
``failed`` stays False so the gateway persists the user turn (transient
branch) and retry-next-message semantics apply.
"""
holder = getattr(agent, "_compression_skipped_due_to_lock", None)
logger.info(
"turn deferred: compression lock held by another path "
"(session=%s holder=%s) — not counting as compression exhaustion",
agent.session_id or "none",
holder if isinstance(holder, str) else "unconfirmed",
)
try:
agent._flush_status_buffer()
except Exception:
pass
_final = (
"Context compression is already running for this session. "
"Please retry in a moment — your next message will be processed "
"once the concurrent compression finishes."
)
return {
"final_response": _final,
"messages": messages,
"completed": False,
"api_calls": api_call_count,
"error": _final,
"partial": True,
"failed": False,
"compression_deferred": True,
"session_id": agent.session_id,
}
def _sync_failover_system_message(agent, api_messages, active_system_prompt):
"""Refresh the in-flight system message after a provider failover.
@ -666,6 +800,136 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt):
return sp
def _apply_context_engine_selection(
agent: Any,
api_messages: List[Dict[str, Any]],
conversation_messages: List[Dict[str, Any]],
incoming_message: Optional[Dict[str, Any]],
*,
logger: Any,
) -> List[Dict[str, Any]]:
"""Run the optional per-turn ``ContextEngine.select_context()`` hook.
Returns the (possibly replaced) request message list. The hook is for
context *selection / routing* (retrieval, topic routing, role switching),
which is distinct from compression and fires every turn independent of
``should_compress()``.
Fail-open by design: a missing hook, any exception, or an invalid return
value yields the unmodified ``api_messages``. The result is request-only
persisted conversation history is never mutated here.
"""
engine = getattr(agent, "context_compressor", None)
if engine is None or not hasattr(engine, "select_context"):
return api_messages
# Skip the no-op base implementation so non-implementing engines —
# including the built-in ContextCompressor — pay nothing per request:
# no history copies below, no call. ``hasattr`` alone is not enough,
# because the ABC defines a default ``select_context`` that every engine
# inherits. Mirrors the base-method short-circuit in
# ``_notify_context_engine_turn_complete``. Lazy import avoids any import
# cycle with agent.context_engine.
try:
from agent.context_engine import ContextEngine as _CE
if getattr(engine.select_context, "__func__", None) is _CE.select_context:
return api_messages
except Exception:
pass
session_label = getattr(agent, "session_id", None) or "-"
# Pass shallow copies of the reference-only inputs so an engine that
# mutates them in place cannot alter persisted transcript state. Only
# ``request_messages`` (the per-call request list) is meant to be acted on,
# and it may be replaced wholesale via the return value — never mutated in
# place either. ``conversation_messages`` / ``incoming_message`` are
# read-only context; copying enforces the request-only contract rather than
# merely documenting it.
_conv_copy = [dict(m) if isinstance(m, dict) else m for m in conversation_messages] \
if conversation_messages is not None else None
_incoming_copy = dict(incoming_message) if isinstance(incoming_message, dict) else incoming_message
try:
selected = engine.select_context(
api_messages,
conversation_messages=_conv_copy,
incoming_message=_incoming_copy,
budget_tokens=getattr(engine, "context_length", 0) or 0,
)
except Exception:
logger.warning(
"Context engine select_context hook failed; using unmodified "
"request messages (session=%s)",
session_label,
exc_info=True,
)
return api_messages
if selected is None:
return api_messages
# Require a NON-EMPTY list of dicts. An empty list must fall open to the
# original request: ``all([])`` is ``True``, so without the emptiness check
# a ``[]`` returned by a buggy/failing engine would replace a valid request
# with an empty message list that the downstream sanitizers cannot restore,
# reaching the provider as an invalid request instead of failing open.
if isinstance(selected, list) and selected and all(isinstance(m, dict) for m in selected):
return selected
logger.warning(
"Context engine select_context returned an invalid value "
"(not a non-empty list of dicts); ignoring (session=%s)",
session_label,
)
return api_messages
def _notify_context_engine_turn_complete(
agent: Any,
messages: List[Dict[str, Any]],
*,
usage: Optional[Dict[str, Any]] = None,
logger: Any,
**meta: Any,
) -> None:
"""Notify the active context engine that a user turn has finished.
Calls the optional ``ContextEngine.on_turn_complete()`` observation hook
once per turn, after the assistant/tool loop has produced the finalized
transcript. The complement to ``select_context()`` (pre-request selection):
this lets an engine ingest / index / summarize the completed turn.
Fail-open: a missing or no-op hook, or any exception, is swallowed.
``messages`` is passed as a shallow copy so the engine cannot mutate the
persisted transcript.
"""
engine = getattr(agent, "context_compressor", None)
hook = getattr(engine, "on_turn_complete", None)
if engine is None or not callable(hook):
return
# Skip the no-op base implementation so non-implementing engines (incl.
# the built-in compressor) pay nothing per turn. Lazy import avoids any
# import cycle with agent.context_engine.
try:
from agent.context_engine import ContextEngine as _CE
if getattr(hook, "__func__", None) is _CE.on_turn_complete:
return
except Exception:
pass
try:
hook(
[dict(m) if isinstance(m, dict) else m for m in messages],
usage=usage,
**meta,
)
except Exception:
logger.warning(
"Context engine on_turn_complete hook failed (session=%s)",
getattr(agent, "session_id", None) or "-",
exc_info=True,
)
def run_conversation(
agent,
user_message: Any,
@ -805,6 +1069,13 @@ def run_conversation(
# over instead of spinning. Reset here so each turn starts fresh. See #26080.
agent._auth_pool_refresh_counts = {}
# Reset the per-turn usage holder forwarded to the context engine's
# on_turn_complete() observation hook. Set after each successful provider
# response (see below); left as None on turns that never reach a response
# (early failure / interrupt) so the hook receives None rather than a
# stale prior turn's usage.
agent._last_turn_usage = None
# Optional opt-in runtime: if api_mode == codex_app_server, hand the
# turn to the codex app-server subprocess (terminal/file ops/patching
# all run inside Codex). Default Hermes path is bypassed entirely.
@ -1053,7 +1324,28 @@ def run_conversation(
# Uses new dicts so the internal messages list retains the fields
# for Codex Responses compatibility.
if agent._should_sanitize_tool_calls():
agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model)
# In MoA mode, agent.model is the virtual preset name
# (e.g. "closed"), not the actual aggregator model. Use
# the resolved aggregator model so Gemini aggregators
# correctly preserve thought_signature (extra_content).
_sanitize_model = agent.model
if agent.provider == "moa":
if moa_config:
_agg = moa_config.get("aggregator") or {}
if _agg.get("model"):
_sanitize_model = _agg["model"]
if _sanitize_model == agent.model:
# Virtual-provider mode: no moa_config is threaded
# through run_conversation — the facade resolves the
# preset internally. Ask the facade for the resolved
# aggregator slot from the previous create() instead
# (set before any history replay that could carry
# thought_signature).
_moa_client = getattr(agent, "client", None)
_agg_slot = getattr(_moa_client, "last_aggregator_slot", None)
if _agg_slot and _agg_slot.get("model"):
_sanitize_model = _agg_slot["model"]
agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model)
# Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context
# The signature field helps maintain reasoning continuity
api_messages.append(api_msg)
@ -1070,9 +1362,9 @@ def run_conversation(
#
# Hermes invariant: the system prompt is built ONCE per session
# (cached on ``_cached_system_prompt``) and replayed verbatim on
# every turn. We send it as a single content string so the
# bytes are byte-stable across turns and upstream prompt caches
# stay warm.
# every turn. ``apply_anthropic_cache_control`` may split its stable
# prefix into content blocks on the wire, but the stored string and
# its byte-stability remain unchanged.
effective_system = active_system_prompt or ""
if agent.ephemeral_system_prompt:
effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip()
@ -1099,7 +1391,18 @@ def run_conversation(
aggregator=moa_config.get("aggregator") or {},
temperature=_preset_temperature(moa_config, "reference_temperature"),
aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"),
max_tokens=moa_config.get("reference_max_tokens"),
reference_max_tokens=moa_config.get("reference_max_tokens"),
# None = no per-preset override; inherit
# auxiliary.moa_reference.timeout via call_llm.
reference_timeout=(
float(moa_config["reference_timeout"])
if moa_config.get("reference_timeout")
else None
),
degraded_reference_policy=str(
moa_config.get("degraded_reference_policy") or "loud"
),
agent=agent,
)
if _moa_context:
for _msg in reversed(api_messages):
@ -1126,18 +1429,25 @@ def run_conversation(
for idx, pfm in enumerate(agent.prefill_messages):
api_messages.insert(sys_offset + idx, pfm.copy())
# Apply Anthropic prompt caching for Claude models on native
# Anthropic, OpenRouter, and third-party Anthropic-compatible
# gateways. Auto-detected: if ``_use_prompt_caching`` is set,
# inject cache_control breakpoints (system + last 3 messages)
# to reduce input token costs by ~75% on multi-turn
# conversations.
if agent._use_prompt_caching:
api_messages = apply_anthropic_cache_control(
api_messages,
cache_ttl=agent._cache_ttl,
native_anthropic=agent._use_native_cache_layout,
)
# Per-turn context selection hook (additive, no-op by default).
# Lets a context engine select/replace which context enters the
# prompt for THIS call only — retrieval, topic routing, role/branch
# switching — distinct from compression and independent of
# should_compress(). Request-only: persisted history is untouched, so
# caching/sanitization below operate on whatever the engine selected.
# Fail-open (see _apply_context_engine_selection).
_sel_incoming = (
messages[current_turn_user_idx]
if 0 <= current_turn_user_idx < len(messages)
else None
)
api_messages = _apply_context_engine_selection(
agent,
api_messages,
messages,
_sel_incoming,
logger=request_logger,
)
# Safety net: strip orphaned tool results / add stubs for missing
# results before sending to the API. Runs unconditionally — not
@ -1197,6 +1507,39 @@ def run_conversation(
# the OpenAI SDK. Sanitizing here prevents the 3-retry cycle.
_sanitize_messages_surrogates(api_messages)
# Apply Anthropic prompt caching for Claude models on native
# Anthropic, OpenRouter, and third-party Anthropic-compatible
# gateways. Auto-detected: if ``_use_prompt_caching`` is set, inject
# cache_control breakpoints for the static system prefix, full system
# prompt, and last two messages (or the legacy system-and-3 layout
# when no static prefix is available).
#
# Runs LAST, after every message mutation above. Marking earlier
# defeats the prefix stability the mutations exist to create:
# ``_apply_cache_marker`` rewrites ``content`` from a plain string
# into a ``[{"type": "text", ...}]`` block, so the marked messages
# no longer match the ``isinstance(content, str)`` test in the
# whitespace-normalization pass and silently keep their raw
# leading/trailing whitespace. A tool result ending in "\n" is
# therefore sent unstripped while it sits in the last-3 window and
# stripped once it rolls out of it — the same message, different
# bytes on consecutive turns, which breaks the prefix match at
# exactly the point the breakpoints were meant to protect. Marking
# last also keeps breakpoints off messages that the orphan sweep or
# the thinking-only drop is about to remove or merge away.
if agent._use_prompt_caching:
_static_system_prefix = getattr(agent, "_cached_system_prompt_static", None)
api_messages = apply_anthropic_cache_control(
api_messages,
cache_ttl=agent._cache_ttl,
native_anthropic=agent._use_native_cache_layout,
static_system_prefix=(
_static_system_prefix
if isinstance(_static_system_prefix, str)
else None
),
)
# Build a persistent-MoA request before measuring compression pressure.
# MoA reference output is injected into the aggregator prompt, but it
# is deliberately ephemeral and therefore absent from ``messages``.
@ -1354,36 +1697,52 @@ def run_conversation(
if _pre_api_status:
agent._emit_status(_pre_api_status)
_last_preflight_pressure = request_pressure_tokens
_pre_api_input = messages
messages, active_system_prompt = agent._compress_context(
messages,
system_message,
approx_tokens=request_pressure_tokens,
task_id=effective_task_id,
)
# Reset retry/empty-response state so the compacted request
# gets a fresh chance instead of inheriting stale recovery
# counters from the pre-compaction history.
agent._empty_content_retries = 0
agent._thinking_prefill_retries = 0
agent._last_content_with_tools = None
agent._last_content_tools_all_housekeeping = False
agent._mute_post_response = False
# Re-baseline the flush cursor for the compaction mode that just
# ran. Legacy session-rotation returns None (the child session has
# not seen the compacted transcript, so the next flush writes it
# whole); in-place compaction returns list(messages) because the
# compacted rows are already persisted under the same session id —
# leaving None there would re-append them, doubling the active
# context and retriggering compression. Mirrors the post-response
# and preflight compaction sites; see
# conversation_history_after_compression().
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
api_call_count -= 1
agent._api_call_count = api_call_count
agent.iteration_budget.refund()
continue
if messages is _pre_api_input and compression_skipped_due_to_lock(agent):
# #69870 lock-skip: another path holds this session's
# compression lock, so this pass no-oped. That is a temporary
# DEFER, not evidence about compressibility — refund the
# attempt (it must not burn the shared overflow-recovery
# budget toward compression_exhausted → gateway auto-reset,
# #9893/#35809) and leave the insufficient-progress blocker
# unarmed. Proceed with the current request: if it truly does
# not fit, the provider's 413/overflow handler returns the
# soft compression_deferred result with that stronger signal.
compression_attempts -= 1
_last_preflight_pressure = None
if pending_moa_prepared_request is _moa_prepared_request:
pending_moa_prepared_request = None
else:
# Reset retry/empty-response state so the compacted request
# gets a fresh chance instead of inheriting stale recovery
# counters from the pre-compaction history.
agent._empty_content_retries = 0
agent._thinking_prefill_retries = 0
agent._last_content_with_tools = None
agent._last_content_tools_all_housekeeping = False
agent._mute_post_response = False
# Re-baseline the flush cursor for the compaction mode that just
# ran. Legacy session-rotation returns None (the child session has
# not seen the compacted transcript, so the next flush writes it
# whole); in-place compaction returns list(messages) because the
# compacted rows are already persisted under the same session id —
# leaving None there would re-append them, doubling the active
# context and retriggering compression. Mirrors the post-response
# and preflight compaction sites; see
# conversation_history_after_compression().
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
api_call_count -= 1
agent._api_call_count = api_call_count
agent.iteration_budget.refund()
continue
elif (
agent.compression_enabled
and len(messages) > 1
@ -2021,6 +2380,8 @@ def run_conversation(
)
continue # Retry the API call
agent._turn_received_provider_response = True
# Check finish_reason before proceeding
if agent.api_mode == "codex_responses":
status = getattr(response, "status", None)
@ -2542,6 +2903,14 @@ def run_conversation(
"reasoning_tokens": canonical_usage.reasoning_tokens,
}
agent.context_compressor.update_from_response(usage_dict)
# Stash this response's canonical usage so the post-turn
# on_turn_complete() observation hook can forward it (the
# same dict shape passed to update_from_response). A turn
# may make several API calls; the engine's per-turn signal
# of interest is the cost/size of the latest assembled
# request, so we keep the most recent call's usage.
agent._last_turn_usage = dict(usage_dict)
elif getattr(
agent.context_compressor,
"awaiting_real_usage_after_compression",
@ -3869,10 +4238,23 @@ def run_conversation(
original_len = len(messages)
original_tokens = estimate_messages_tokens_rough(messages)
_overflow_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message, approx_tokens=approx_tokens,
task_id=effective_task_id,
)
if messages is _overflow_input and compression_skipped_due_to_lock(agent):
# #69870 lock-skip: the provider proved the request
# does not fit, but this compression pass no-oped only
# because another path holds the session's compression
# lock. Temporary defer, not exhaustion — refund the
# attempt and end the turn softly so the gateway does
# NOT auto-reset the session (#9893/#35809).
compression_attempts -= 1
agent._persist_session(messages, conversation_history)
return _compression_deferred_result(
agent, messages, api_call_count
)
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
@ -4110,10 +4492,23 @@ def run_conversation(
original_len = len(messages)
original_tokens = estimate_messages_tokens_rough(messages)
_overflow_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message, approx_tokens=approx_tokens,
task_id=effective_task_id,
)
if messages is _overflow_input and compression_skipped_due_to_lock(agent):
# #69870 lock-skip: the provider proved the request
# does not fit, but this compression pass no-oped only
# because another path holds the session's compression
# lock. Temporary defer, not exhaustion — refund the
# attempt and end the turn softly so the gateway does
# NOT auto-reset the session (#9893/#35809).
compression_attempts -= 1
agent._persist_session(messages, conversation_history)
return _compression_deferred_result(
agent, messages, api_call_count
)
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
@ -5354,6 +5749,10 @@ def run_conversation(
# flag so it can fire again if the model goes empty on
# a LATER tool round.
agent._post_tool_empty_retried = False
# A landed tool call means any earlier dropped-tool-call stall
# was recovered — refresh that budget too so it guards each
# stall independently rather than capping the whole run.
agent._dropped_toolcall_retries = 0
previous_msg = messages[-1] if messages else None
current_interim_visible = agent._interim_assistant_visible_text(assistant_msg)
@ -5516,14 +5915,27 @@ def run_conversation(
if callable(_clear_warn):
_clear_warn()
agent._safe_print(" ⟳ compacting context…")
_post_tool_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message,
approx_tokens=agent.context_compressor.last_prompt_tokens,
task_id=effective_task_id,
)
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
if (
messages is _post_tool_input
and compression_skipped_due_to_lock(agent)
):
# #69870 lock-skip: this pass no-oped because another
# path holds the session's compression lock — a
# temporary defer, not evidence about compressibility.
# Refund the attempt so a lock-loser tool loop does not
# burn the shared per-turn budget toward
# compression_exhausted (#9893/#35809).
compression_attempts -= 1
else:
conversation_history = conversation_history_after_compression(
agent, messages, conversation_history
)
elif agent.compression_enabled:
# Over threshold but compression is blocked (summary-LLM
# cooldown or anti-thrashing). Surface a deduped warning so
@ -5543,15 +5955,72 @@ def run_conversation(
_real_tokens,
int(getattr(_compressor, "threshold_tokens", 0) or 0),
)
# Proactive tool-result prune: reclaim re-sent history on
# large-window models long before should_compress() (≈50% of
# the window) would ever fire. Deterministic, no LLM call;
# protects the recent tail. No-op unless proactive_prune_tokens
# is configured and _real_tokens is above it — and even then
# the prune only commits when it reclaims at least
# proactive_prune_min_reclaim_tokens, so prompt-cache breaks
# stay episodic like compression's (the one sanctioned cache
# break) instead of firing every tool iteration. See
# ContextCompressor.prune_tool_results_only.
# getattr guard: plugin context engines predating the hook and
# minimal test doubles (SimpleNamespace compressors) lack the
# method — treat absence as a no-op.
_prune = getattr(_compressor, "prune_tool_results_only", None)
if callable(_prune):
try:
_pruned_msgs, _pruned_n = _prune(
messages, current_tokens=_real_tokens
)
except Exception:
logger.debug(
"proactive tool-result prune failed; skipping",
exc_info=True,
)
_pruned_msgs, _pruned_n = messages, 0
# Standard no-op caller contract: only commit when the
# engine returned a NEW list object with a non-zero count.
if _pruned_n and _pruned_msgs is not messages:
# Do NOT rebuild conversation_history here. Unlike the
# compression branch, the prune neither rotates the session
# nor calls archive_and_compact(), so there is no new
# persistence baseline to establish. _prune_old_tool_results
# returns per-message copies that preserve the
# _DB_PERSISTED_MARKER, so the marker-based flush dedup (see
# _flush_messages_to_session_db) already prevents both
# duplicate writes and dropped rows. Calling
# conversation_history_after_compression (a compaction-only
# helper keyed on the _last_compaction_in_place flag) would be
# a no-op at best, and on a stale in-place flag could seed
# this turn's fresh, not-yet-persisted rows into history_ids
# and skip writing them.
messages = _pruned_msgs
# Save session log incrementally (so progress is visible even if interrupted)
agent._session_messages = messages
# Touch activity before continuing so the gateway's
# inactivity monitor never sees a stale timestamp
# between tool completion and the start of the next
# API call. Without this, a tool-call result (which
# takes ~0s to process) followed by slow post-tool
# processing (compression, persist) and a slow
# follow-up API call can exceed the gateway inactivity
# timeout (HERMES_AGENT_TIMEOUT, default 1800s) and the
# gateway kills the session before the next activity
# touch fires (#69559, #69131).
agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}")
# Continue loop for next response
continue
else:
# No tool calls - this is the final response
# No tool calls - this is the final response.
# (Dropped tool-call recovery — finish_reason=="tool_calls" with
# an empty tool_calls array — is handled at the finalization
# chokepoint below, after final_msg is built, so it catches
# every path that reaches turn finalization, not just this one.)
final_response = assistant_message.content or ""
# Fix: unmute output when entering the no-tool-call branch
@ -5883,6 +6352,64 @@ def run_conversation(
final_msg = agent._build_assistant_message(assistant_message, finish_reason)
# ── Dropped tool-call recovery (copilot/Claude) ────────
# Some providers (observed: claude-opus-4.8 / claude-sonnet-4.5
# on GitHub Copilot, ~2026-07) return finish_reason="tool_calls"
# while the parsed tool_calls array is empty — the model
# signalled it wanted to act but the payload shipped no call.
# Reaching finalization with that mismatch means the turn is
# about to end with the task unstarted (the narration, which may
# be in content or only in the reasoning field, gets treated as
# the final answer). Re-prompt (bounded to 3 CONSECUTIVE stalls;
# the budget resets after any successful tool round) to make the
# model emit the call instead of exiting. finish_reason="stop"
# text finishes never enter this guard.
if (
finish_reason == "tool_calls"
and not assistant_message.tool_calls
and getattr(agent, "_dropped_toolcall_retries", 0) < 3
):
agent._dropped_toolcall_retries = getattr(agent, "_dropped_toolcall_retries", 0) + 1
logger.warning(
"finish_reason=tool_calls with empty tool_calls array "
"(narration only) — re-prompting to emit the call "
"(retry %d/3, model=%s provider=%s)",
agent._dropped_toolcall_retries, agent.model, agent.provider,
)
agent._emit_status(
"↻ Model signaled a tool call but sent none — "
f"re-prompting ({agent._dropped_toolcall_retries}/3)"
)
# Both halves of the re-prompt pair are ephemeral recovery
# scaffolding (mirrors the empty-response nudge pattern):
# the interim narration-only assistant turn exists solely to
# keep role alternation valid for the nudge, and the nudge
# exists solely to drive the retry. Flag both so the
# persistence layer never writes them to the durable
# transcript and the finalization pop below can strip an
# unanswered tail pair. A recovered (answered) pair stays
# buried mid-list in live memory but is skipped by the
# flush regardless of position.
final_msg["_dropped_toolcall_nudge"] = True
messages.append(final_msg)
messages.append({
"role": "user",
"content": (
"Your previous turn indicated a tool call but none was "
"included. Do not narrate a plan or restate intent — issue "
"the actual tool call now to continue the task."
),
"_dropped_toolcall_nudge": True,
})
agent._session_messages = messages
final_response = None
continue
# Reached finalization without the dropped-tool-call mismatch —
# a genuine turn end. Clear the consecutive-stall budget so the
# next turn starts fresh.
agent._dropped_toolcall_retries = 0
# Pop thinking-only prefill and empty-response retry
# scaffolding before appending either a final response or a
# verification-stop follow-up. These internal turns are only
@ -5895,6 +6422,7 @@ def run_conversation(
messages[-1].get("_thinking_prefill")
or messages[-1].get("_empty_recovery_synthetic")
or messages[-1].get("_empty_terminal_sentinel")
or messages[-1].get("_dropped_toolcall_nudge")
)
):
messages.pop()

View file

@ -512,7 +512,7 @@ class CopilotACPClient:
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
text=True, encoding='utf-8', errors='replace',
bufsize=1,
cwd=self._acp_cwd,
env=_build_subprocess_env(),
@ -708,7 +708,7 @@ class CopilotACPClient:
if block_error:
raise PermissionError(block_error)
try:
content = path.read_text()
content = path.read_text(encoding="utf-8")
except FileNotFoundError:
content = ""
line = params.get("line")
@ -736,7 +736,7 @@ class CopilotACPClient:
if denied:
raise PermissionError(denied)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(str(params.get("content") or ""))
path.write_text(str(params.get("content") or ""), encoding="utf-8")
response = {
"jsonrpc": "2.0",
"id": message_id,

View file

@ -594,6 +594,14 @@ class CredentialPool:
# Re-armed to None on every successful selection so a recover→re-exhaust
# transition logs promptly instead of being swallowed by a stale window.
self._last_no_entries_log_at: Optional[float] = None
# #70401: consecutive mark_exhausted_and_rotate() calls whose supplied
# credential identity matched no pool entry (OAuth wrappers whose
# runtime key rotates, entries pruned by another process, ...). These
# rotations mark nothing exhausted, so without a cap the pool can
# never converge to "no available entries" and the caller's 401 retry
# loop runs unbounded and non-interruptible. Reset whenever a real
# entry is identified or an escape path returns None.
self._unmatched_rotation_streak: int = 0
def has_credentials(self) -> bool:
with self._lock:
@ -622,6 +630,28 @@ class CredentialPool:
with self._lock:
return self._current_unlocked()
def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]:
"""Return the stable id for the runtime credential in use.
Prefer the current selection when it still supplies ``api_key_hint``.
If the cursor was cleared, fall back to an unambiguous key match.
"""
with self._lock:
current = self._current_unlocked()
if current is not None and (
api_key_hint is None
or current.runtime_api_key == api_key_hint
):
return current.id
if api_key_hint is None:
return None
matches = [
entry
for entry in self._entries
if entry.runtime_api_key == api_key_hint
]
return matches[0].id if len(matches) == 1 else None
def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
"""Swap an entry in-place by id, preserving sort order."""
for idx, entry in enumerate(self._entries):
@ -1562,7 +1592,13 @@ class CredentialPool:
def select(self) -> Optional[PooledCredential]:
with self._lock:
return self._select_unlocked()
entry = self._select_unlocked()
if entry is not None:
# A normal (non-recovery) selection starts a fresh episode —
# don't let a leftover unmatched-rotation streak from an old
# failure trip the #70401 bound early next time.
self._unmatched_rotation_streak = 0
return entry
def _available_entries(self, *, clear_expired: bool = False, refresh: bool = False) -> List[PooledCredential]:
"""Return entries not currently in exhaustion cooldown.
@ -1763,10 +1799,17 @@ class CredentialPool:
status_code: Optional[int],
error_context: Optional[Dict[str, Any]] = None,
api_key_hint: Optional[str] = None,
credential_id: Optional[str] = None,
) -> Optional[PooledCredential]:
with self._lock:
entry = None
if api_key_hint:
identity_supplied = bool(credential_id or api_key_hint)
if credential_id:
entry = next(
(e for e in self._entries if e.id == credential_id),
None,
)
if entry is None and api_key_hint:
# Prefer the specific entry whose API key matches the one that
# actually failed. When this pool was freshly loaded from disk
# (another process already rotated), current() is None and
@ -1775,20 +1818,59 @@ class CredentialPool:
(e for e in self._entries if e.runtime_api_key == api_key_hint),
None,
)
if entry is None:
# The failed key is identifiable but matches no entry
# (rotated away, or a wrapper whose runtime key differs).
# Falling through to current()/_select_unlocked() would
# mark an INNOCENT healthy key exhausted for the full
# cooldown TTL. Don't guess — just hand back a fresh
# selection so the caller can retry.
logger.info(
"credential pool: failed key hint matched no %s entry; "
"rotating without marking any credential exhausted",
if entry is None and identity_supplied:
# The failed credential is identifiable but matches no entry
# (rotated away, or a wrapper whose runtime key differs).
# Falling through to current()/_select_unlocked() would mark an
# innocent healthy key exhausted for the full cooldown TTL.
#
# #70401: this branch must still be BOUNDED. With OAuth-token
# auth the upstream 401's key hint never matches any entry's
# ``runtime_api_key``, so every retry lands here, nothing is
# ever marked exhausted, and the pool can never reach the
# "no available entries" state — the caller retries the same
# dead token forever (~6/sec, starving the event loop so chat
# interrupts are never processed). The single-entry case
# below already escapes; multi-entry pools could still
# ping-pong A→B→A indefinitely without marking anything.
# Cap consecutive no-mark rotations at one full lap of the
# available entries: past that, every candidate has been
# handed back at least once without recovery, so stop
# guessing and surface the error (no cooldown is written for
# anybody — healthy keys stay available for the next turn).
self._unmatched_rotation_streak += 1
available_count = len(self._available_entries())
if self._unmatched_rotation_streak > max(available_count, 1):
logger.warning(
"credential pool: failed credential identity matched no "
"%s entry for %d consecutive rotations (pool size %d) — "
"surfacing the error instead of rotating again",
self.provider,
self._unmatched_rotation_streak,
available_count,
)
self._unmatched_rotation_streak = 0
self._current_id = None
return self._select_unlocked()
return None
logger.info(
"credential pool: failed credential identity matched no %s "
"entry; rotating without marking any credential exhausted",
self.provider,
)
self._current_id = None
next_entry = self._select_unlocked()
if next_entry is not None and len(self._available_entries()) == 1:
# A single-entry pool cannot rotate. Returning its only
# entry reports a successful recovery without changing
# the credential, so the caller retries the same 401
# indefinitely. Let fallback/error propagation proceed.
self._unmatched_rotation_streak = 0
self._current_id = None
return None
return next_entry
# A real entry was identified — any prior unmatched-rotation
# streak is stale (this mark WILL advance pool state).
self._unmatched_rotation_streak = 0
if entry is None:
entry = self._current_unlocked() or self._select_unlocked()
if entry is None:
@ -1806,12 +1888,13 @@ class CredentialPool:
# disconnects (a ~2.5min hang with no error surfaced to the user).
# Mark every entry sharing the failed key so the pool can reach the
# "no available entries" state and let the error propagate.
if api_key_hint:
failed_runtime_key = getattr(entry, "runtime_api_key", None)
if identity_supplied and failed_runtime_key:
siblings_marked = False
for sibling in self._entries:
if sibling.id == entry.id:
continue
if sibling.runtime_api_key == api_key_hint:
if sibling.runtime_api_key == failed_runtime_key:
self._mark_exhausted(
sibling, status_code, error_context, persist=False
)
@ -1885,9 +1968,11 @@ class CredentialPool:
return self._try_refresh_current_unlocked()
def try_refresh_matching(
self, api_key_hint: Optional[str] = None
self,
api_key_hint: Optional[str] = None,
credential_id: Optional[str] = None,
) -> Optional[PooledCredential]:
"""Force-refresh the entry that supplied ``api_key_hint``.
"""Force-refresh the entry that supplied the failed request.
Direct provider integrations may reload the pool after a request has
already failed, so they cannot rely on ``current_id`` identifying the
@ -1897,17 +1982,29 @@ class CredentialPool:
"""
with self._lock:
entry = None
if api_key_hint:
if credential_id:
entry = next(
(
candidate
for candidate in self._entries
if candidate.runtime_api_key == api_key_hint
if candidate.id == credential_id
),
None,
)
else:
entry = self._current_unlocked() or self._select_unlocked(refresh=False)
if entry is None:
if api_key_hint:
entry = next(
(
candidate
for candidate in self._entries
if candidate.runtime_api_key == api_key_hint
),
None,
)
else:
entry = self._current_unlocked() or self._select_unlocked(
refresh=False
)
if entry is None:
return None
self._current_id = entry.id

View file

@ -164,7 +164,7 @@ def _remove_env_source(provider: str, removed) -> RemovalResult:
if env_path.exists():
env_in_dotenv = any(
line.strip().startswith(f"{env_var}=")
for line in env_path.read_text(errors="replace").splitlines()
for line in env_path.read_text(errors="replace", encoding="utf-8").splitlines()
)
except OSError:
pass

View file

@ -325,7 +325,7 @@ def apply_automatic_transitions(now: Optional[datetime] = None) -> Dict[str, int
counts = {"marked_stale": 0, "archived": 0, "reactivated": 0, "checked": 0, "seeded": 0}
for row in _u.agent_created_report():
for row in _u.curated_report():
counts["checked"] += 1
name = row["name"]
if row.get("pinned"):
@ -422,7 +422,9 @@ CURATOR_REVIEW_PROMPT = (
"INSTRUCTIONS AND EXPERIENTIAL KNOWLEDGE. A collection of hundreds of "
"narrow skills where each one captures one session's specific bug is "
"a FAILURE of the library — not a feature. An agent searching skills "
"matches on descriptions, not on exact names; one broad umbrella "
"matches on descriptions, not on exact names (note: long descriptions "
"are truncated to 57 chars in the system prompt skill index — keep the "
"trigger class in that window). One broad umbrella "
"skill with labeled subsections beats five narrow siblings for "
"discoverability, not the other way around.\n\n"
"The right target shape is CLASS-LEVEL skills with rich SKILL.md "
@ -1470,15 +1472,16 @@ def _render_report_markdown(p: Dict[str, Any]) -> str:
# ---------------------------------------------------------------------------
def _render_candidate_list() -> str:
"""Human/agent-readable list of agent-created skills with usage stats."""
rows = skill_usage.agent_created_report()
"""Human/agent-readable list of curator-managed skills with usage stats."""
rows = skill_usage.curated_report()
if not rows:
return "No agent-created skills to review."
return "No curator-managed skills to review."
cron_referenced = _cron_referenced_skills()
lines = [f"Agent-created skills ({len(rows)}):\n"]
lines = [f"Curator-managed skills ({len(rows)}):\n"]
for r in rows:
lines.append(
f"- {r['name']} "
f"provenance={r.get('provenance', 'agent')} "
f"state={r['state']} "
f"pinned={'yes' if r.get('pinned') else 'no'} "
f"cron={'yes' if r['name'] in cron_referenced else 'no'} "
@ -1531,7 +1534,7 @@ def run_curator_review(
if dry_run:
# Count candidates without mutating state.
try:
report = skill_usage.agent_created_report()
report = skill_usage.curated_report()
counts = {
"checked": len(report),
"marked_stale": 0,
@ -1584,7 +1587,7 @@ def run_curator_review(
nonlocal auto_summary
# Snapshot skill state BEFORE the LLM pass so the report can diff.
try:
before_report = skill_usage.agent_created_report()
before_report = skill_usage.curated_report()
except Exception:
before_report = []
before_names = {r.get("name") for r in before_report if isinstance(r, dict)}
@ -1610,7 +1613,7 @@ def run_curator_review(
state2["last_run_duration_seconds"] = elapsed
state2["last_run_summary"] = final_summary
try:
after_report = skill_usage.agent_created_report()
after_report = skill_usage.curated_report()
except Exception:
after_report = []
try:
@ -1697,7 +1700,7 @@ def run_curator_review(
try:
rename_lines = _build_rename_summary(
before_names=before_names,
after_report=skill_usage.agent_created_report(),
after_report=skill_usage.curated_report(),
tool_calls=llm_meta.get("tool_calls", []) or [],
model_final=llm_meta.get("final", "") or "",
)
@ -1715,7 +1718,7 @@ def run_curator_review(
# reporting bug never breaks the curator itself. Report path is
# recorded in state so `hermes curator status` can point at it.
try:
after_report = skill_usage.agent_created_report()
after_report = skill_usage.curated_report()
except Exception:
after_report = []
try:

View file

@ -270,8 +270,12 @@ def _translate_tool_call_to_gemini(tool_call: Dict[str, Any]) -> Dict[str, Any]:
}
}
thought_signature = _tool_call_extra_signature(tool_call)
if thought_signature:
part["thoughtSignature"] = thought_signature
# Fallback sentinel for cross-provider tool_calls (e.g. fallback from
# xAI/Anthropic to Gemini, where the original tool_call carries no
# Gemini thoughtSignature). Mirrors gemini_cloudcode_adapter.py:106.
# Without this, Gemini 3 thinking models reject replayed history with
# 400 INVALID_ARGUMENT on the missing thoughtSignature.
part["thoughtSignature"] = thought_signature or "skip_thought_signature_validator"
return part

View file

@ -25,7 +25,8 @@ Language resolution order:
3. ``display.language`` from config.yaml
4. ``"en"`` (baseline)
Supported languages: en, zh, ja, de, es, fr, tr, uk. Unknown values fall back to en.
Supported languages: en, zh, zh-hant, ja, de, es, fr, tr, uk, af, ko, it, ga,
pt, ru, hu, ar. Unknown values fall back to en.
"""
from __future__ import annotations
@ -41,7 +42,7 @@ logger = logging.getLogger(__name__)
SUPPORTED_LANGUAGES: tuple[str, ...] = (
"en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk",
"af", "ko", "it", "ga", "pt", "ru", "hu",
"af", "ko", "it", "ga", "pt", "ru", "hu", "ar",
)
DEFAULT_LANGUAGE = "en"
@ -78,6 +79,9 @@ _LANGUAGE_ALIASES: dict[str, str] = {
"russian": "ru", "русский": "ru", "ru-ru": "ru",
# Hungarian
"hungarian": "hu", "magyar": "hu", "hu-hu": "hu",
# Arabic — bare "arabic"/endonym plus the common regional BCP-47 tags.
"arabic": "ar", "العربية": "ar",
"ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar",
}
_catalog_cache: dict[str, dict[str, str]] = {}

View file

@ -56,6 +56,8 @@ from pathlib import Path
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set
from urllib.parse import quote, unquote
from hermes_cli._subprocess_compat import windows_hide_flags
from agent.lsp.protocol import (
ERROR_CONTENT_MODIFIED,
ERROR_METHOD_NOT_FOUND,
@ -294,6 +296,12 @@ class LSPClient:
cmd = self._command
if sys.platform == "win32":
cmd = self._win_wrap_cmd(cmd)
# Suppress the cmd.exe console window that would otherwise flash
# every time we launch a ``.cmd``-wrapped language server
# (e.g. pyright-langserver.CMD) from a console-less host such as
# a VS Code/Zed extension running the ACP adapter.
# windows_hide_flags() is CREATE_NO_WINDOW on Windows, 0 on POSIX.
creationflags = windows_hide_flags()
try:
# start_new_session=True detaches the LSP server into its own
@ -312,6 +320,7 @@ class LSPClient:
env=env,
cwd=self._cwd,
start_new_session=True,
creationflags=creationflags,
)
except FileNotFoundError as e:
raise LSPProtocolError(

View file

@ -35,6 +35,8 @@ import threading
from pathlib import Path
from typing import Any, Dict, Optional
from hermes_cli._subprocess_compat import windows_hide_flags
logger = logging.getLogger("agent.lsp.install")
# Package-name → install-strategy hint registry. Each entry is a
@ -265,9 +267,10 @@ def _install_npm(
[npm, "install", "--prefix", str(staging), "--silent", "--no-fund", "--no-audit", *install_targets],
check=False,
capture_output=True,
text=True,
text=True, encoding="utf-8", errors="replace",
timeout=300,
stdin=subprocess.DEVNULL,
creationflags=windows_hide_flags(),
)
if proc.returncode != 0:
logger.warning(
@ -313,10 +316,11 @@ def _install_go(pkg: str, bin_name: str) -> Optional[str]:
[go, "install", pkg],
check=False,
capture_output=True,
text=True,
text=True, encoding="utf-8", errors="replace",
timeout=600,
env=env,
stdin=subprocess.DEVNULL,
creationflags=windows_hide_flags(),
)
if proc.returncode != 0:
logger.warning(

View file

@ -80,8 +80,17 @@ def normalize_tool_schema(schema: Any) -> Optional[Dict[str, Any]]:
return schema
def memory_provider_tools_enabled(enabled_toolsets: Optional[List[str]]) -> bool:
def memory_provider_tools_enabled(
enabled_toolsets: Optional[List[str]],
disabled_toolsets: Optional[List[str]] = None,
*,
memory_tool_present: bool = False,
) -> bool:
"""Return whether external memory-provider tools should be exposed."""
if disabled_toolsets and "memory" in disabled_toolsets:
return False
if memory_tool_present:
return True
if enabled_toolsets is None:
return True
if not enabled_toolsets:
@ -110,9 +119,10 @@ def inject_memory_provider_tools(agent: Any) -> int:
for tool in tools
if isinstance(tool, dict)
}
if (
"memory" not in existing_tool_names
and not memory_provider_tools_enabled(getattr(agent, "enabled_toolsets", None))
if not memory_provider_tools_enabled(
getattr(agent, "enabled_toolsets", None),
getattr(agent, "disabled_toolsets", None),
memory_tool_present="memory" in existing_tool_names,
):
return 0

File diff suppressed because it is too large Load diff

View file

@ -215,6 +215,7 @@ DEFAULT_CONTEXT_LENGTHS = {
# OpenRouter-prefixed models resolve via OpenRouter live API or models.dev.
"claude-fable-5": 1000000,
"claude-fable": 1000000,
"claude-opus-5": 1000000,
"claude-sonnet-5": 1000000,
"claude-opus-4-8": 1000000,
"claude-opus-4.8": 1000000,

View file

@ -117,7 +117,7 @@ def record_nous_rate_limit(
# Atomic write: write to temp file + rename
fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp")
try:
with os.fdopen(fd, "w") as f:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(state, f)
atomic_replace(tmp_path, path)
except Exception:

View file

@ -192,7 +192,13 @@ SKILLS_GUIDANCE = (
"skill with skill_manage so you can reuse it next time.\n"
"When using a skill and finding it outdated, incomplete, or wrong, "
"patch it immediately with skill_manage(action='patch') — don't wait to be asked. "
"Skills that aren't maintained become liabilities."
"Skills that aren't maintained become liabilities.\n"
"\n"
"## Skill Safety Rule\n"
"1. **UNAVAILABLE** — If a skill placeholder contains `[SKILL_PRUNED]`, the skill content was lost in compression and is inaccessible.\n"
"2. **RELOAD** — Before performing any action that depends on a skill, re-check its content with `skill_view(name='...')` if it shows `[SKILL_PRUNED]`.\n"
"3. **WAIT** — If a skill is loading or was just pruned, wait for the reload confirmation before proceeding.\n"
"4. **DEDUP** — After reloading a pruned skill, **ignore any remaining `[SKILL_PRUNED]` markers for that same skill** — they are historical artifacts from previous compactions and do not need further action."
)
KANBAN_GUIDANCE = (

View file

@ -1,9 +1,11 @@
"""Anthropic prompt caching strategy.
Single layout: ``system_and_3``. 4 cache_control breakpoints system
prompt + last 3 non-system messages, all at the same TTL (5m or 1h).
Reduces input token costs by ~75% on multi-turn conversations within a
single session.
The default layout uses 4 cache_control breakpoints: the static system
prefix, the end of the system prompt, and the last 2 non-system messages.
When a static system prefix is unavailable, it falls back to one system
breakpoint plus the last 3 messages. All markers use the same TTL (5m or 1h).
This preserves intra-session caching while allowing new sessions to reuse the
stable system-prompt prefix.
Pure functions -- no class state, no AIAgent dependency.
"""
@ -81,15 +83,55 @@ def _build_marker(ttl: str) -> Dict[str, str]:
return marker
def _apply_system_cache_markers(
message: dict,
cache_marker: dict,
static_system_prefix: str | None,
*,
native_anthropic: bool,
) -> int:
"""Mark the static system prefix and full prompt when they can be split.
The system prompt remains one stored string. Splitting it only in the
outgoing request keeps session persistence and non-Anthropic transports
unchanged while making the stable prefix independently cacheable.
"""
content = message.get("content")
if (
isinstance(static_system_prefix, str)
and static_system_prefix
and isinstance(content, str)
and content.startswith(static_system_prefix)
):
suffix = content[len(static_system_prefix):]
if suffix:
message["content"] = [
{
"type": "text",
"text": static_system_prefix,
"cache_control": cache_marker,
},
{"type": "text", "text": suffix, "cache_control": cache_marker},
]
return 2
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
return 1
def apply_anthropic_cache_control(
api_messages: List[Dict[str, Any]],
cache_ttl: str = "5m",
native_anthropic: bool = False,
static_system_prefix: str | None = None,
) -> List[Dict[str, Any]]:
"""Apply system_and_3 caching strategy to messages for Anthropic models.
"""Apply Anthropic cache-control markers to API messages.
Places up to 4 cache_control breakpoints: system prompt + last 3 non-system
messages, all at the same TTL.
When ``static_system_prefix`` exactly matches the beginning of a string
system prompt, it receives an early marker and the full system prompt gets
a trailing marker. The remaining two markers target the latest cacheable
non-system messages. Without that prefix, the legacy system-and-3 layout
is retained.
Returns:
Deep copy of messages with cache_control breakpoints injected.
@ -103,8 +145,12 @@ def apply_anthropic_cache_control(
breakpoints_used = 0
if messages[0].get("role") == "system":
_apply_cache_marker(messages[0], marker, native_anthropic=native_anthropic)
breakpoints_used += 1
breakpoints_used = _apply_system_cache_markers(
messages[0],
marker,
static_system_prefix,
native_anthropic=native_anthropic,
)
remaining = 4 - breakpoints_used
non_sys = [

View file

@ -0,0 +1,8 @@
"""Egress proxy integrations.
Currently ships an iron-proxy (ironsh/iron-proxy) wrapper that intercepts
outbound traffic from remote terminal sandboxes and swaps proxy tokens
for real upstream credentials at the network edge.
Design notes live in :mod:`agent.proxy_sources.iron_proxy`.
"""

File diff suppressed because it is too large Load diff

View file

@ -102,6 +102,7 @@ _REASONING_STALE_TIMEOUT_FLOORS: tuple[tuple[str, int], ...] = (
# ``claude-opus-4`` so non-thinking Claude 3.x or future
# non-reasoning Claude variants don't match.
("claude-opus-4", 240),
("claude-opus-5", 240),
("claude-sonnet-5", 180),
("claude-sonnet-4.5", 180),
("claude-sonnet-4.6", 180),

View file

@ -295,7 +295,7 @@ def run_secret_cli(
list(argv),
env=env,
capture_output=True,
text=True,
text=True, encoding="utf-8", errors="replace",
timeout=timeout,
stdin=subprocess.DEVNULL,
)

View file

@ -200,7 +200,7 @@ def _platform_asset_name() -> str:
res = subprocess.run(
["ldd", "--version"],
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=2,
stdin=subprocess.DEVNULL,
)
@ -684,7 +684,7 @@ def _run_bws_list(
cmd,
env=env,
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=_BWS_RUN_TIMEOUT,
stdin=subprocess.DEVNULL,
)

View file

@ -464,7 +464,7 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
input=stdin_json,
capture_output=True,
timeout=spec.timeout,
text=True,
text=True, encoding='utf-8', errors='replace',
shell=False,
**_popen_kwargs,
)
@ -632,7 +632,7 @@ def allowlist_path() -> Path:
def load_allowlist() -> Dict[str, Any]:
"""Return the parsed allowlist, or an empty skeleton if absent."""
try:
raw = json.loads(allowlist_path().read_text())
raw = json.loads(allowlist_path().read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError, OSError):
return {"approvals": []}
if not isinstance(raw, dict):

View file

@ -54,6 +54,21 @@ _BUNDLE_MARKER = " skill bundle,"
_BUNDLE_USER_INSTRUCTION = "\nUser instruction: "
_BUNDLE_FIRST_SKILL_BLOCK = "\n\n[Loaded as part of the "
# The skill name sits in the first quoted span of the activation note, for both
# the single-skill and the bundle header ("work" / "/clean /work").
_SKILL_NAME_RE = re.compile(re.escape(_SKILL_INVOCATION_PREFIX) + r'"([^"]*)"')
# SQL LIKE pattern matching a skill-expanded turn, for listing queries that
# have to recognize scaffolding before the row reaches Python. The prefix
# contains no LIKE wildcards (`%`, `_`), so it needs no ESCAPE clause.
SKILL_SCAFFOLD_SQL_LIKE = _SKILL_INVOCATION_PREFIX + "%"
# Marks where a preview query joined the head and tail of a long scaffolded
# message. ``describe_skill_invocation`` may hand back a span that runs across
# the joint (a bundle instruction cut off by the head window); callers cut the
# description there rather than show the skill body on the far side.
SKILL_EXCERPT_JOINT = "\x1e"
def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
"""Recover the user's instruction from a slash-skill-expanded turn.
@ -82,6 +97,41 @@ def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
return None
def describe_skill_invocation(content: Any) -> Optional[str]:
"""Render a slash-skill-expanded turn the way the user typed it.
The expanded message embeds the whole skill body, so any surface that
summarizes a user turn from its raw content session titles, sidebar
previews, the ``/rewind`` picker otherwise shows the skill's own prose
as if the user had written it. That is how a skill's opening line ends up
as a session title.
Returns ``"/work — fix the title leak"``, or ``"/work"`` for a bare
invocation, or ``None`` when *content* is not skill scaffolding (the
caller should then summarize it as an ordinary message).
"""
if not isinstance(content, str) or not content.startswith(_SKILL_INVOCATION_PREFIX):
return None
match = _SKILL_NAME_RE.match(content)
name = (match.group(1) if match else "").strip()
# Bundle headers already carry their typed "/a /b" keys; a single skill is
# a bare name.
label = name if name.startswith("/") else f"/{name}"
instruction = extract_user_instruction_from_skill_message(content)
if instruction and instruction is not content:
# An excerpted message (head + tail, joined by SKILL_EXCERPT_JOINT) can
# put the joint inside the matched span — keep only the side the
# instruction marker was found on.
instruction = instruction.split(SKILL_EXCERPT_JOINT)[0]
instruction = " ".join(instruction.split())
if instruction:
return f"{label}{instruction}" if name else instruction
return label if name else None
def _extract_single_skill_user_instruction(message: str) -> Optional[str]:
# Single-skill format appends the user instruction after the skill body, so
# the last occurrence is the user-provided one; the body may quote this text.
@ -453,8 +503,8 @@ def reload_skills() -> Dict[str, Any]:
}
``description`` is the skill's full SKILL.md frontmatter
``description:`` field the same string the system prompt renders
as `` - name: description`` for pre-existing skills.
``description:`` field. Note: the system prompt skill index
truncates this to the first 57 chars; see ``extract_skill_description``.
"""
# Snapshot pre-reload state (name -> description) from the current
# slash-command cache. Using dicts lets the post-rescan diff carry

View file

@ -74,7 +74,7 @@ def run_inline_shell(command: str, cwd: Path | None, timeout: int) -> str:
["bash", "-c", command],
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=max(1, int(timeout)),
check=False,
stdin=subprocess.DEVNULL,

View file

@ -779,18 +779,31 @@ def resolve_skill_config_values(
# ── Description extraction ────────────────────────────────────────────────
SKILL_PROMPT_DESC_LIMIT = 60
def _normalize_skill_description(frontmatter: Dict[str, Any]) -> str:
"""Normalize a skill's description field for comparison/truncation."""
raw_desc = frontmatter.get("description", "")
return str(raw_desc).strip().strip("'\"") if raw_desc else ""
def extract_skill_description(frontmatter: Dict[str, Any]) -> str:
"""Extract a truncated description from parsed frontmatter."""
raw_desc = frontmatter.get("description", "")
if not raw_desc:
"""Extract a system-prompt-length description from parsed frontmatter."""
desc = _normalize_skill_description(frontmatter)
if not desc:
return ""
desc = str(raw_desc).strip().strip("'\"")
if len(desc) > 60:
return desc[:57] + "..."
if len(desc) > SKILL_PROMPT_DESC_LIMIT:
return desc[:SKILL_PROMPT_DESC_LIMIT - 3] + "..."
return desc
def is_skill_description_truncated_for_prompt(frontmatter: Dict[str, Any]) -> bool:
"""True when the description will be truncated in the system prompt skill index."""
desc = _normalize_skill_description(frontmatter)
return len(desc) > SKILL_PROMPT_DESC_LIMIT
# ── File iteration ────────────────────────────────────────────────────────

View file

@ -31,7 +31,8 @@ def _skip_ssl_guard_enabled() -> bool:
def _repair_hint() -> str:
return (
"Repair: python -m pip install --force-reinstall certifi openai httpx\n"
"Repair: run `hermes doctor --fix` (auto-reinstalls certifi), or "
"manually: python -m pip install --force-reinstall certifi openai httpx\n"
"If you configured a custom corporate CA bundle, fix or unset the "
"broken CA bundle environment variable."
)

View file

@ -12,9 +12,11 @@ Three tiers are joined with ``\\n\\n``:
* ``stable`` identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool
guidance, computer-use guidance, nous subscription block, tool-use
enforcement guidance + per-model operational guidance, skills prompt,
alibaba model-name workaround, environment hints, platform hints.
alibaba model-name workaround, environment hints, coding guidance,
platform hints.
* ``context`` caller-supplied ``system_message`` plus context files
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``.
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``,
plus the session's coding-workspace snapshot.
* ``volatile`` memory snapshot, USER.md profile, external memory
provider block, timestamp/session/model/provider line.
@ -145,14 +147,14 @@ def _tui_embedded_pane_clarifier(hint: str) -> str:
def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]:
"""Assemble the system prompt as three ordered parts.
"""Assemble the system prompt as three ordered cache tiers.
Returns a dict with three keys:
* ``stable`` identity, tool guidance, skills prompt,
environment hints, platform hints, model-family operational
guidance.
* ``context`` context files (AGENTS.md, .cursorrules, etc.)
and caller-supplied system_message.
* ``stable`` the cross-session-stable prefix, through the coding
operating brief when a workspace snapshot follows.
* ``context`` the workspace snapshot followed by the remaining
session-stable guidance, context files, and caller-supplied
system_message.
* ``volatile`` memory snapshot, user profile, external
memory provider block, timestamp line.
@ -345,25 +347,35 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
stable_parts.append(_env_hints)
# Coding posture (base Hermes, any interactive coding surface in a code
# workspace — see agent/coding_context.py). The operating brief + the live
# git/workspace snapshot are built once here and cached for the session;
# the snapshot is never re-probed per turn (that would break the prompt
# cache), so the brief tells the model to re-check git before relying on it.
# workspace — see agent/coding_context.py). Keep the operating brief in
# the cross-session-stable prefix, while placing the live git/workspace
# snapshot behind its own cache boundary. The post-snapshot blocks must
# stay in their historical position after the workspace snapshot.
coding_workspace_parts: List[str] = []
coding_trailing_parts: List[str] = []
if agent.valid_tool_names:
try:
from agent.coding_context import coding_system_blocks
from agent.coding_context import coding_system_prompt_parts
stable_parts.extend(
coding_system_blocks(
platform=agent.platform,
cwd=resolve_context_cwd(),
model=agent.model,
)
coding_prefix_parts, coding_workspace_parts, coding_trailing_parts = coding_system_prompt_parts(
platform=agent.platform,
cwd=resolve_context_cwd(),
model=agent.model,
)
stable_parts.extend(coding_prefix_parts)
except Exception:
# Coding-context probing must never block prompt build.
pass
# Guidance assembled after the coding posture historically followed the
# workspace snapshot. With no snapshot, the coding tail instead remains
# directly after the coding prefix in the cacheable prefix.
if coding_workspace_parts:
post_workspace_parts: List[str] = []
else:
stable_parts.extend(coding_trailing_parts)
post_workspace_parts = stable_parts
# Local Python toolchain probe — names python/pip/uv/PEP-668 state when
# something is non-default so the model can pick the right install
# strategy without discovering by failure. Emits a single line; emits
@ -376,7 +388,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
from tools.env_probe import get_environment_probe_line
_probe_line = get_environment_probe_line()
if _probe_line:
stable_parts.append(_probe_line)
post_workspace_parts.append(_probe_line)
except Exception:
# Probe failure must never block prompt build.
pass
@ -394,7 +406,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
except Exception:
active_profile = "default"
if active_profile == "default":
stable_parts.append(
post_workspace_parts.append(
"Active Hermes profile: default. Other profiles (if any) live "
"under " + str(get_hermes_home()) + "/profiles/<name>/. Each profile has its own "
"skills/, plugins/, cron/, and memories/ that affect a different "
@ -403,7 +415,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
"you to."
)
else:
stable_parts.append(
post_workspace_parts.append(
f"Active Hermes profile: {active_profile}. This session reads "
f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default "
f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, "
@ -449,11 +461,16 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
if platform_key == "tui" and _effective_hint:
_effective_hint = _tui_embedded_pane_clarifier(_effective_hint)
if _effective_hint:
stable_parts.append(_effective_hint)
post_workspace_parts.append(_effective_hint)
# ── Context tier (cwd-dependent, may change between sessions) ─
context_parts: List[str] = []
if coding_workspace_parts:
context_parts.extend(coding_workspace_parts)
context_parts.extend(coding_trailing_parts)
context_parts.extend(post_workspace_parts)
# Note: ephemeral_system_prompt is NOT included here. It's injected at
# API-call time only so it stays out of the cached/stored system prompt.
if system_message is not None:
@ -515,6 +532,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
timestamp_line += f"\nModel: {agent.model}"
if agent.provider:
timestamp_line += f"\nProvider: {agent.provider}"
if agent.platform:
timestamp_line += f"\nPlatform: {agent.platform}"
volatile_parts.append(timestamp_line)
return {
@ -541,6 +560,7 @@ def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str
"""
parts = build_system_prompt_parts(agent, system_message=system_message)
joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p)
agent._cached_system_prompt_static = parts["stable"]
# Surface context-file truncation warnings through the normal agent status
# channel so gateway/CLI users see them in chat instead of only in logs.
@ -557,6 +577,7 @@ def invalidate_system_prompt(agent: Any) -> None:
so the rebuilt prompt captures any writes from this session.
"""
agent._cached_system_prompt = None
agent._cached_system_prompt_static = None
if agent._memory_store:
agent._memory_store.load_from_disk()

View file

@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool:
return True
def _summarize_user_message(user_message: str) -> str:
"""Collapse a slash-skill-expanded turn back to what the user typed.
A ``/skill`` invocation expands into a message that embeds the whole skill
body, so feeding it to the titler verbatim titles the session after the
*skill's* prose — "Kick off a task in a fresh isolated git worktree" — not
after the user's request. Reuse the canonical scaffolding parser so the
model sees ``/work fix the title leak`` instead.
"""
if not user_message:
return ""
try:
from agent.skill_commands import describe_skill_invocation
described = describe_skill_invocation(user_message)
except Exception:
logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True)
return user_message
return described if described is not None else user_message
def generate_title(
user_message: str,
assistant_response: str,
@ -110,7 +131,7 @@ def generate_title(
logger.debug("Title runtime validator raised; proceeding", exc_info=True)
# Truncate long messages to keep the request small
user_snippet = user_message[:500] if user_message else ""
user_snippet = _summarize_user_message(user_message)[:500]
assistant_snippet = assistant_response[:500] if assistant_response else ""
language = _title_language()
@ -143,6 +164,11 @@ def generate_title(
title = title.strip('"\'')
if title.lower().startswith("title:"):
title = title[6:].strip()
# A title is one line. A model that ignores "return ONLY the title" and
# answers the prompt instead (a shell transcript, a bulleted plan) would
# otherwise be stored verbatim and truncated mid-command. Keep the first
# non-empty line — the closest thing to a title in that response.
title = next((line.strip() for line in title.splitlines() if line.strip()), "")
# Enforce reasonable length
if len(title) > 80:
title = title[:77] + "..."

View file

@ -1204,7 +1204,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}")
agent._current_tool = None
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)")
_status_suffix = " (error)" if is_error else ""
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}")
if not blocked and agent.tool_complete_callback:
try:
@ -1826,7 +1827,8 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
logging.debug(f"Tool progress callback error: {cb_err}")
agent._current_tool = None
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)")
_status_suffix = " (error)" if _is_error_result else ""
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}")
if agent.verbose_logging:
logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s")

View file

@ -162,7 +162,7 @@ def build_trace_jsonl(
if cwd:
r = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
capture_output=True, text=True, timeout=3, cwd=cwd,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3, cwd=cwd,
)
if r.returncode == 0:
git_branch = r.stdout.strip()

View file

@ -7,6 +7,7 @@ streaming, or the _run_codex_stream() call path.
import hashlib
import json
import re
from typing import Any, Dict, List, Optional
from agent.transports.base import ProviderTransport
@ -27,6 +28,49 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
return f"pck_{digest}"
_EXTENDED_PROMPT_CACHE_MODELS = (
"gpt-5.5-pro",
"gpt-5.5",
"gpt-5.4",
"gpt-5.2",
"gpt-5.1-codex-max",
"gpt-5.1-codex-mini",
"gpt-5.1-chat-latest",
"gpt-5.1-codex",
"gpt-5.1",
"gpt-5-codex",
"gpt-5",
"gpt-4.1",
)
_EXTENDED_PROMPT_CACHE_MODEL_RE = re.compile(
rf"(?:^|[./:])(?:{'|'.join(re.escape(name) for name in _EXTENDED_PROMPT_CACHE_MODELS)})"
r"(?:-\d{4}-\d{2}-\d{2})?$"
)
def _default_prompt_cache_retention_for_request(
model: str,
base_url: Any,
) -> Optional[str]:
"""Return ``24h`` for supported models on Amazon Bedrock Mantle."""
from utils import base_url_hostname
hostname_parts = base_url_hostname(str(base_url or "")).split(".")
is_bedrock_mantle = (
len(hostname_parts) == 4
and hostname_parts[0] == "bedrock-mantle"
and bool(hostname_parts[1])
and hostname_parts[2:] == ["api", "aws"]
)
if not is_bedrock_mantle:
return None
normalized = str(model or "").strip().lower().replace("_", "-")
if _EXTENDED_PROMPT_CACHE_MODEL_RE.search(normalized):
return "24h"
return None
def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]:
"""Content-address the prompt cache key from the static request prefix.
@ -284,6 +328,13 @@ class ResponsesApiTransport(ProviderTransport):
if not is_github_responses and not is_xai_responses and cache_key:
kwargs["prompt_cache_key"] = cache_key
cache_retention = _default_prompt_cache_retention_for_request(
model,
params.get("base_url"),
)
if cache_retention:
kwargs.setdefault("prompt_cache_retention", cache_retention)
if reasoning_enabled and is_xai_responses:
from agent.model_metadata import grok_supports_reasoning_effort

View file

@ -394,7 +394,7 @@ def check_codex_binary(
proc = subprocess.run(
[codex_bin, "--version"],
capture_output=True,
text=True,
text=True, encoding='utf-8', errors='replace',
timeout=10,
stdin=subprocess.DEVNULL,
)

View file

@ -34,7 +34,9 @@ from typing import Any, Dict, List, Mapping, Optional
from agent.conversation_compression import (
IDLE_COMPACTION_STATUS_TEMPLATE,
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
compression_skipped_due_to_lock,
conversation_history_after_compression,
recover_rotated_compression_session,
)
from agent.context_engine import automatic_compaction_status_message
from agent.iteration_budget import IterationBudget
@ -352,6 +354,13 @@ def build_turn_context(
# Guard stdio against OSError from broken pipes (systemd/headless/daemon).
install_safe_stdio()
# Recover a session rotated by another path before binding log/turn ids or
# copying client-supplied history. Everything in this turn must consistently
# belong to the canonical child, including observability metadata.
recovered_history = recover_rotated_compression_session(agent)
if recovered_history is not None:
conversation_history = recovered_history
# NOTE: the DB session row is created later, AFTER the system prompt is
# restored/built (see _ensure_db_session() below the system-prompt block).
# Creating it here — before _cached_system_prompt is populated — inserts a
@ -710,6 +719,8 @@ def build_turn_context(
# issue #27405 (a few very large messages slipping past the count gate).
_preflight_compressed = False
_preflight_compression_blocked = False
agent._turn_received_provider_response = False
agent._turn_preflight_display_snapshot = None
if agent.compression_enabled and _should_run_preflight_estimate(
messages,
agent.context_compressor.protect_first_n,
@ -722,6 +733,21 @@ def build_turn_context(
tools=agent.tools or None,
)
_compressor = agent.context_compressor
# getattr guard: minimal compressor doubles (SimpleNamespace in the
# engine-preflight tests) and plugin context engines lack this
# ContextCompressor-only method — absence means no snapshot, and the
# finalizer's rollback stays disarmed for the turn (display-only).
_snapshot_fn = getattr(
_compressor, "snapshot_preflight_display_tokens", None
)
if callable(_snapshot_fn):
_snapshot_val = _snapshot_fn()
# Type pin: MagicMock compressors return truthy Mock objects —
# only a real int snapshot may arm the interrupted-turn rollback.
if isinstance(_snapshot_val, int) and not isinstance(
_snapshot_val, bool
):
agent._turn_preflight_display_snapshot = _snapshot_val
_defer_preflight = getattr(
_compressor,
"should_defer_preflight_to_real_usage",
@ -838,10 +864,29 @@ def build_turn_context(
for _pass in range(_max_preflight_passes):
_orig_len = len(messages)
_orig_tokens = _preflight_tokens
_preflight_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message, approx_tokens=_preflight_tokens,
task_id=effective_task_id,
)
if (
messages is _preflight_input
and compression_skipped_due_to_lock(agent)
):
# #69870 lock-skip: another path holds this session's
# compression lock, so the pass no-oped. That is a
# temporary DEFER, not proof the transcript cannot
# compress — do NOT arm the insufficient-progress
# blocker (the loop's error handlers must keep their
# provider-proven retry budget) and stop preflight
# passes for this turn; the lock winner is shrinking
# the same session concurrently.
logger.info(
"Preflight compression deferred: compression lock "
"held by another path (session %s)",
agent.session_id or "none",
)
break
# Re-estimate now so size-only compression (same row count,
# lower token count — e.g. summarising tool outputs) is
# recognised as progress instead of being misread as

View file

@ -201,6 +201,36 @@ def finalize_turn(
)
)
# Preflight can seed the display count before the provider receives the
# request. Roll that estimate back only when an interrupt wins the race
# before any successful provider response. Compaction state remains owned
# by the real-usage/post-compaction path, including its ``-1`` sentinel.
# Guard rules (test-double density on this path is high):
# - snapshot is type-pinned to a real int — MagicMock agents auto-create
# truthy Mock attributes that must never arm the rollback;
# - the received-response flag is pinned to ``is not True`` — its real
# domain is True/False, and only a literal True means a provider
# response completed;
# - the compressor method gets a getattr+callable guard — SimpleNamespace
# compressor doubles and plugin context engines lack it.
_preflight_snapshot = getattr(
agent, "_turn_preflight_display_snapshot", None
)
if (
interrupted is True
and isinstance(_preflight_snapshot, int)
and not isinstance(_preflight_snapshot, bool)
and getattr(agent, "_turn_received_provider_response", False) is not True
and getattr(agent, "context_compressor", None) is not None
):
_rollback_fn = getattr(
agent.context_compressor,
"rollback_interrupted_preflight_display_tokens",
None,
)
if callable(_rollback_fn):
_rollback_fn(_preflight_snapshot)
# Post-loop cleanup must never lose the response. Trajectory save,
# resource teardown, and session persistence all touch fallible
# surfaces — file I/O / JSON serialization (_save_trajectory), remote
@ -495,6 +525,34 @@ def finalize_turn(
except Exception as exc:
logger.warning("post_llm_call hook failed: %s", exc)
# Context engine observation hook: notify the active engine that this
# turn has finished, with the finalized transcript. Complements the
# per-request select_context() hook (selection before the request;
# observation after the turn). No-op default, fail-open.
try:
from agent.conversation_loop import _notify_context_engine_turn_complete
# Forward the turn's canonical usage when the host has it. The loop
# stashes the most recent API response's usage dict (the same
# canonical buckets fed to ``update_from_response``) on the agent as
# ``_last_turn_usage``. It is ``None`` on turns that never reached a
# provider response (early failure / interrupt), which is exactly the
# contract: real usage when available, ``None`` otherwise.
_turn_usage = getattr(agent, "_last_turn_usage", None)
_notify_context_engine_turn_complete(
agent,
messages,
usage=_turn_usage,
logger=logger,
turn_id=turn_id,
task_id=effective_task_id,
api_call_count=api_call_count,
interrupted=interrupted,
failed=failed,
turn_exit_reason=_turn_exit_reason,
)
except Exception as exc:
logger.warning("on_turn_complete notification failed: %s", exc)
# Extract reasoning from the CURRENT turn only. Walk backwards
# but stop at the user message that started this turn — anything
# earlier is from a prior turn and must not leak into the reasoning
@ -627,4 +685,7 @@ def finalize_turn(
except Exception as exc:
logger.warning("on_session_end hook failed: %s", exc)
agent._turn_preflight_display_snapshot = None
agent._turn_received_provider_response = False
return result

View file

@ -13,10 +13,11 @@ import shlex
import sqlite3
import tempfile
import threading
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Optional
from typing import Any, Iterator, Optional
from hermes_constants import get_hermes_home
@ -65,13 +66,38 @@ def _connect() -> sqlite3.Connection:
path = _db_path()
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path)
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
conn.execute("PRAGMA busy_timeout=5000")
conn.row_factory = sqlite3.Row
_ensure_schema(conn)
try:
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
conn.execute("PRAGMA busy_timeout=5000")
_ensure_schema(conn)
except Exception:
# A PRAGMA/DDL failure after a successful connect() must not leak the
# just-opened connection back to the caller.
conn.close()
raise
return conn
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
"""Open a connection, commit/rollback on exit, and ALWAYS close it.
``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the
transaction; they do not close the connection. Using ``with _connect()``
alone therefore leaks a connection and its WAL/SHM file descriptors on
every call, deferring the close to the garbage collector, which over a
long-running process can exhaust ``RLIMIT_NOFILE`` (the cron-ledger sibling
of this bug was #69567 / PR #69594).
"""
conn = _connect()
try:
with conn:
yield conn
finally:
conn.close()
def _ensure_schema(conn: sqlite3.Connection) -> None:
conn.execute(
"""
@ -454,7 +480,7 @@ def record_terminal_result(
created_at = _utc_now()
with _DB_LOCK:
with _connect() as conn:
with _transaction() as conn:
cur = conn.execute(
"""
INSERT INTO verification_events(
@ -520,7 +546,7 @@ def mark_workspace_edited(
edited_at = _utc_now()
with _DB_LOCK:
with _connect() as conn:
with _transaction() as conn:
row = conn.execute(
"""
SELECT changed_paths_json FROM verification_state
@ -570,7 +596,7 @@ def verification_status(
sid = str(session_id or "default")
root = str(facts.get("root") or Path(cwd or ".").resolve())
with _DB_LOCK:
with _connect() as conn:
with _transaction() as conn:
state = conn.execute(
"""
SELECT last_event_id, last_edit_at, changed_paths_json

View file

@ -30,7 +30,7 @@ Already have the Hermes CLI? Just run:
hermes desktop
```
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. On first launch Hermes walks you through picking a provider and model; nothing else to configure.
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. If Desktop cannot find a usable runtime or saved remote connection, first launch lets you connect to an existing Hermes gateway or install Hermes locally. Local onboarding then walks you through choosing a provider and model.
### Prebuilt installers
@ -134,6 +134,19 @@ Desktop supports a managed local backend, explicit remote gateways, and Hermes
Cloud connections. Remote and cloud modes use the same remote-capability path;
authentication and discovery differ, not the renderer feature model.
When no usable local runtime or saved remote connection exists, the first-run
screen offers **Connect to existing Hermes** before starting the local installer.
Desktop probes the gateway to discover token or OAuth authentication, requires a
successful HTTP and WebSocket connection test, and saves the connection using
the same encrypted Desktop configuration used by Settings. A saved remote
connection bypasses this choice on later launches. The regular Desktop build
still includes the local-install option; this is a remote operating mode, not a
separate client-only application.
In remote mode the gateway host is the execution boundary: agent tools,
terminal commands, and file operations run against the remote Hermes host, not
the computer displaying the Desktop UI.
Projects are the workspace abstraction. A project may own multiple folders,
repositories, worktrees, and sessions; a bare new chat remains detached unless
the user enters a project or configures a default project directory. Use the

View file

@ -21,8 +21,16 @@ const INFERENCE_SWITCH_TRIGGER = 'E2E_INFERENCE_SWITCH_TRIGGER'
const INFERENCE_PROMPT = `${INFERENCE_SWITCH_TRIGGER}: original inference prompt must remain singular.`
const INFERENCE_CORRECTION = `${INFERENCE_SWITCH_TRIGGER}: correction sent while inference is live.`
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
// renderer's keep-alive visibility policy instead of relying on DOM order.
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
function activeSurface(page: Page) {
return page.locator(SURFACE).last()
}
async function send(page: Page, text: string): Promise<void> {
const composer = page.locator('[contenteditable="true"]').first()
const composer = activeSurface(page).locator('[contenteditable="true"]').first()
await composer.waitFor({ state: 'visible', timeout: 15_000 })
await composer.click()
await composer.type(text, { delay: 5 })
@ -30,8 +38,9 @@ async function send(page: Page, text: string): Promise<void> {
}
async function steer(page: Page, text: string): Promise<void> {
const composer = page.locator('[contenteditable="true"]').first()
const primary = page.locator('[data-slot="composer-root"] button[type="submit"]')
const surface = activeSurface(page)
const composer = surface.locator('[contenteditable="true"]').first()
const primary = surface.locator('[data-slot="composer-root"] button[type="submit"]')
await composer.waitFor({ state: 'visible', timeout: 15_000 })
await composer.click()
@ -42,55 +51,80 @@ async function steer(page: Page, text: string): Promise<void> {
async function waitForTranscriptText(page: Page, text: string): Promise<void> {
await page.waitForFunction(
(expected: string) => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
text,
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const active = surfaces[surfaces.length - 1]
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
},
[text, SURFACE] as [string, string],
{ timeout: 30_000 },
)
}
async function textNodeOccurrences(page: Page, text: string): Promise<number> {
return page.evaluate((expected: string) => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return 0
return page.evaluate(
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return 0
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
let count = 0
while (walker.nextNode()) {
if (walker.currentNode.textContent?.includes(expected)) {
count += 1
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
let count = 0
while (walker.nextNode()) {
if (walker.currentNode.textContent?.includes(expected)) {
count += 1
}
}
}
return count
}, text)
return count
},
[text, SURFACE] as [string, string],
)
}
async function transcriptTextOrder(page: Page): Promise<string[]> {
return page.evaluate(() => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
return page.evaluate((surfaceSelector: string) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return []
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="message"], [data-message-id]'))
.map(message => message.textContent?.trim() ?? '')
.filter(Boolean)
})
}, SURFACE)
}
async function transcriptMessageOrder(page: Page): Promise<string[]> {
return page.evaluate(() => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
return page.evaluate((surfaceSelector: string) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return []
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"]'))
return Array.from(
viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"], [data-role="system"]'),
)
.map(message => message.textContent?.trim() ?? '')
.filter(Boolean)
})
}, SURFACE)
}
/**
* The sidebar "+" opens a NEW TAB beside the current chat rather than
* replacing it, so the prior session stays mounted in its own surface. Wait
* for the newly-mounted surface to show an empty transcript instead of waiting
* for the old text to disappear from the page (it never will).
*/
async function openFreshDraft(page: Page, priorSessionText: string): Promise<void> {
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
await page.waitForFunction(
(priorText: string) => !(document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(priorText),
priorSessionText,
([priorText, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const active = surfaces[surfaces.length - 1]
const transcript = active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
return surfaces.length > 0 && !transcript.includes(priorText)
},
[priorSessionText, SURFACE] as [string, string],
{ timeout: 15_000 },
)
}
@ -116,7 +150,12 @@ async function reopenInferenceSession(page: Page): Promise<void> {
}
function relevantOrder(messages: string[]): string[] {
return messages.filter(message => message.includes(ORIGINAL_PROMPT) || message.includes(CORRECTION))
return messages.flatMap(message => {
if (message.includes(ORIGINAL_PROMPT)) return [ORIGINAL_PROMPT]
if (message.includes(CORRECTION)) return [CORRECTION]
return []
})
}
function steerTurnOrder(messages: string[]): string[] {

View file

@ -0,0 +1,197 @@
/**
* Regression coverage for an attached image in a durable session. The gateway
* persists the turn, the builder exits, and desktop renders it from SessionDB
* for the first time the "quit and relaunch" case, where the transcript used
* to come back as vision-enrichment prose instead of a thumbnail.
*
* The fixture pins `image_input_mode: native` because that is the majority
* routing path (any vision-capable model) and the one where a text-only
* persist override is silently dropped. The image also sits behind directory
* and file names containing spaces, mirroring the macOS composer's
* `~/Library/Application Support/...` staging path.
*/
import * as fs from 'node:fs'
import * as path from 'node:path'
import {
buildAppEnv,
createSandbox,
launchDesktop,
type Sandbox,
waitForAppReady,
writeEnvFile,
writeMockProviderConfig,
} from './fixtures'
import { type MockServer, startMockServer } from './mock-server'
import { RealSessionBuilder } from './real-session-builder'
import { type ElectronApplication, expect, type Page, test } from './test'
// A seeded session has no generated title, so every label falls back to the
// session preview — the first 60 characters of the first user message.
const SESSION_TITLE = 'E2E attached image session'
const CAPTION = 'E2E attached image must survive a relaunch'
const IMAGE_DIR = 'Application Support/e2e shots'
const IMAGE_NAME = 'e2e capture.png'
const NATIVE_IMAGE_CONFIG = 'agent:\n image_input_mode: native'
/** A 160x100 framed magenta block — small, but visible in the screenshots. */
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAKAAAABkCAIAAACO1KzYAAAA30lEQVR42u3dwQ2AIBAAQTAWAx1iBXYI7diCuWhEMvP2dZsj+CL30hLr2oxAYARGYARGYARGYIERGIH53n7nozpOk5rQqIcNdkQjMAIjMNPeomP3N54V+5exwY5oBEZgBEZgBEZggREYgREYgREYgQVGYARGYARGYARGYIERGIERGIERGIEFRmAERmAERmAEFhiBERiBERiBERiBBUZgBEZgBEZgBBYYgREYgREYgRFYYCMQGIERmPSj94Njb9ligxEYgRFYYJaQe2mmYIMRGIERGIERGIEFRmAERmDedAFtjAtAGWDnoAAAAABJRU5ErkJggg=='
interface SeededFixture {
app: ElectronApplication
mock: MockServer
page: Page
sandbox: Sandbox
cleanup: () => Promise<void>
}
function writeImage(sandbox: Sandbox): string {
const dir = path.join(sandbox.root, IMAGE_DIR)
fs.mkdirSync(dir, { recursive: true })
const imagePath = path.join(dir, IMAGE_NAME)
fs.writeFileSync(imagePath, Buffer.from(PNG_BASE64, 'base64'))
return imagePath
}
async function setupSeededDesktop(): Promise<SeededFixture> {
const mock = await startMockServer()
const sandbox = createSandbox('image-attachment')
writeMockProviderConfig(sandbox.hermesHome, mock.url, undefined, NATIVE_IMAGE_CONFIG)
writeEnvFile(sandbox.hermesHome)
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
try {
await builder.createSession({
title: SESSION_TITLE,
turns: [{ images: [writeImage(sandbox)], text: CAPTION }],
})
} finally {
await builder.close()
}
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
return {
app,
mock,
page,
sandbox,
cleanup: async () => {
await app.close().catch(() => undefined)
await mock.close()
sandbox.cleanup()
},
}
}
function sessionRow(page: Page) {
return page.locator('[data-slot="sidebar"] button').filter({ hasText: CAPTION }).first()
}
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
// renderer's keep-alive visibility policy instead of relying on DOM order.
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
function activeViewportText(surfaceSelector: string): string {
const surfaces = document.querySelectorAll(surfaceSelector)
return surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
}
async function openSeededSession(page: Page): Promise<void> {
const row = sessionRow(page)
await row.waitFor({ state: 'visible', timeout: 60_000 })
await row.click()
await page.waitForFunction(
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
return text.includes(expected)
},
[CAPTION, SURFACE] as [string, string],
{ timeout: 30_000 },
)
}
/**
* The sidebar "+" opens a NEW TAB beside the current chat instead of replacing
* it, so the seeded session stays mounted in its own surface. Assert the new
* surface is empty rather than waiting for the old caption to leave the page.
*/
async function openNewSession(page: Page): Promise<void> {
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
await page.waitForFunction(
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
return surfaces.length > 0 && !text.includes(expected)
},
[CAPTION, SURFACE] as [string, string],
{ timeout: 15_000 },
)
}
async function transcriptText(page: Page): Promise<string> {
return page.evaluate(activeViewportText, SURFACE)
}
async function assertRendersThumbnail(page: Page, label: string): Promise<void> {
const thumbnail = page.locator('[data-slot="aui_directive-image"] img')
await expect(thumbnail, `${label}: the attachment should render as an image`).toHaveCount(1)
await expect(thumbnail, `${label}: the thumbnail should resolve off disk`).toHaveAttribute('src', /^data:image\//)
const text = await transcriptText(page)
expect(text, `${label}: the caption should survive alongside the image`).toContain(CAPTION)
// A broken ref falls back to a chip whose label leaks the path, and a
// flattened multimodal turn leaves the agent's placeholder behind.
expect(text, `${label}: the raw image path should not leak into the transcript`).not.toContain(IMAGE_NAME)
expect(text, `${label}: the image directive should not render literally`).not.toContain('@image:')
expect(text, `${label}: the flattening placeholder should not render`).not.toContain('[screenshot]')
}
test.describe('attached image resume', () => {
let fixture: SeededFixture | null = null
test.afterEach(async () => {
await fixture?.cleanup()
fixture = null
})
test('renders a persisted attachment as a thumbnail on first open and after a cold reload', async ({}, testInfo) => {
// Seeding through the real gateway plus two full app boots does not fit the
// default per-test budget on a cold runner.
test.slow()
fixture = await setupSeededDesktop()
await waitForAppReady(fixture, 120_000)
// The sidebar labels a session by its preview, so the caption has to lead
// the persisted turn — a leading directive reads as a truncated file path.
const row = sessionRow(fixture.page)
await row.waitFor({ state: 'visible', timeout: 60_000 })
const label = (await row.textContent())?.trim() ?? ''
expect(label.startsWith(CAPTION), `sidebar label should open with the caption: ${label}`).toBe(true)
await openSeededSession(fixture.page)
await assertRendersThumbnail(fixture.page, 'first open')
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-first-open.png') })
// A reload drops every cached attachment ref, so the transcript has to come
// back from the persisted turn alone.
await fixture.page.reload()
await waitForAppReady(fixture, 120_000)
await openNewSession(fixture.page)
await openSeededSession(fixture.page)
await assertRendersThumbnail(fixture.page, 'cold reload')
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-cold-reload.png') })
})
})

View file

@ -14,8 +14,11 @@
* prove the full boot gateway inference renderer chain works.
*/
import fs from 'node:fs'
import http from 'node:http'
import type { ServerResponse } from 'node:http'
import os from 'node:os'
import nodePath from 'node:path'
/** A canned assistant reply used for every chat completion request. */
export const MOCK_REPLY = 'Hello from the mock inference server! The full boot chain is working.'
@ -27,6 +30,14 @@ export interface MockServerOptions {
holdFirstCompletionContaining?: string
/** Absolute sandbox path written by the verify-on-stop scripted tool call. */
verificationWritePath?: string
/**
* Sentinel path that ends the E2E_SIDEBAR_CROSS background process.
*
* Without it that process is a bare `sleep 5`, which races the agent turn and
* the 4s auto-dismiss linger see `createBackgroundReleaseHandle`. Pass a
* handle's `path` to let the test decide when the process exits.
*/
backgroundReleasePath?: string
}
export interface MockServer {
@ -167,37 +178,68 @@ const SIDEBAR_SCRIPT: ScriptedTurn[] = [
// ─── Sidebar cross-session script ──────────────────────────────────────
//
// E2E_SIDEBAR_CROSS trigger uses a longer background process (sleep 5) so
// the "background running" dot is visible long enough for the test to:
// E2E_SIDEBAR_CROSS starts a long background process plus a subagent so the
// tests can:
// 1. See the background dot while the subagent runs.
// 2. Open a different session and see session A's dot transition to
// "finished unread" when the background process completes.
//
// The background process must outlive the agent turn — the whole point is a
// dot that is still "running" after the final answer lands. A fixed `sleep`
// cannot guarantee that: on a loaded CI runner the turn (two model round
// trips + a real subagent delegation) can take longer than the sleep, the
// process exits early, the 4s success linger elapses, and the dot is gone
// before the test looks. That is a wall-clock race between three independent
// timers, and it made this the flakiest spec in the suite.
//
// When `backgroundReleasePath` is set the process instead blocks until the
// test creates that sentinel file, so the test — not the clock — decides when
// the dot clears. `sleep 5` remains the fallback for callers that don't pass
// a handle.
function sidebarCrossBgCommand(releasePath?: string): string {
if (!releasePath) {
return 'echo "long bg output" && sleep 5 && echo "finished"'
}
// Bounded wait (60s): if a test forgets to release (or crashes mid-way),
// the process still exits instead of hanging the worker until the suite
// times out.
const quoted = JSON.stringify(releasePath)
return [
'echo "long bg output"',
`for _ in $(seq 1 600); do [ -e ${quoted} ] && break; sleep 0.1; done`,
'echo "finished"',
].join(' && ')
}
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = [
{
text: 'Starting a long background task and delegating work.',
toolCalls: [
{
name: 'terminal',
args: {
command: 'echo "long bg output" && sleep 5 && echo "finished"',
background: true,
notify_on_complete: true,
function sidebarCrossScript(releasePath?: string): ScriptedTurn[] {
return [
{
text: 'Starting a long background task and delegating work.',
toolCalls: [
{
name: 'terminal',
args: {
command: sidebarCrossBgCommand(releasePath),
background: true,
notify_on_complete: true,
},
},
},
{
name: 'delegate_task',
args: {
goal: 'Analyze cross-session state',
context: 'Testing that the background dot updates across sessions.',
{
name: 'delegate_task',
args: {
goal: 'Analyze cross-session state',
context: 'Testing that the background dot updates across sessions.',
},
},
},
],
},
{
text: 'Both tasks are running in the background now.',
},
]
],
},
{
text: 'Both tasks are running in the background now.',
},
]
}
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = sidebarCrossScript()
const QUEUE_STOP_SCRIPT: ScriptedTurn[] = [
{
@ -423,7 +465,8 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
}
if (isSidebarCrossTrigger) {
const turn = SIDEBAR_CROSS_SCRIPT[_sidebarCrossIndex] ?? SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1]
const script = sidebarCrossScript(options.backgroundReleasePath)
const turn = script[_sidebarCrossIndex] ?? script[script.length - 1]
_sidebarCrossIndex++
if (stream) {
@ -722,6 +765,65 @@ export function restartMockServer(): void {
resetScriptIndex()
}
/** Test-controlled lifetime for the E2E_SIDEBAR_CROSS background process. */
export interface BackgroundReleaseHandle {
/** Sentinel path — pass as `backgroundReleasePath` to `startMockServer`. */
path: string
/** End the background process now (creates the sentinel). */
release: () => void
/** Remove the sentinel if it still exists. Safe to call twice. */
cleanup: () => void
}
/**
* Create a sentinel that keeps the E2E_SIDEBAR_CROSS background process alive
* until the test explicitly releases it.
*
* The cross-session sidebar tests need a background process that is still
* RUNNING after the agent turn finishes that is the state under test (a
* session whose turn is done but whose background work is not). With a fixed
* `sleep`, three independent clocks race: the sleep, the agent turn (two model
* round trips plus a real subagent delegation), and the 4s success linger
* before a finished task auto-dismisses. When a loaded CI runner makes the
* turn slower than the sleep, the process is already gone and the assertion
* samples an empty sidebar. Observed on CI 2026-07-26 across two unrelated
* PRs: the "should appear" poll needed 7.5s to see the dot, by which point
* `sleep 5` had exited.
*
* With a sentinel there is one clock and the test owns it:
*
* ```ts
* const release = createBackgroundReleaseHandle()
* const mock = await startMockServer({ backgroundReleasePath: release.path })
* // ... assert the dot is visible; it cannot vanish on its own ...
* release.release() // now, and only now, the process exits
* ```
*/
export function createBackgroundReleaseHandle(): BackgroundReleaseHandle {
const path = nodePath.join(
os.tmpdir(),
`hermes-e2e-bg-release-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
)
return {
path,
release: () => {
try {
fs.writeFileSync(path, 'release')
} catch {
// The process also has a bounded fallback wait; a failed write must
// not crash the test before its real assertions run.
}
},
cleanup: () => {
try {
fs.rmSync(path, { force: true })
} catch {
// Best-effort — the sentinel lives in the OS temp dir.
}
},
}
}
/**
* The interim script's text constants, exported for test assertions.
* Each entry is the visible text of one turn. Turns with empty text
@ -756,8 +858,12 @@ export const SIDEBAR_CROSS_TEXTS = {
interimText: SIDEBAR_CROSS_SCRIPT[0].text,
/** The final answer text. */
finalText: SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1].text,
/** The longer background process command (sleep 5). */
bgCommand: 'echo "long bg output" && sleep 5 && echo "finished"',
/**
* The default (unheld) background process command. Tests that pass a
* `backgroundReleasePath` get a sentinel-waiting command instead see
* `createBackgroundReleaseHandle`.
*/
bgCommand: sidebarCrossBgCommand(),
/** The subagent's goal. */
subagentGoal: 'Analyze cross-session state',
} as const

View file

@ -28,11 +28,18 @@ interface CreatedSession {
stored_session_id: string
}
export interface RealSessionTurn {
/** Local image paths attached before the prompt, as the composer would. */
images?: readonly string[]
text: string
}
export interface RealSessionSpec {
/** Human-visible sidebar title, persisted by the first completed turn. */
/** Session label. The durable row stores no title, so clients fall back to
* the preview (the first 60 characters of the first user message). */
title: string
/** Each item becomes one real user prompt followed by the mock provider's reply. */
turns: readonly string[]
turns: readonly (RealSessionTurn | string)[]
}
export interface RealSession {
@ -107,7 +114,13 @@ export class RealSessionBuilder {
const runtimeId = requireString(created, 'session_id')
const sessionId = requireString(created, 'stored_session_id')
for (const text of spec.turns) {
for (const turn of spec.turns) {
const { images = [], text } = typeof turn === 'string' ? { text: turn } : turn
for (const image of images) {
await this.request('image.attach', { session_id: runtimeId, path: image })
}
const completion = this.waitForEvent(
frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId,
)

View file

@ -122,11 +122,16 @@ auxiliary:
// A normal message crosses the tiny configured context budget. The mock
// blocks only the resulting summary request, so these assertions run
// during automatic compaction rather than a slash-command path.
// The payload must cross threshold_tokens (22k) on its OWN weight
// (~12k tokens) on top of the system prompt. Do not shrink it: at
// repeat(500) the trigger only worked because the ambient system prompt
// (skills index + tool schemas) happened to carry it over the line, and
// a 160-token skills-index cleanup on main broke the test for a day.
await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_ONE '.repeat(5))
await waitForTranscript(page, MOCK_REPLY)
await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_TWO '.repeat(5))
await waitForTranscript(page, MOCK_REPLY)
await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(500))
await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(1500))
await fixture.mock.waitForHeldCompletion()
await expect(page.getByRole('status', { name: 'Summarizing thread' }).last()).toBeVisible()

View file

@ -16,7 +16,12 @@ import {
setupMockBackend,
waitForAppReady,
} from './fixtures'
import { SIDEBAR_CROSS_TEXTS, SIDEBAR_TEXTS, restartMockServer } from './mock-server'
import {
createBackgroundReleaseHandle,
restartMockServer,
SIDEBAR_CROSS_TEXTS,
SIDEBAR_TEXTS,
} from './mock-server'
/** Background-running dot aria-label (from i18n en.ts). */
const BG_DOT_LABEL = 'Background task running'
@ -176,21 +181,30 @@ test.describe('sidebar states — cross-session dot transition', () => {
test.describe.configure({ mode: 'serial' })
let fixture: MockBackendFixture
// Keeps the background process alive until this test releases it, so the
// "still running after the turn finished" state can't expire on its own.
const bgRelease = createBackgroundReleaseHandle()
test.beforeAll(async () => {
restartMockServer()
fixture = await setupMockBackend()
fixture = await setupMockBackend({
mockServer: { backgroundReleasePath: bgRelease.path },
})
await waitForAppReady(fixture, 120_000)
})
test.afterAll(async () => {
// Release first so the process exits even if the test failed early,
// then drop the sentinel file.
bgRelease.release()
await fixture?.cleanup()
bgRelease.cleanup()
})
test('background dot transitions to finished when viewing another session', async () => {
const page = fixture.page
// Start a turn with a long background process (sleep 5).
// Start a turn whose background process runs until we release it.
const composer = page.locator('[contenteditable="true"]').first()
await composer.waitFor({ state: 'visible', timeout: 10_000 })
await composer.click()
@ -212,8 +226,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
{ timeout: 90_000 },
)
// The background dot should still be visible (sleep 5 hasn't finished yet,
// or auto-dismiss hasn't fired).
// The background dot must still be visible: the turn is done but the
// process is held open by the sentinel, so this is a stable state rather
// than a window we have to catch in time.
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
@ -225,8 +240,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
await page.locator('button:has-text("New session")').first().click()
await page.waitForTimeout(2000)
// Now wait for the background process to finish (sleep 5 + auto-dismiss).
// The session A dot should transition away from "background running".
// Now let the background process finish. The session A dot should
// transition away from "background running".
bgRelease.release()
await expect
.poll(
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),

View file

@ -22,7 +22,12 @@ import {
setupMockBackend,
waitForAppReady,
} from './fixtures'
import { SIDEBAR_CROSS_TEXTS, restartMockServer } from './mock-server'
import {
type BackgroundReleaseHandle,
createBackgroundReleaseHandle,
restartMockServer,
SIDEBAR_CROSS_TEXTS,
} from './mock-server'
/** Finished-unread dot aria-label. */
const UNREAD_DOT_LABEL = 'Finished — unread'
@ -34,7 +39,7 @@ function sessionRow(page: import('@playwright/test').Page, text: string) {
return page.locator('[data-slot="sidebar"] button').filter({ hasText: text }).first()
}
/** Common setup: start a turn with a sleep 5 bg process + subagent, wait for
/** Common setup: start a turn with a held bg process + subagent, wait for
* the turn to complete, then switch to a new session so the first session is
* no longer $selectedStoredSessionId (required before opening a tile). */
async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
@ -67,7 +72,9 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
{ timeout: 90_000 },
)
// The background dot should still be visible (sleep 5 hasn't finished).
// The background dot must still be visible: the turn is done but the
// process is held open by the sentinel, so this is a stable state rather
// than a window we have to catch in time.
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
@ -77,8 +84,12 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
await page.waitForTimeout(2000)
}
/** Wait for the background process to finish (sleep 5 + auto-dismiss). */
async function waitForBgProcessToFinish(page: import('@playwright/test').Page) {
/** Release the held background process, then wait for its dot to clear. */
async function waitForBgProcessToFinish(
page: import('@playwright/test').Page,
release?: BackgroundReleaseHandle,
) {
release?.release()
await expect
.poll(
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),
@ -95,15 +106,20 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
test.describe.configure({ mode: 'serial' })
let fixture: MockBackendFixture
const bgRelease = createBackgroundReleaseHandle()
test.beforeAll(async () => {
restartMockServer()
fixture = await setupMockBackend()
fixture = await setupMockBackend({
mockServer: { backgroundReleasePath: bgRelease.path },
})
await waitForAppReady(fixture, 120_000)
})
test.afterAll(async () => {
bgRelease.release()
await fixture?.cleanup()
bgRelease.cleanup()
})
test('session opened as a tab (not visible) correctly gets unread dot', async () => {
@ -123,12 +139,21 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
// Evidence: the tab is open but the session is not visible on screen.
await page.screenshot({ path: 'test-results/tile-bug-tab-opened.png' })
await waitForBgProcessToFinish(page)
await waitForBgProcessToFinish(page, bgRelease)
// A tab that's not the active tab IS hidden — the unread dot is correct.
// The user is NOT looking at it, so marking it "unread" is right.
const unreadCount = await page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count()
expect(unreadCount, 'hidden tab should be marked unread').toBeGreaterThan(0)
//
// Poll rather than sampling once: "finished-unread" is an event-driven
// transition that lands slightly after the running dot clears, and with a
// released (rather than slowly-expiring) process there is no incidental
// slack between the two. Same reasoning as the cross-session spec.
await expect
.poll(
() => page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count(),
{ timeout: 30_000, message: 'hidden tab should be marked unread' },
)
.toBeGreaterThan(0)
await page.screenshot({ path: 'test-results/tile-bug-tab-unread-correct.png' })
})
@ -142,15 +167,20 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
test.describe.configure({ mode: 'serial' })
let fixture: MockBackendFixture
const bgRelease = createBackgroundReleaseHandle()
test.beforeAll(async () => {
restartMockServer()
fixture = await setupMockBackend()
fixture = await setupMockBackend({
mockServer: { backgroundReleasePath: bgRelease.path },
})
await waitForAppReady(fixture, 120_000)
})
test.afterAll(async () => {
bgRelease.release()
await fixture?.cleanup()
bgRelease.cleanup()
})
test('session visible in a split tile does NOT get unread dot when it finishes', async () => {
@ -196,7 +226,7 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
// Evidence: the split tile is now open side-by-side — both sessions visible.
await page.screenshot({ path: 'test-results/tile-bug-split-opened.png' })
await waitForBgProcessToFinish(page)
await waitForBgProcessToFinish(page, bgRelease)
// THE BUG: the session visible in the split tile should NOT have the green
// "finished unread" dot — the user is looking right at it. This assertion

View file

@ -24,6 +24,8 @@
* MutationObserver burst), but `$messages` was still set twice.
*
* The test passes when bursts === 1 AND reconciles === 0.
* The sidebar "+" keeps the session warm in another tab. Its reactivation
* follows the same contract: one additive paint and zero reconciles.
*
* Prerequisite: `npm run build` must have been run so dist/ exists.
*/
@ -43,6 +45,11 @@ import { startMockServer } from './mock-server'
import { RealSessionBuilder } from './real-session-builder'
const SESSION_TITLE = 'E2E Warm Resume Jitter Test'
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
// renderer's keep-alive visibility policy instead of relying on DOM order.
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
const ALL_SURFACES = '[data-composer-target]'
/** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */
const MESSAGE_COUNT = 32
/** Seeded PRNG so the generated content is deterministic across runs. */
@ -154,15 +161,29 @@ test.afterAll(async () => {
* after the initial paint, catching key-based reconciles that don't
* add/remove nodes.
*/
async function installRenderCounter(page: import('@playwright/test').Page): Promise<void> {
await page.evaluate(() => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
async function installRenderCounter(
page: import('@playwright/test').Page,
transcriptText?: string,
): Promise<void> {
await page.evaluate(([visibleSelector, allSelector, expected]: [string, string, string | undefined]) => {
const surfaces = [...document.querySelectorAll(expected ? allSelector : visibleSelector)]
const surface = expected
? surfaces.find(candidate =>
(candidate.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
)
: surfaces.at(-1)
const viewport = surface?.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) {
throw new Error('Thread viewport not found before warm resume')
}
const state = { bursts: 0, mutations: 0, timeline: [] as number[], stopped: false, reconciles: 0 }
;(window as unknown as { __RENDER_COUNT__: typeof state }).__RENDER_COUNT__ = state
const debugWindow = window as unknown as {
__RENDER_COUNT__: typeof state
__RENDER_VIEWPORT__: Element
}
debugWindow.__RENDER_COUNT__ = state
debugWindow.__RENDER_VIEWPORT__ = viewport
let currentBatch = 0
let flushTimer: ReturnType<typeof setTimeout> | null = null
@ -224,7 +245,53 @@ async function installRenderCounter(page: import('@playwright/test').Page): Prom
hasMessages = true
}
}, 2)
})
}, [SURFACE, ALL_SURFACES, transcriptText] as [string, string, string | undefined])
}
/** Wait until the ACTIVE chat surface's transcript contains `text`. */
async function waitForActiveTranscriptText(
page: import('@playwright/test').Page,
text: string,
timeout = 30_000,
): Promise<void> {
await page.waitForFunction(
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const active = surfaces[surfaces.length - 1]
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
},
[text, SURFACE] as [string, string],
{ timeout },
)
}
async function waitForActiveTranscriptWithoutText(
page: import('@playwright/test').Page,
text: string,
): Promise<void> {
await page.waitForFunction(
([expected, surfaceSelector]: [string, string]) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const active = surfaces[surfaces.length - 1]
return !(active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
},
[text, SURFACE] as [string, string],
{ timeout: 15_000 },
)
}
/** Replace the primary surface with a draft while retaining its warm cache. */
async function openFreshDraft(page: import('@playwright/test').Page, priorText: string): Promise<void> {
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+N' : 'Control+N')
await waitForActiveTranscriptWithoutText(page, priorText)
}
/** Stack an empty tab while leaving the current transcript mounted and warm. */
async function openNewSessionTab(page: import('@playwright/test').Page, priorText: string): Promise<void> {
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
await waitForActiveTranscriptWithoutText(page, priorText)
}
/** Stop the render counter and return the recorded burst/reconcile counts. */
@ -245,6 +312,30 @@ async function readRenderCount(page: import('@playwright/test').Page): Promise<{
})
}
async function observedViewportIsActive(page: import('@playwright/test').Page): Promise<boolean> {
return page.evaluate((surfaceSelector: string) => {
const surfaces = document.querySelectorAll(surfaceSelector)
const activeViewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
const observedViewport = (window as unknown as { __RENDER_VIEWPORT__?: Element }).__RENDER_VIEWPORT__
return activeViewport === observedViewport
}, SURFACE)
}
/** A kept-alive tab must become visible without rebuilding its transcript. */
function assertNoRepaint(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
expect(
result!.bursts,
`Expected no additive render bursts for a kept-alive tab, but got ${result!.bursts}. ` +
`Mutation timeline: ${JSON.stringify(result!.timeline)}.`,
).toBe(0)
expect(
result!.reconciles,
`Expected no transcript reconciles for a kept-alive tab, but got ${result!.reconciles}.`,
).toBe(0)
}
/** Assert the render counter shows exactly one paint with no re-renders. */
function assertNoJitter(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
@ -261,7 +352,7 @@ function assertNoJitter(result: { bursts: number; mutations: number; timeline: n
).toBe(0)
}
test('warm-route resume paints transcript exactly once (no jitter)', async ({}, testInfo) => {
test('tab reactivation preserves the mounted transcript without repainting', async ({}, testInfo) => {
const page = fixture!.page
// Wait for the sidebar to populate with our seeded session.
@ -277,63 +368,29 @@ test('warm-route resume paints transcript exactly once (no jitter)', async ({},
// Wait for the transcript to appear — the first user message text confirms
// the cold-path prefetch painted.
await page.waitForFunction(
(text: string) =>
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
false,
FIRST_USER_MSG,
{ timeout: 30_000 },
)
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
// Wait for the session to fully settle (cold-path RPC + reconciliation).
await page.waitForTimeout(2_000)
// Step 2: Navigate away to a new chat — this does NOT evict the warm cache.
const newSessionButton = page
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
.first()
await newSessionButton.click()
// Wait for the new-chat empty state.
await page.waitForFunction(
(firstMsg: string) => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return false
const text = viewport.textContent ?? ''
return !text.includes(firstMsg)
},
FIRST_USER_MSG,
{ timeout: 15_000 },
)
// Stack a new tab, then observe the seeded transcript while it is hidden.
// Installing after the switch isolates reactivation from mutations caused
// while the new tab was being created.
await openNewSessionTab(page, FIRST_USER_MSG)
await page.waitForTimeout(500)
await installRenderCounter(page, FIRST_USER_MSG)
// Step 3: Install render counter, click back (warm resume), wait, assert.
await installRenderCounter(page)
// Step 3: Click back and verify the same kept-alive viewport becomes active
// without rebuilding or reconciling its transcript.
await sessionRow.click()
await page.waitForFunction(
(text: string) =>
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
false,
FIRST_USER_MSG,
{ timeout: 30_000 },
)
// Wait for at least 1 burst, then settle.
await page.waitForFunction(
() => {
const w = window as unknown as { __RENDER_COUNT__?: { bursts: number } }
return Boolean(w.__RENDER_COUNT__ && w.__RENDER_COUNT__.bursts > 0)
},
undefined,
{ timeout: 10_000 },
)
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
await page.waitForTimeout(2_000)
expect(await observedViewportIsActive(page), 'Reactivation should reveal the observed kept-alive viewport').toBe(true)
const result = await readRenderCount(page)
await page.screenshot({ path: testInfo.outputPath('warm-resume-idle.png') })
assertNoJitter(result)
assertNoRepaint(result)
})
test('warm-route resume after background inference completes (no jitter)', async ({}, testInfo) => {
@ -354,13 +411,7 @@ test('warm-route resume after background inference completes (no jitter)', async
// Step 1: Cold resume — populate the warm cache.
await sessionRow.click()
await page.waitForFunction(
(text: string) =>
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
false,
FIRST_USER_MSG,
{ timeout: 30_000 },
)
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
await page.waitForTimeout(2_000)
// Step 2: Send a message — triggers inference via the mock server.
@ -373,34 +424,15 @@ test('warm-route resume after background inference completes (no jitter)', async
// Wait for the mock response to appear in the transcript, confirming
// the turn completed and message.complete fired (which updates the warm
// cache via updateSessionState).
await page.waitForFunction(
() => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
return viewport?.textContent?.includes('mock inference server') ?? false
},
undefined,
{ timeout: 60_000 },
)
await waitForActiveTranscriptText(page, 'mock inference server', 60_000)
// Extra settle for message.complete → updateSessionState → cache write.
await page.waitForTimeout(2_000)
// Verify the prompt was received by the mock server.
expect(mock.receivedPrompts).toContain(PROMPT)
// Step 3: Navigate away — the warm cache retains the updated messages.
const newSessionButton = page
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
.first()
await newSessionButton.click()
await page.waitForFunction(
(prompt: string) => {
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
if (!viewport) return false
return !(viewport.textContent ?? '').includes(prompt)
},
PROMPT,
{ timeout: 15_000 },
)
// Step 3: Replace the primary chat; the warm cache retains the updated messages.
await openFreshDraft(page, PROMPT)
await page.waitForTimeout(500)
// Step 4: Install render counter, click back (warm resume), wait, assert.
@ -409,13 +441,7 @@ test('warm-route resume after background inference completes (no jitter)', async
// Wait for the transcript to reappear — the warm cache should already
// have the completed turn (updated by message.complete events).
await page.waitForFunction(
(text: string) =>
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
false,
FIRST_USER_MSG,
{ timeout: 30_000 },
)
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
// Wait for at least 1 burst, then settle.
await page.waitForFunction(

View file

@ -68,6 +68,26 @@ test('buildDesktopBackendEnv extends PYTHONPATH and backend PATH together', () =
assert.ok(env.PATH.includes('/opt/homebrew/bin'))
})
test('buildDesktopBackendEnv forces PYTHONUTF8 unless the user set it explicitly', () => {
const defaulted = buildDesktopBackendEnv({
hermesHome: '/Users/test/.hermes',
currentEnv: { PATH: '/usr/bin' },
platform: 'darwin',
pathModule: path.posix
})
assert.equal(defaulted.PYTHONUTF8, '1')
const optedOut = buildDesktopBackendEnv({
hermesHome: '/Users/test/.hermes',
currentEnv: { PATH: '/usr/bin', PYTHONUTF8: '0' },
platform: 'darwin',
pathModule: path.posix
})
assert.equal(optedOut.PYTHONUTF8, '0')
})
test('normalizeHermesHomeRoot maps profile homes back to the global Hermes root', () => {
assert.equal(
normalizeHermesHomeRoot('/Users/test/.hermes/profiles/oracle', { pathModule: path.posix }),

View file

@ -104,6 +104,13 @@ function buildDesktopBackendEnv({
return {
PYTHONPATH: appendUniquePathEntries([...pythonPathEntries, currentPythonPath], { delimiter }),
// Force PEP 540 UTF-8 mode in the spawned Python backend so its stdio and
// subprocess defaults are UTF-8 even on non-UTF-8 Windows locales (GBK,
// cp1252, ...). hermes_bootstrap sets this inside the child too, but only
// after import — anything emitted earlier (interpreter startup errors,
// pre-bootstrap tracebacks) still decodes with the locale default without
// this. User's explicit setting wins. Re-port of PR #56499 (echoriver89).
PYTHONUTF8: currentEnv?.PYTHONUTF8 ?? '1',
[key]: buildDesktopBackendPath({
hermesHome,
venvRoot,

View file

@ -0,0 +1,340 @@
import assert from 'node:assert/strict'
import { test } from 'vitest'
import {
DEFAULT_HEALTH_PROBE_TIMEOUT_MS,
isAuthRejectionError,
isGatedMissingHealthError,
isMissingHealthEndpointError,
isReauthRequiredError,
waitForHermesReady
} from './backend-health'
const GATE_401 = '401: {"error":"unauthenticated","detail":"Unauthorized","reason":"no_cookie","login_url":"/login"}'
test('uses lightweight /api/health for current backends', async () => {
const calls: string[][] = []
await waitForHermesReady('http://127.0.0.1:9000/', {
token: 'secret-token',
fetchPublicJson: async url => {
calls.push(['public', url])
return { ok: true }
},
fetchJson: async url => {
calls.push(['token', url])
throw new Error('status should not be called')
},
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
})
assert.deepEqual(calls, [['public', 'http://127.0.0.1:9000/api/health']])
})
test('falls back to /api/status only for old backends without /api/health', async () => {
const calls: string[][] = []
await waitForHermesReady('http://127.0.0.1:9000', {
token: 'secret-token',
fetchPublicJson: async url => {
calls.push(['public', url])
throw new Error('404: {"detail":"Not Found"}')
},
fetchJson: async (url, token) => {
calls.push(['token', url, token ?? ''])
return { version: 'old' }
},
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
})
assert.deepEqual(calls, [
['public', 'http://127.0.0.1:9000/api/health'],
['token', 'http://127.0.0.1:9000/api/status', 'secret-token']
])
})
test('does not fall back to heavyweight /api/status for transient health failures', async () => {
const calls: string[][] = []
let currentTime = 0
await assert.rejects(
waitForHermesReady('http://127.0.0.1:9000', {
fetchPublicJson: async url => {
calls.push(['public', url])
throw new Error('Timed out connecting to Hermes backend after 15000ms')
},
fetchJson: async url => {
calls.push(['token', url])
},
sleep: async () => {},
now: () => {
currentTime += 20
return currentTime
},
timeoutMs: 50,
pollMs: 1
}),
/Timed out connecting/
)
assert.ok(calls.length > 0)
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
})
test('probes health on a short timeout but leaves the legacy fallback its own', async () => {
const timeouts: (number | undefined)[] = []
await waitForHermesReady('http://127.0.0.1:9000', {
fetchPublicJson: async (_url, options) => {
timeouts.push(options?.timeoutMs)
throw new Error('404: {"detail":"Not Found"}')
},
fetchJson: async (_url, _token, options) => {
timeouts.push(options?.timeoutMs)
return { version: 'old' }
},
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
})
assert.deepEqual(timeouts, [DEFAULT_HEALTH_PROBE_TIMEOUT_MS, undefined])
})
test('aborts as superseded when the bootstrap signal fires', async () => {
const controller = new AbortController()
controller.abort()
await assert.rejects(
waitForHermesReady('http://127.0.0.1:9000', {
signal: controller.signal,
fetchPublicJson: async () => {
throw new Error('should not probe after abort')
},
fetchJson: async () => {
throw new Error('should not probe after abort')
},
timeoutMs: 100,
pollMs: 1
}),
(error: any) => error.kind === 'superseded'
)
})
test('recognizes missing-route shapes only', () => {
assert.equal(isMissingHealthEndpointError(new Error('404: {"detail":"Not Found"}')), true)
assert.equal(
isMissingHealthEndpointError(
new Error('Expected JSON from /api/health but got HTML. The endpoint is likely missing on the Hermes backend.')
),
true
)
assert.equal(isMissingHealthEndpointError(new Error('Timed out connecting to Hermes backend after 15000ms')), false)
assert.equal(isMissingHealthEndpointError(new Error('500: boom')), false)
})
// --- Gated backends that predate /api/health (release 0.19.0 and earlier) ---
//
// The dashboard auth gate runs ahead of the SPA catch-all, so on a backend
// without the route an ANONYMOUS probe is rejected as unauthenticated rather
// than 404 — verified against a simulated 0.19.0 backend:
// credential-free: /api/health -> 401 no_cookie, /api/status -> 200
// credentialed: /api/health -> 404, /api/sessions -> 200
test('anonymous gate-shaped 401 falls back to /api/status (backend predates /api/health)', async () => {
const calls: string[][] = []
await waitForHermesReady('http://192.168.1.132:9119', {
token: null,
fetchPublicJson: async url => {
calls.push(['public', url])
throw new Error(GATE_401)
},
fetchJson: async (url, token) => {
calls.push(['token', url, token == null ? 'null' : token])
return { version: '0.19.0', auth_required: true }
},
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
})
assert.deepEqual(calls, [
['public', 'http://192.168.1.132:9119/api/health'],
['token', 'http://192.168.1.132:9119/api/status', 'null']
])
})
test('a credentialed 401 fails fast for reauth instead of reporting a dead session ready', async () => {
// The regression a blanket 401->fallback introduces: /api/status is public,
// so an expired session would answer 200 and boot would report "ready",
// deferring the no_cookie to the first real API call.
const calls: string[][] = []
await assert.rejects(
waitForHermesReady('https://gateway.example', {
token: 'session-token',
fetchPublicJson: async () => {
throw new Error('public probe must not be used when credentialed')
},
fetchJson: async url => {
calls.push(['status', url])
return { version: '0.19.0' }
},
probeHealth: async url => {
calls.push(['probe', url])
throw new Error(GATE_401)
},
probeIsCredentialed: true,
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
}),
(error: any) => {
assert.equal(isReauthRequiredError(error), true)
assert.equal(error.needsOauthLogin, true)
assert.match(error.message, /remote gateway session has expired/i)
return true
}
)
// Fail fast: never reached the public /api/status leg.
assert.deepEqual(calls, [['probe', 'https://gateway.example/api/health']])
})
test('a credentialed 403 is also a terminal reauth failure', async () => {
await assert.rejects(
waitForHermesReady('https://gateway.example', {
fetchPublicJson: async () => ({}),
fetchJson: async () => ({}),
probeHealth: async () => {
throw new Error('403: {"detail":"Forbidden"}')
},
probeIsCredentialed: true,
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
}),
(error: any) => isReauthRequiredError(error)
)
})
test('a credentialed probe still uses the 404 fallback for a genuinely missing route', async () => {
// With credentials the gate lets the request through to the SPA catch-all,
// so an old backend answers a real 404 — that must still fall back, not be
// mistaken for a rejected session.
const calls: string[][] = []
await waitForHermesReady('https://gateway.example', {
token: 'session-token',
fetchPublicJson: async () => {
throw new Error('public probe must not be used when credentialed')
},
fetchJson: async url => {
calls.push(['status', url])
return { version: '0.19.0' }
},
probeHealth: async url => {
calls.push(['probe', url])
throw new Error('404: {"detail":"Not Found"}')
},
probeIsCredentialed: true,
sleep: async () => {},
timeoutMs: 100,
pollMs: 1
})
assert.deepEqual(calls, [
['probe', 'https://gateway.example/api/health'],
['status', 'https://gateway.example/api/status']
])
})
test('a non-gate 401 keeps polling rather than skipping a misconfigured health route', async () => {
const calls: string[][] = []
let currentTime = 0
await assert.rejects(
waitForHermesReady('http://127.0.0.1:9000', {
fetchPublicJson: async url => {
calls.push(['public', url])
throw new Error('401: {"detail":"Unauthorized"}')
},
fetchJson: async url => {
calls.push(['token', url])
},
sleep: async () => {},
now: () => {
currentTime += 20
return currentTime
},
timeoutMs: 50,
pollMs: 1
}),
/401: \{"detail":"Unauthorized"\}/
)
assert.ok(calls.length > 0)
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
})
test('credentialed 5xx and 429 keep polling — only 401/403 are terminal', async () => {
for (const transient of ['500: boom', '429: {"detail":"Too Many Requests"}']) {
let attempts = 0
let currentTime = 0
await assert.rejects(
waitForHermesReady('https://gateway.example', {
fetchPublicJson: async () => ({}),
fetchJson: async () => ({}),
probeHealth: async () => {
attempts += 1
throw new Error(transient)
},
probeIsCredentialed: true,
sleep: async () => {},
now: () => {
currentTime += 20
return currentTime
},
timeoutMs: 100,
pollMs: 1
}),
(error: any) => isReauthRequiredError(error) === false
)
assert.ok(attempts > 1, `${transient} should have retried, got ${attempts} attempt(s)`)
}
})
test('error-shape predicates', () => {
assert.equal(isGatedMissingHealthError(new Error(GATE_401)), true)
assert.equal(isGatedMissingHealthError(new Error('401: {"detail":"Unauthorized"}')), false)
assert.equal(isGatedMissingHealthError(new Error('404: {"detail":"Not Found"}')), false)
assert.equal(isAuthRejectionError(new Error(GATE_401)), true)
assert.equal(isAuthRejectionError(new Error('403: {"detail":"Forbidden"}')), true)
assert.equal(isAuthRejectionError(new Error('404: {"detail":"Not Found"}')), false)
assert.equal(isAuthRejectionError(new Error('429: slow down')), false)
assert.equal(isAuthRejectionError(new Error('500: boom')), false)
// A gated 401 must NOT be conflated with a missing route by the 404 predicate.
assert.equal(isMissingHealthEndpointError(new Error(GATE_401)), false)
})

View file

@ -0,0 +1,169 @@
export const DEFAULT_BACKEND_READY_TIMEOUT_MS = 45_000
export const DEFAULT_BACKEND_READY_POLL_MS = 500
// A cold backend can stall its event loop for tens of seconds while Windows
// scans and byte-compiles the gateway import tree. At the default 15s socket
// timeout only three probes fit in the budget; a short one keeps retrying
// across the stall. Health only — the legacy /api/status fallback is genuinely
// slow to answer and keeps the caller's default timeout.
export const DEFAULT_HEALTH_PROBE_TIMEOUT_MS = 5_000
type FetchPublicJson = (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
type FetchJson = (url: string, token?: string | null, options?: { timeoutMs?: number }) => Promise<unknown>
export interface HermesReadyOptions {
fetchPublicJson: FetchPublicJson
fetchJson: FetchJson
token?: string | null
signal?: AbortSignal
timeoutMs?: number
pollMs?: number
healthProbeTimeoutMs?: number
sleep?: (ms: number) => Promise<void>
now?: () => number
/**
* Credentialed health probe. When supplied, readiness is probed with the
* connection's own credentials instead of anonymously which is what lets
* a gated backend answer 404 for a genuinely missing /api/health, and what
* makes a 401 from this probe mean "session rejected" rather than "route
* behind a gate". Defaults to the credential-free `fetchPublicJson`.
*/
probeHealth?: (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
/**
* Whether `probeHealth` actually presents credentials. Distinguishes the
* two very different meanings of a 401 (see `waitForHermesReady`).
*/
probeIsCredentialed?: boolean
}
export const REMOTE_SESSION_EXPIRED_MESSAGE =
'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.'
export function isMissingHealthEndpointError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? '')
return /^404:/.test(message) || message.includes('endpoint is likely missing')
}
/**
* True for a hard auth rejection (401/403) as opposed to a transient failure.
* Deliberately shape-based: 429 is a throttle and 5xx is a server fault, and
* both must keep polling.
*/
export function isAuthRejectionError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? '')
return /^40[13]:/.test(message)
}
/**
* True for an auth rejection carrying the dashboard gate's "no session at all"
* shape. On a backend that predates `/api/health`, the gate runs ahead of the
* SPA catch-all, so an unknown `/api/*` path is rejected as unauthenticated
* instead of 404 this is the signal that an ANONYMOUS probe cannot reach the
* route, and the reason a credential-free 401 must fall back to `/api/status`
* rather than be reported as a boot failure.
*/
export function isGatedMissingHealthError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? '')
return isAuthRejectionError(error) && message.includes('no_cookie')
}
/** Tag a terminal reauth failure the main process latches and the overlay keys on. */
export function makeReauthRequiredError(detail?: string): Error {
const error = new Error(REMOTE_SESSION_EXPIRED_MESSAGE) as any
error.needsOauthLogin = true
error.isReauthRequired = true
if (detail) {
error.detail = detail
}
return error
}
export function isReauthRequiredError(error: unknown): boolean {
return Boolean((error as any)?.isReauthRequired)
}
function supersededError() {
const error: any = new Error('SSH bootstrap was superseded by newer connection settings.')
error.kind = 'superseded'
return error
}
export async function waitForHermesReady(baseUrl: string, options: HermesReadyOptions): Promise<void> {
const timeoutMs = options.timeoutMs ?? DEFAULT_BACKEND_READY_TIMEOUT_MS
const pollMs = options.pollMs ?? DEFAULT_BACKEND_READY_POLL_MS
const healthProbeTimeoutMs = options.healthProbeTimeoutMs ?? DEFAULT_HEALTH_PROBE_TIMEOUT_MS
const now = options.now ?? Date.now
const signal = options.signal
const sleep =
options.sleep ??
(ms =>
new Promise<void>((resolve, reject) => {
const timer = setTimeout(resolve, ms)
signal?.addEventListener(
'abort',
() => {
clearTimeout(timer)
reject(supersededError())
},
{ once: true }
)
}))
const base = baseUrl.replace(/\/+$/, '')
const deadline = now() + timeoutMs
const probeHealth = options.probeHealth ?? options.fetchPublicJson
const probeIsCredentialed = Boolean(options.probeIsCredentialed)
let lastError: unknown = null
let useStatusFallback = false
while (now() < deadline) {
if (signal?.aborted) {
throw supersededError()
}
try {
if (useStatusFallback) {
await options.fetchJson(`${base}/api/status`, options.token)
} else {
await probeHealth(`${base}/api/health`, { timeoutMs: healthProbeTimeoutMs })
}
return
} catch (error) {
lastError = error
// A confirmed 401/403 from a CREDENTIALED probe means the session was
// rejected, not that the route is missing. Fail fast into a reauth
// state: falling back to the public /api/status would answer 200 and
// report a dead session as "ready", deferring the failure to the first
// real API call. Applies to the /api/status leg too — it is routed
// through the same credentials.
if (probeIsCredentialed && isAuthRejectionError(error)) {
throw makeReauthRequiredError(error instanceof Error ? error.message : String(error))
}
// An explicitly missing route means the backend predates /api/health.
// So does a gate-shaped 401 on an ANONYMOUS probe: the dashboard auth
// gate runs ahead of the SPA catch-all, so a pre-/api/health backend
// rejects the unknown path as unauthenticated instead of 404 and a
// credential-free probe can never observe the 404. Timeouts, 5xx, 429,
// and non-gate 401s keep polling health.
if (!useStatusFallback && (isMissingHealthEndpointError(error) || isGatedMissingHealthError(error))) {
useStatusFallback = true
continue
}
await sleep(pollMs)
}
}
const detail = lastError instanceof Error ? lastError.message : 'timeout'
throw new Error(`Hermes backend did not become ready: ${detail}`)
}

View file

@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
import { test } from 'vitest'
import { shouldLatchBackendStartFailure } from './backend-start-failure'
import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure'
test('latches a LOCAL backend failure so the install-retry loop is broken', () => {
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false }), true)
@ -21,3 +21,32 @@ test('the two branches are mutually exclusive (a failure either latches or stays
assert.equal(latched, !attemptedRemote)
}
})
test('latches a CONFIRMED remote reauth failure so the overlay stays clickable', () => {
// Without this the non-latching remote path re-runs startHermes on every
// getConnection/api call, re-emits running:true, and the overlay hides
// itself — the "Sign in" button flickers away before it can be clicked.
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: true }), true)
})
test('does not latch a transient remote failure as reauth', () => {
// A mint timeout or a host unreachable across sleep must still self-heal.
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: false }), false)
})
test('never latches a LOCAL failure as reauth (that is backendStartFailure job)', () => {
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: true }), false)
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: false }), false)
})
test('the two latches never fire for the same failure', () => {
// They are complementary, not overlapping: local failures latch via
// backendStartFailure, confirmed remote reauth latches via its own flag.
for (const attemptedRemote of [true, false]) {
for (const isReauth of [true, false]) {
const start = shouldLatchBackendStartFailure({ attemptedRemote })
const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth })
assert.ok(!(start && reauth), `both latched for remote=${attemptedRemote} reauth=${isReauth}`)
}
}
})

View file

@ -39,3 +39,34 @@ export interface BackendStartFailureContext {
export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean {
return !context.attemptedRemote
}
export interface RemoteReauthFailureContext {
/** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */
attemptedRemote: boolean
/**
* True when the failure was a CONFIRMED auth rejection (a credentialed
* probe got 401/403), not a transient connectivity fault.
*/
isReauth: boolean
}
/**
* Whether a failed remote boot should latch as a reauth failure.
*
* This is the deliberate counterpart to `shouldLatchBackendStartFailure`,
* which never latches a remote failure because remote faults are usually
* transient and must stay retryable. A *confirmed* reauth rejection is the
* exception: it cannot self-heal, because nothing will change until the user
* signs in again.
*
* Without a latch, the non-latching remote path actively prevents recovery.
* Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits
* `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error)
* && !boot.running`) hides itself — so the "Sign in" button flickers out from
* under the user before they can click it. Latching holds the overlay still
* and clickable. Cleared on every recovery path (reset, repair, apply-config,
* and a confirmed sign-in) so a fresh session boots normally.
*/
export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean {
return context.attemptedRemote && context.isReauth
}

View file

@ -1,6 +1,7 @@
async function applyConnectionChange({
cancelAndWait,
isPrimary,
rehomePrimary = null,
scope,
sendApplied,
stopPool,
@ -16,6 +17,12 @@ async function applyConnectionChange({
return
}
if (rehomePrimary) {
await rehomePrimary()
return
}
await teardownPrimary()
sendApplied()
}

View file

@ -0,0 +1,54 @@
import assert from 'node:assert/strict'
import { test } from 'vitest'
import { findGitBash } from './find-git-bash'
const yes = () => true
const no = () => false
test('HERMES_GIT_BASH_PATH override takes precedence', () => {
const result = findGitBash({
isWindows: true,
env: { HERMES_GIT_BASH_PATH: 'D:\\CustomGit\\bin\\bash.exe' },
fileExists: yes,
findOnPath: () => null
})
assert.equal(result, 'D:\\CustomGit\\bin\\bash.exe')
})
test('HERMES_GIT_BASH_PATH invalid path falls through to candidates', () => {
const env = {
HERMES_GIT_BASH_PATH: 'X:\\Missing\\bash.exe',
LOCALAPPDATA: 'C:\\Users\\test\\AppData\\Local',
ProgramFiles: 'C:\\Program Files',
'ProgramFiles(x86)': 'C:\\Program Files (x86)'
}
const fileExists = (p: string) => p !== 'X:\\Missing\\bash.exe' && p.includes('Program Files\\Git\\bin\\bash.exe')
const result = findGitBash({ isWindows: true, env, fileExists, findOnPath: () => null })
assert.equal(result, 'C:\\Program Files\\Git\\bin\\bash.exe')
})
test('HERMES_GIT_BASH_PATH empty string is ignored', () => {
const result = findGitBash({
isWindows: true,
env: { HERMES_GIT_BASH_PATH: '', LOCALAPPDATA: '' },
fileExists: no,
findOnPath: () => 'C:\\msys64\\usr\\bin\\bash.exe'
})
assert.equal(result, 'C:\\msys64\\usr\\bin\\bash.exe')
})
test('non-Windows uses findOnPath', () => {
const result = findGitBash({
isWindows: false,
env: {},
fileExists: no,
findOnPath: () => '/usr/bin/bash'
})
assert.equal(result, '/usr/bin/bash')
})

View file

@ -0,0 +1,67 @@
import path from 'node:path'
export interface GitBashOptions {
isWindows: boolean
env: Record<string, string | undefined>
fileExists: (filePath: string) => boolean
findOnPath?: (command: string) => string | null
}
/**
* Locate bash.exe on Windows.
* Resolution order (first match wins):
* 1. HERMES_GIT_BASH_PATH env var override
* 2. PortableGit under %LOCALAPPDATA%\hermes\git\ (install.ps1)
* 3. Standard Git for Windows install locations
* 4. %LOCALAPPDATA%\Programs\Git\ (user-scoped)
* 5. bash on PATH
*/
export function findGitBash(opts: GitBashOptions): string | null {
const { isWindows, env, fileExists, findOnPath } = opts
if (!isWindows) {
return findOnPath ? findOnPath('bash') : null
}
// Respect HERMES_GIT_BASH_PATH if set (mirrors tools/environments/local.py:_find_bash).
const gitBashPath = env.HERMES_GIT_BASH_PATH
if (gitBashPath && fileExists(gitBashPath)) {
return gitBashPath
}
const localAppData = env.LOCALAPPDATA || ''
const candidates: string[] = []
// Candidate paths are Windows paths regardless of host platform (tests run
// on POSIX CI hosts too), so join with win32 semantics explicitly.
const joinWin = path.win32.join
if (localAppData) {
candidates.push(joinWin(localAppData, 'hermes', 'git', 'bin', 'bash.exe'))
candidates.push(joinWin(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe'))
}
candidates.push(joinWin(env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe'))
candidates.push(joinWin(env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe'))
if (localAppData) {
candidates.push(joinWin(localAppData, 'Programs', 'Git', 'bin', 'bash.exe'))
}
for (const candidate of candidates) {
if (fileExists(candidate)) {
return candidate
}
}
if (findOnPath) {
const onPath = findOnPath('bash')
if (onPath) {
return onPath
}
}
return null
}

View file

@ -0,0 +1,133 @@
import assert from 'node:assert/strict'
import { test } from 'vitest'
import { createFirstRunSetupGate } from './first-run-setup-gate'
const bootstrapBackend = {
activeRoot: '/tmp/hermes-home/hermes-agent',
kind: 'bootstrap-needed',
platform: 'linux'
}
function delay(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms))
}
async function settledState(promise: Promise<unknown>) {
return Promise.race([promise.then(() => 'resolved'), delay(10).then(() => 'pending')])
}
test('first-run setup gate skips non-bootstrap backends', async () => {
const prompts = []
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
await gate.wait({ kind: 'remote' })
await gate.wait(null)
assert.deepEqual(prompts, [])
assert.equal(gate.hasWaiter(), false)
})
test('first-run setup gate prompts once for concurrent waits', async () => {
const prompts = []
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
const first = gate.wait(bootstrapBackend)
const second = gate.wait(bootstrapBackend)
assert.equal(gate.hasWaiter(), true)
assert.equal(prompts.length, 1)
assert.equal(await settledState(first), 'pending')
gate.continueLocal()
assert.deepEqual(await Promise.all([first, second]), ['continue-local', 'continue-local'])
assert.equal(gate.hasWaiter(), false)
assert.equal(gate.isLocalBootstrapConfirmed(), true)
})
test('continueLocal keeps the setup choice visible until bootstrap owns the overlay', async () => {
let hidden = 0
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
gate.continueLocal()
assert.equal(await pending, 'continue-local')
assert.equal(hidden, 0)
assert.equal(gate.isLocalBootstrapConfirmed(), true)
})
test('retry reset preserves the local install confirmation', async () => {
const prompts = []
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
gate.continueLocal()
await pending
gate.resetForRetry()
await gate.wait(bootstrapBackend)
assert.equal(gate.isLocalBootstrapConfirmed(), true)
assert.equal(prompts.length, 1)
assert.equal(gate.hasWaiter(), false)
})
test('retry reset explicitly settles an active waiter without allowing local bootstrap', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
gate.resetForRetry()
assert.equal(await pending, 'reset')
assert.equal(gate.hasWaiter(), false)
assert.equal(gate.isLocalBootstrapConfirmed(), false)
})
test('repair reset clears the local install confirmation and shows the gate again', async () => {
const prompts = []
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
gate.continueLocal()
await pending
gate.resetForRepair()
const next = gate.wait(bootstrapBackend)
assert.equal(gate.isLocalBootstrapConfirmed(), false)
assert.equal(prompts.length, 2)
assert.equal(gate.hasWaiter(), true)
gate.continueLocal()
await next
})
test('remote apply settles the gated boot for remote re-resolution and hides the choice', async () => {
let hidden = 0
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
const resumedWaiter = gate.abandonForRemoteApply()
assert.equal(resumedWaiter, true)
assert.equal(hidden, 1)
assert.equal(gate.hasWaiter(), false)
assert.equal(gate.isLocalBootstrapConfirmed(), false)
assert.equal(await pending, 'remote-applied')
})
test('remote apply without a waiter has no first-run side effects', async () => {
let hidden = 0
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
const pending = gate.wait(bootstrapBackend)
gate.continueLocal()
await pending
assert.equal(gate.abandonForRemoteApply(), false)
assert.equal(hidden, 0)
assert.equal(gate.isLocalBootstrapConfirmed(), true)
})

View file

@ -0,0 +1,146 @@
interface FirstRunSetupBackend {
activeRoot?: string
kind?: string
platform?: string
}
interface FirstRunSetupGateOptions {
hideChoice?: () => void
log?: (message: string) => void
onStuck?: (backend: FirstRunSetupBackend, stuckAfterMs: number) => void
promptChoice?: (backend: FirstRunSetupBackend) => void
stuckAfterMs?: number
}
export type FirstRunSetupDecision = 'continue-local' | 'remote-applied' | 'reset'
export function createFirstRunSetupGate({
hideChoice,
log,
onStuck,
promptChoice,
stuckAfterMs = 120000
}: FirstRunSetupGateOptions = {}) {
let localBootstrapConfirmed = false
let waiter: {
promise: Promise<FirstRunSetupDecision>
resolve: (decision: FirstRunSetupDecision) => void
} | null = null
let stuckTimer: ReturnType<typeof setTimeout> | null = null
const clearStuckTimer = () => {
if (stuckTimer) {
clearTimeout(stuckTimer)
stuckTimer = null
}
}
const armStuckTimer = (backend: FirstRunSetupBackend) => {
clearStuckTimer()
if (!Number.isFinite(stuckAfterMs) || stuckAfterMs <= 0 || typeof log !== 'function') {
return
}
stuckTimer = setTimeout(() => {
onStuck?.(backend, stuckAfterMs)
log(
`[bootstrap] still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)}s ` +
`(platform=${backend?.platform || 'unknown'})`
)
}, stuckAfterMs)
if (typeof stuckTimer.unref === 'function') {
stuckTimer.unref()
}
}
const shouldGate = (backend?: FirstRunSetupBackend | null) =>
Boolean(backend && backend.kind === 'bootstrap-needed' && !localBootstrapConfirmed)
const wait = async (backend?: FirstRunSetupBackend | null) => {
if (!shouldGate(backend)) {
return 'continue-local' as const
}
if (waiter) {
return waiter.promise
}
promptChoice?.(backend)
armStuckTimer(backend)
let resolveWaiter: (decision: FirstRunSetupDecision) => void = () => {}
const promise = new Promise<FirstRunSetupDecision>(resolve => {
resolveWaiter = resolve
})
waiter = { promise, resolve: resolveWaiter }
return promise
}
const settleWaiter = (decision: FirstRunSetupDecision) => {
clearStuckTimer()
if (!waiter) {
return false
}
const activeWaiter = waiter
waiter = null
activeWaiter.resolve(decision)
return true
}
const continueLocal = () => {
localBootstrapConfirmed = true
settleWaiter('continue-local')
}
const resetForRetry = () => {
// Reset paths are followed by a renderer reload / fresh startHermes() call.
// Settle the old boot explicitly so it cannot fall through into local
// bootstrap and cannot leak a forever-pending connection promise.
settleWaiter('reset')
}
const resetForRepair = () => {
resetForRetry()
localBootstrapConfirmed = false
}
const abandonForRemoteApply = () => {
// Resume the gated startHermes() with an explicit remote decision. The
// caller re-resolves the newly-persisted remote config instead of falling
// through into local bootstrap or leaking the original connection promise.
const resumedWaiter = settleWaiter('remote-applied')
if (!resumedWaiter) {
return false
}
localBootstrapConfirmed = false
hideChoice?.()
return true
}
const isLocalBootstrapConfirmed = () => localBootstrapConfirmed
const hasWaiter = () => Boolean(waiter)
return {
abandonForRemoteApply,
continueLocal,
hasWaiter,
isLocalBootstrapConfirmed,
resetForRepair,
resetForRetry,
shouldGate,
wait
}
}

View file

@ -0,0 +1,119 @@
import assert from 'node:assert/strict'
import { test, vi } from 'vitest'
import { applyConnectionChange } from './connection-apply'
import { createFirstRunSetupGate } from './first-run-setup-gate'
import { runPrimaryBackendStartup } from './primary-backend-startup'
import { rehomePrimaryConnection } from './primary-connection-rehome'
test('a first-run bootstrap-needed remote apply connects without ensuring or bootstrapping locally', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const bootstrapBackend = {
activeRoot: '/tmp/hermes-home/hermes-agent',
kind: 'bootstrap-needed',
platform: 'linux'
}
const candidateRemote = {
authMode: 'token',
baseUrl: 'https://gateway.example.com/hermes',
source: 'settings',
token: 'secret',
wsUrl: 'wss://gateway.example.com/hermes/api/ws?token=secret'
}
let savedRemote: typeof candidateRemote | null = null
const resolveRemote = vi.fn(async () => savedRemote)
const connectRemote = vi.fn(async remote => ({ ...remote, mode: 'remote' as const }))
const runBootstrap = vi.fn()
const ensureLocalRuntime = vi.fn(async backend => {
await runBootstrap()
return { ...backend, command: 'hermes' }
})
const teardownPrimaryBackend = vi.fn(async () => {})
const cancelSshBootstrap = vi.fn(async () => {})
const teardownSsh = vi.fn(async () => {})
const clearLocalBootstrapFailure = vi.fn()
const notifyConnectionApplied = vi.fn()
const waitForLocalStart = vi.fn(async () => {})
const prepareLocalBackend = vi.fn(async () => bootstrapBackend)
const pendingConnection = runPrimaryBackendStartup({
connectRemote,
ensureLocalRuntime,
prepareLocalBackend,
resolveRemote,
waitForDecision: gate.wait,
waitForLocalStart
})
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
// Mirrors the IPC handler's production ordering: persist the tested config,
// then re-home. The pending start must re-resolve this saved value.
savedRemote = candidateRemote
await applyConnectionChange({
cancelAndWait: cancelSshBootstrap,
isPrimary: true,
rehomePrimary: () =>
rehomePrimaryConnection({
clearLocalBootstrapFailure,
mode: 'remote',
notifyConnectionApplied,
resumeFirstRunRemote: gate.abandonForRemoteApply,
teardownPrimaryBackend
}),
scope: '',
sendApplied: notifyConnectionApplied,
stopPool: vi.fn(),
teardownPrimary: teardownPrimaryBackend,
teardownSsh
})
assert.deepEqual(await pendingConnection, {
kind: 'remote',
connection: { ...candidateRemote, mode: 'remote' }
})
assert.deepEqual(resolveRemote.mock.calls, [[], []])
assert.deepEqual(connectRemote.mock.calls, [[candidateRemote]])
assert.deepEqual(waitForLocalStart.mock.calls, [[]])
assert.deepEqual(prepareLocalBackend.mock.calls, [[]])
assert.equal(ensureLocalRuntime.mock.calls.length, 0)
assert.equal(runBootstrap.mock.calls.length, 0)
assert.deepEqual(cancelSshBootstrap.mock.calls, [['']])
assert.deepEqual(teardownSsh.mock.calls, [['']])
assert.equal(teardownPrimaryBackend.mock.calls.length, 0)
assert.equal(clearLocalBootstrapFailure.mock.calls.length, 1)
assert.equal(notifyConnectionApplied.mock.calls.length, 0)
})
test('a primary apply without an active first-run gate tears down before reconnect notification', async () => {
const order: string[] = []
const clearLocalBootstrapFailure = vi.fn(() => order.push('clear-failure'))
const teardownPrimaryBackend = vi.fn(async () => {
order.push('teardown')
})
const notifyConnectionApplied = vi.fn(() => order.push('notify'))
assert.deepEqual(
await rehomePrimaryConnection({
clearLocalBootstrapFailure,
mode: 'remote',
notifyConnectionApplied,
resumeFirstRunRemote: () => false,
teardownPrimaryBackend
}),
{ resumedFirstRunRemote: false }
)
assert.deepEqual(teardownPrimaryBackend.mock.calls, [[{ soft: true }]])
assert.deepEqual(order, ['clear-failure', 'teardown', 'notify'])
})

View file

@ -34,9 +34,10 @@ import { stopBackendChild as stopBackendChildImpl } from './backend-child'
import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command'
import { createBackendConnectionState } from './backend-connection-state'
import { buildDesktopBackendEnv, normalizeHermesHomeRoot } from './backend-env'
import { isReauthRequiredError, waitForHermesReady } from './backend-health'
import { canImportHermesCli, shouldTrustHermesOverride, verifyHermesCli } from './backend-probes'
import { waitForDashboardPortAnnouncement } from './backend-ready'
import { shouldLatchBackendStartFailure } from './backend-start-failure'
import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure'
import { detectRemoteDisplay, isWindowsBinaryPathInWsl, isWslEnvironment } from './bootstrap-platform'
import { runBootstrap } from './bootstrap-runner'
import { applyConnectionChange, resolveTerminalConnection } from './connection-apply'
@ -78,6 +79,8 @@ import {
} from './desktop-uninstall'
import { installEmbedReferer } from './embed-referer'
import { createEventDeduper } from './event-dedupe'
import { findGitBash as _findGitBash } from './find-git-bash'
import { createFirstRunSetupGate } from './first-run-setup-gate'
import { readDirForIpc } from './fs-read-dir'
import { probeGatewayWebSocket } from './gateway-ws-probe'
import { scanGitRepos } from './git-repo-scan'
@ -116,7 +119,13 @@ import {
} from './hardening'
import { createLinkTitleWindow, guardLinkTitleSession, readLinkTitleWindowTitle } from './link-title-window'
import { ensureMainWindow } from './main-window-lifecycle'
import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions'
import {
oauthGuardMayHardFail,
oauthSessionIsLive,
resolveJsonBody,
resolveOauthRestAuth,
resolveReadinessProbeAuth
} from './native-auth-decisions'
import {
nativeRefreshUrl,
type NativeTokenSet,
@ -127,6 +136,8 @@ import {
import { runNativeLogin } from './native-oauth-login'
import { serializeJsonBody, setJsonRequestHeaders } from './oauth-net-request'
import { createKeepAwake } from './power-save'
import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup'
import { rehomePrimaryConnection } from './primary-connection-rehome'
import { decideProfileDeleteAction, profileNameFromDeleteRequest, resolveRouteProfile } from './profile-delete-routing'
import * as remoteLifecycle from './remote-lifecycle'
import { RemoteLivenessTracker, RemoteRevalidationCoordinator, revalidateRemoteConnection } from './remote-liveness'
@ -1011,6 +1022,13 @@ let bootstrapFailure = null
// Latched non-bootstrap backend spawn failure — stops getConnection() from
// respawning hermes serve backend children in a tight loop while boot is broken.
let backendStartFailure = null
// Latched CONFIRMED remote reauth failure. Remote failures deliberately do not
// latch via backendStartFailure (they're usually transient and must stay
// retryable), but a rejected session cannot self-heal — and the non-latching
// path actively breaks recovery: each retry re-emits running:true and hides
// the boot-failure overlay, so the "Sign in" button flickers away before it
// can be clicked. Cleared on every recovery path and on a confirmed sign-in.
let remoteReauthFailure = null
// Active first-launch install, so the renderer's Cancel button (and app quit)
// can abort the in-flight install.sh/ps1 instead of leaving it running.
let bootstrapAbortController = null
@ -1398,13 +1416,17 @@ let bootstrapState = {
log: [],
startedAt: null,
completedAt: null,
setupChoice: null,
unsupportedPlatform: null
}
let firstRunSetupGate = null
function broadcastBootstrapEvent(ev) {
if (ev.type === 'manifest') {
bootstrapState.manifest = ev
bootstrapState.active = true
bootstrapState.setupChoice = null
bootstrapState.startedAt = bootstrapState.startedAt || Date.now()
bootstrapState.stages = {}
@ -1432,14 +1454,30 @@ function broadcastBootstrapEvent(ev) {
} else if (ev.type === 'failed') {
bootstrapState.active = false
bootstrapState.error = ev.error || 'unknown error'
bootstrapState.setupChoice = null
} else if (ev.type === 'unsupported-platform') {
bootstrapState.active = false
bootstrapState.setupChoice = null
bootstrapState.unsupportedPlatform = {
platform: ev.platform,
activeRoot: ev.activeRoot,
installCommand: ev.installCommand,
docsUrl: ev.docsUrl
}
} else if (ev.type === 'setup-choice') {
bootstrapState.active = false
bootstrapState.error = null
bootstrapState.manifest = null
bootstrapState.stages = {}
bootstrapState.setupChoice = ev.active
? {
platform: ev.platform,
activeRoot: ev.activeRoot
}
: null
bootstrapState.unsupportedPlatform = null
} else if (ev.type === 'dismissed') {
resetBootstrapSnapshot()
}
if (!mainWindow || mainWindow.isDestroyed()) {
@ -1459,6 +1497,100 @@ function getBootstrapState() {
return bootstrapState
}
function resetBootstrapSnapshot() {
bootstrapState = {
active: false,
manifest: null,
stages: {},
error: null,
log: [],
startedAt: null,
completedAt: null,
setupChoice: null,
unsupportedPlatform: null
}
}
function promptFirstRunSetupChoice(backend) {
broadcastBootstrapEvent({
type: 'setup-choice',
active: true,
platform: backend.platform || process.platform,
activeRoot: backend.activeRoot || ACTIVE_HERMES_ROOT
})
}
function hideFirstRunSetupChoice() {
if (bootstrapState.setupChoice) {
broadcastBootstrapEvent({ type: 'setup-choice', active: false })
}
}
function getFirstRunSetupGate() {
if (!firstRunSetupGate) {
firstRunSetupGate = createFirstRunSetupGate({
hideChoice: hideFirstRunSetupChoice,
log: rememberLog,
onStuck: (_backend, stuckAfterMs) => {
updateBootProgress(
{
error: null,
message: `Still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)} seconds`,
phase: 'bootstrap.choice',
progress: 12,
running: true
},
{ allowDecrease: true }
)
},
promptChoice: promptFirstRunSetupChoice
})
}
return firstRunSetupGate
}
async function waitForFirstRunSetupChoice(backend) {
const gate = getFirstRunSetupGate()
if (!gate.shouldGate(backend)) {
return 'continue-local'
}
updateBootProgress(
{
error: null,
message: 'Waiting for first-run setup choice',
phase: 'bootstrap.choice',
progress: 12,
running: true
},
{ allowDecrease: true }
)
return gate.wait(backend)
}
function continueFirstRunLocalBootstrap() {
getFirstRunSetupGate().continueLocal()
}
function abandonFirstRunSetupChoiceForRemoteApply() {
const gate = getFirstRunSetupGate()
if (!gate.hasWaiter()) {
return false
}
const resumedGatedConnection = gate.abandonForRemoteApply()
if (resumedGatedConnection) {
broadcastBootstrapEvent({ type: 'dismissed' })
}
return resumedGatedConnection
}
function updateBootProgress(update, options: { allowDecrease?: boolean } = {}) {
const nextProgressRaw =
typeof update.progress === 'number' ? clampBootProgress(update.progress) : bootProgressState.progress
@ -1915,48 +2047,16 @@ function findSystemPython() {
return null
}
// findGitBash — locate bash.exe on Windows. Hermes' terminal tool requires
// bash (POSIX shell), and on Windows that's almost always Git for Windows'
// bundled Git Bash. We check the same set of locations tools/environments/
// local.py:_find_bash() checks at runtime, so a positive result here means
// the agent will be able to start a terminal too.
//
// On non-Windows hosts bash is part of the OS and this just returns the
// first bash on PATH.
// findGitBash — locate bash.exe on Windows. Resolves HERMES_GIT_BASH_PATH
// first (mirrors tools/environments/local.py:_find_bash), then PortableGit,
// standard install locations, and finally PATH.
function findGitBash() {
if (!IS_WINDOWS) {
return findOnPath('bash')
}
// install.ps1 drops PortableGit at %LOCALAPPDATA%\hermes\git\... — checked
// first so users who installed via install.ps1 are detected before we
// start probing system-wide locations.
const localAppData = process.env.LOCALAPPDATA || ''
const candidates = []
if (localAppData) {
candidates.push(path.join(localAppData, 'hermes', 'git', 'bin', 'bash.exe'))
candidates.push(path.join(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe'))
}
// Standard Git for Windows install locations.
candidates.push(path.join(process.env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe'))
candidates.push(path.join(process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe'))
if (localAppData) {
candidates.push(path.join(localAppData, 'Programs', 'Git', 'bin', 'bash.exe'))
}
for (const candidate of candidates) {
if (fileExists(candidate)) {
return candidate
}
}
// Last resort — bash on PATH (covers WSL bash, MSYS2, custom installs).
// On WSL hosts findOnPath itself filters out Windows-binary paths via
// isWindowsBinaryPathInWsl, so we won't hand back a wsl.exe shim either.
return findOnPath('bash')
return _findGitBash({
isWindows: IS_WINDOWS,
env: process.env,
fileExists,
findOnPath
})
}
function getVenvPython(venvRoot) {
@ -2686,6 +2786,11 @@ async function applyUpdates(opts = {}) {
const venvBin = path.join(updateRoot, 'venv', IS_WINDOWS ? 'Scripts' : 'bin')
// ── Pre-flight state.db integrity guard (#68474) ─────────────────
// Emergency backup and header verification before the update touches
// anything. Runs while the backend is still alive.
preflightStateDb(HERMES_HOME, rememberLog)
// Stop our own backend(s) and wait for the venv shim to unlock BEFORE we
// spawn the updater. Without this the updater races a still-locked
// hermes.exe (held by the backend child / its grandchildren) and the update
@ -2883,6 +2988,92 @@ function runningAppBundle() {
return dir.endsWith('.app') ? dir : null
}
// ── Pre-flight state.db integrity guard (#68474) ─────────────────────
// Take an emergency snapshot of state.db and verify the live copy is
// intact before any update process mutates the install. Runs in the
// desktop Electron process itself, before the backend is killed and
// before the updater is spawned — a separate safety net from the
// Python-level pre-update snapshot inside `hermes update`.
function preflightStateDb(hermesHome, rememberLog) {
const stateDbPath = path.join(hermesHome, 'state.db')
if (!fileExists(stateDbPath)) {
rememberLog('[updates] state.db pre-flight: not found (fresh install?)')
return
}
try {
const stat = fs.statSync(stateDbPath)
if (stat.size > 100) {
const fd = fs.openSync(stateDbPath, 'r')
const header = Buffer.alloc(16)
fs.readSync(fd, header, 0, 16, 0)
fs.closeSync(fd)
const expectedHeader = Buffer.from('SQLite format 3\0')
const headerOk = header.equals(expectedHeader)
rememberLog(
`[updates] state.db pre-flight: size=${stat.size}, ` +
`headerOk=${headerOk}, headerHex=${header.toString('hex')}`
)
if (!headerOk) {
rememberLog(
'[updates] state.db header is INVALID before update — ' +
'this indicates pre-existing corruption or a concurrent write issue'
)
}
// Emergency timestamped backup, separate from the Python-level snapshot.
const ts = new Date().toISOString().replace(/[:.]/g, '-')
const emergencyPath = path.join(hermesHome, `state.db.pre-update-emergency-${ts}.bak`)
try {
fs.copyFileSync(stateDbPath, emergencyPath)
const emergStat = fs.statSync(emergencyPath)
rememberLog(`[updates] emergency state.db backup: ${emergencyPath} ` + `(${emergStat.size} bytes)`)
// Prune to the 2 most recent emergency backups.
try {
const homeDir = fs.readdirSync(hermesHome)
const backups = homeDir
.filter(
f =>
f.startsWith('state.db.pre-update-emergency-') &&
f.endsWith('.bak') &&
f !== path.basename(emergencyPath)
)
.sort()
.reverse()
for (const old of backups.slice(2)) {
try {
fs.unlinkSync(path.join(hermesHome, old))
} catch {
void 0
}
}
} catch {
void 0
}
} catch (copyErr) {
rememberLog(`[updates] emergency state.db backup failed: ${copyErr.message}`)
}
} else {
rememberLog(`[updates] state.db too small (${stat.size} bytes) for a valid SQLite database`)
}
} catch (statErr) {
rememberLog(`[updates] could not stat state.db before update: ${statErr.message}`)
}
}
function shellQuote(value) {
return `'${String(value).replace(/'/g, `'\\''`)}'`
}
@ -2901,9 +3092,12 @@ async function applyUpdatesPosixInApp(opts: any) {
return { ok: true, manual: true, command: 'hermes update', hermesRoot: updateRoot }
}
// ── Pre-flight state.db integrity guard (#68474) ──
preflightStateDb(HERMES_HOME, rememberLog)
// Put the Hermes-managed Node and the venv on PATH so `hermes desktop`'s
// npm build can find them on a machine with no system Node. Windows portable
// Node lives directly under %LOCALAPPDATA%\hermes\node, not node\bin.
// Node lives directly under %LOCALAPPDATA%\\hermes\\node, not node\\bin.
// PYTHONUNBUFFERED: `hermes update` writes to a pipe here, so CPython
// block-buffers stdout and long quiet steps (the pre-update backup can zip
// multi-GB archives for minutes) stream nothing to the progress UI — users
@ -4634,40 +4828,88 @@ function closePreviewWatchers() {
}
}
async function waitForHermes(baseUrl, token, signal?) {
const deadline = Date.now() + 45_000
let lastError = null
// Best-effort read of a gateway's advertised auth providers, cached per base
// URL for the life of the process. Used by the oauth pre-flight guard to tell
// a password-provider gateway (which cannot satisfy the bearer/cookie checks
// by design) from a real OAuth one. Any failure returns [] so callers keep the
// strict guard — backends predating /api/auth/providers are unaffected.
const gatewayAuthProvidersCache = new Map<string, any[]>()
while (Date.now() < deadline) {
if (signal?.aborted) {
const error: any = new Error('SSH bootstrap was superseded by newer connection settings.')
error.kind = 'superseded'
throw error
async function gatewayAuthProviders(baseUrl) {
const cached = gatewayAuthProvidersCache.get(baseUrl)
if (cached) {
return cached
}
let providers = []
try {
const body = (await fetchPublicJson(`${baseUrl}/api/auth/providers`, { timeoutMs: 8_000 })) as any
if (Array.isArray(body?.providers)) {
providers = body.providers
.filter(p => p && typeof p === 'object')
.map(p => ({ name: String(p.name || ''), supportsPassword: Boolean(p.supports_password) }))
.filter(p => p.name)
}
} catch {
// Optional metadata — an unreadable list keeps the strict guard.
}
try {
await fetchJson(`${baseUrl}/api/status`, token)
gatewayAuthProvidersCache.set(baseUrl, providers)
return
} catch (error) {
lastError = error
await new Promise((resolve, reject) => {
const timer = setTimeout(resolve, 500)
signal?.addEventListener(
'abort',
() => {
clearTimeout(timer)
const aborted: any = new Error('SSH bootstrap was superseded by newer connection settings.')
aborted.kind = 'superseded'
reject(aborted)
},
{ once: true }
)
})
return providers
}
// Build the readiness probe for a connection's auth mode. A gated gateway
// must be probed with the SAME credentials the rest of the connection uses:
// an anonymous probe 401s forever against a live session, and it can never
// see the 404 that identifies a backend predating /api/health (the auth gate
// answers before the SPA catch-all). `probeIsCredentialed` tells
// waitForHermesReady how to read a 401 — rejected session vs gated route.
async function buildReadinessHealthProbe(baseUrl, authMode, token) {
const nativeAt = authMode === 'oauth' ? await ensureNativeAccessToken(baseUrl).catch(() => null) : null
const probeAuth = resolveReadinessProbeAuth(authMode, nativeAt, token)
if (probeAuth.kind === 'bearer') {
return {
// fetchJson takes the bearer via `options.bearer` — a raw `headers`
// option is ignored, so passing one here would silently probe
// uncredentialed and reintroduce the 401 loop.
probeHealth: (url, options: any = {}) => fetchJson(url, null, { ...options, bearer: probeAuth.token }),
probeIsCredentialed: true
}
}
throw new Error(`Hermes backend did not become ready: ${lastError?.message || 'timeout'}`)
if (probeAuth.kind === 'cookie') {
return {
probeHealth: (url, options: any = {}) => fetchJsonViaOauthSession(url, options),
probeIsCredentialed: true
}
}
if (probeAuth.kind === 'token' && probeAuth.token) {
return {
probeHealth: (url, options: any = {}) => fetchJson(url, probeAuth.token, options),
probeIsCredentialed: true
}
}
return { probeHealth: fetchPublicJson, probeIsCredentialed: false }
}
async function waitForHermes(baseUrl, token, signal?, authMode?) {
const { probeHealth, probeIsCredentialed } = await buildReadinessHealthProbe(baseUrl, authMode, token)
return waitForHermesReady(baseUrl, {
token,
signal,
fetchPublicJson,
fetchJson: probeIsCredentialed ? (url, _token, options) => probeHealth(url, options) : fetchJson,
probeHealth,
probeIsCredentialed
})
}
function getWindowButtonPosition() {
@ -6668,7 +6910,10 @@ async function buildRemoteConnection(
// here would reject a freshly-completed native sign-in and loop the UI back
// into "not signed in" even though mintGatewayWsTicket would succeed with
// the stored bearer.
if (!oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl))) {
if (
!oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl)) &&
oauthGuardMayHardFail(await gatewayAuthProviders(baseUrl))
) {
const err = new Error(
'Remote Hermes gateway uses OAuth, but you are not signed in. ' +
'Open Settings → Gateway and click "Sign in", or switch back to Local.'
@ -6909,7 +7154,7 @@ async function bootstrapSshConnectionInner(profile, sshConfig, reuseToken, sourc
forward: (localPort, remotePort) => ssh.forward(localPort, remotePort),
cancelForward: (localPort, remotePort) => ssh.cancelForward(localPort, remotePort),
pickLocalPort,
waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal),
waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal, 'token'),
probeReuseProof: sshProbeReuseProof,
adoptServedToken: adoptServedDashboardToken,
rememberLog: sshRememberLog,
@ -7382,6 +7627,7 @@ function stopBackendChild(child) {
// switch / crash recovery), which still resets boot progress + reloads.
function resetHermesConnection({ soft = false } = {}) {
backendStartFailure = null
remoteReauthFailure = null
remoteLiveness.clear()
const hermesProcess = backendConnectionState.invalidate()
stopBackendChild(hermesProcess)
@ -7582,7 +7828,7 @@ async function spawnPoolBackend(profile, entry) {
const remote = await resolveRemoteBackend(profile)
if (remote) {
await waitForHermes(remote.baseUrl, remote.token)
await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode)
return {
...remote,
@ -7774,6 +8020,13 @@ async function startHermes() {
throw backendStartFailure
}
// A confirmed remote reauth rejection is terminal until the user signs in.
// Short-circuiting here keeps the boot-failure overlay latched and its
// "Sign in" button clickable, instead of re-driving boot on every retry.
if (remoteReauthFailure) {
throw remoteReauthFailure
}
// E2E: simulate a boot failure without breaking the real backend. The boot
// progresses a few steps, then fails with the given error message.
if (BOOT_FAKE_ERROR) {
@ -7798,16 +8051,9 @@ async function startHermes() {
let attemptedRemote = primaryBackendIsRemote()
const connectionPromise = (async () => {
await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8)
// Resolve for the desktop's primary profile so a per-profile remote
// override on the active profile is honored (falls back to env / global).
// Re-read once resolved so the classification tracks the value actually used.
attemptedRemote = primaryBackendIsRemote()
const remote = await resolveRemoteBackend(primaryProfileKey())
if (remote) {
const connectRemote = async remote => {
await advanceBootProgress('backend.remote', `Connecting to remote Hermes backend at ${remote.baseUrl}`, 24)
await waitForHermes(remote.baseUrl, remote.token)
await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode)
updateBootProgress({
phase: 'backend.ready',
message: 'Remote Hermes backend is ready',
@ -7831,14 +8077,9 @@ async function startHermes() {
}
}
// Mutual exclusion with an in-app update (#50238). If this instance was
// relaunched while the Tauri updater is still applying an update, spawning
// a local backend now re-locks the venv shim and gets killed by the
// updater's straggler cleanup — looping. Park until the update finishes (or
// is detected stale), THEN start the backend. Local backends only; remote
// connections returned above and never touch the install tree.
await waitForUpdateToFinish()
await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8)
// Resolve for the desktop's primary profile so a per-profile remote
// override on the active profile is honored (falls back to env / global).
const token = crypto.randomBytes(32).toString('base64url')
// --port 0: the OS assigns an ephemeral port; the child announces it on stdout.
const backendArgs = ['serve', '--host', '127.0.0.1', '--port', '0']
@ -7853,8 +8094,32 @@ async function startHermes() {
backendArgs.unshift('--profile', activeProfile)
}
await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28)
const backend = await ensureRuntime(resolveHermesBackend(backendArgs))
const setup = await runPrimaryBackendStartup({
connectRemote,
ensureLocalRuntime: ensureRuntime,
prepareLocalBackend: async () => {
await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28)
return resolveHermesBackend(backendArgs)
},
resolveRemote: () => {
// Classify immediately before each throwing resolve. This callback runs
// both for an already-saved remote and after first-run remote Apply.
attemptedRemote = primaryBackendIsRemote()
return resolveRemoteBackend(primaryProfileKey())
},
waitForDecision: waitForFirstRunSetupChoice,
// Mutual exclusion with an in-app update (#50238). Remote connections
// return before this waiter; local starts park until the updater exits.
waitForLocalStart: waitForUpdateToFinish
})
if (setup.kind === 'remote') {
return setup.connection
}
const backend = setup.backend
// Route old runtimes (no `serve`) through the legacy `dashboard --no-open`.
backend.args = getBackendArgsForRuntime(backend)
const hermesCwd = resolveHermesCwd()
@ -8010,6 +8275,10 @@ async function startHermes() {
throw error
}
if (error instanceof FirstRunSetupResetError) {
throw error
}
const message = error instanceof Error ? error.message : String(error)
// Only latch LOCAL boot failures. A remote failure (lapsed session / mint
@ -8021,6 +8290,12 @@ async function startHermes() {
backendStartFailure = error instanceof Error ? error : new Error(message)
}
// A confirmed reauth rejection latches separately: it can't self-heal, and
// leaving it unlatched hides the overlay's "Sign in" button on every retry.
if (shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) })) {
remoteReauthFailure = error instanceof Error ? error : new Error(message)
}
updateBootProgress(
{
error: message,
@ -8824,16 +9099,9 @@ ipcMain.handle('hermes:bootstrap:reset', async () => {
await teardownPrimaryBackendAndWait()
bootstrapFailure = null
backendStartFailure = null
bootstrapState = {
active: false,
manifest: null,
stages: {},
error: null,
log: [],
startedAt: null,
completedAt: null,
unsupportedPlatform: null
}
remoteReauthFailure = null
getFirstRunSetupGate().resetForRetry()
resetBootstrapSnapshot()
return { ok: true }
})
@ -8854,10 +9122,18 @@ ipcMain.handle('hermes:bootstrap:repair', async () => {
bootstrapFailure = null
backendStartFailure = null
remoteReauthFailure = null
getFirstRunSetupGate().resetForRepair()
resetHermesConnection()
return { ok: true }
})
ipcMain.handle('hermes:bootstrap:continue-local', async () => {
rememberLog('[bootstrap] local install selected by renderer; continuing first-launch bootstrap')
continueFirstRunLocalBootstrap()
return { ok: true }
})
ipcMain.handle('hermes:bootstrap:cancel', async () => {
// Renderer's Cancel button during first-launch install. Abort the running
// install script (SIGTERM via the runner's abortSignal). runBootstrap
@ -8956,6 +9232,9 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) =>
})
_storeNativeTokens(baseUrl, tokens)
// Confirmed sign-in — release the reauth latch so the next
// startHermes() re-dials instead of replaying the stale rejection.
remoteReauthFailure = null
return { ok: true, baseUrl, connected: true }
} catch (error) {
@ -8972,7 +9251,16 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) =>
// Legacy embedded-webview cookie flow.
await openOauthLoginWindow(baseUrl)
return { ok: true, baseUrl, connected: await hasOauthSessionCookie(baseUrl) }
const connected = await hasOauthSessionCookie(baseUrl)
// Only a CONFIRMED sign-in releases the latch. A cancelled/closed login
// window must leave it set, or the overlay's "Sign in" button starts
// flickering again on the next retry.
if (connected) {
remoteReauthFailure = null
}
return { ok: true, baseUrl, connected }
})
ipcMain.handle('hermes:connection-config:oauth-logout', async (_event, rawUrl) => {
const baseUrl = rawUrl ? normalizeRemoteBaseUrl(rawUrl) : ''
@ -9036,6 +9324,18 @@ ipcMain.handle('hermes:connection-config:apply', async (_event, payload) => {
await applyConnectionChange({
cancelAndWait: value => sshBootstrapCoordinator.cancelAndWait(value),
isPrimary: !key || key === primaryProfileKey(),
rehomePrimary: () =>
rehomePrimaryConnection({
clearLocalBootstrapFailure: () => {
// A remote connection bypasses local runtime/bootstrap failures. Clear
// the local-install latch so unsupported/failure escape paths can re-home.
bootstrapFailure = null
},
mode: config.mode,
notifyConnectionApplied: sendConnectionApplied,
resumeFirstRunRemote: abandonFirstRunSetupChoiceForRemoteApply,
teardownPrimaryBackend: teardownPrimaryBackendAndWait
}),
scope,
sendApplied: sendConnectionApplied,
stopPool: stopPoolBackend,

View file

@ -1,7 +1,7 @@
/**
* Regression tests for electron/native-auth-decisions.ts the three pure
* decision seams behind the RFC 8252 native-app auth flow, each of which was a
* real runtime bug that the mocked flow tests could not catch.
* Regression tests for electron/native-auth-decisions.ts the pure decision
* seams behind the RFC 8252 native-app auth flow, each of which was a real
* runtime bug that the mocked flow tests could not catch.
*
* Run via the vitest `electron` project (electron/**\/*.test.ts).
*/
@ -10,7 +10,13 @@ import assert from 'node:assert/strict'
import { test } from 'vitest'
import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions'
import {
oauthGuardMayHardFail,
oauthSessionIsLive,
resolveJsonBody,
resolveOauthRestAuth,
resolveReadinessProbeAuth
} from './native-auth-decisions'
// --- 1. body encoding (guards the double-JSON.stringify 422) ---
@ -66,3 +72,61 @@ test('resolveOauthRestAuth falls back to cookie when there is no native token',
// Empty string is not a usable bearer — must fall back, not send "Bearer ".
assert.deepEqual(resolveOauthRestAuth(''), { kind: 'cookie' })
})
// --- 4. readiness-probe auth (guards the credential-free 401 boot loop) ---
test('resolveReadinessProbeAuth reuses the oauth bearer-vs-cookie choice', () => {
assert.deepEqual(resolveReadinessProbeAuth('oauth', 'native-at'), { kind: 'bearer', token: 'native-at' })
assert.deepEqual(resolveReadinessProbeAuth('oauth', null), { kind: 'cookie' })
assert.deepEqual(resolveReadinessProbeAuth('oauth', ''), { kind: 'cookie' })
})
test('resolveReadinessProbeAuth sends the session token for a token gateway', () => {
assert.deepEqual(resolveReadinessProbeAuth('token', null, 'session-token'), {
kind: 'token',
token: 'session-token'
})
assert.deepEqual(resolveReadinessProbeAuth('token', null, null), { kind: 'token', token: null })
})
test('resolveReadinessProbeAuth stays public for local and unknown modes', () => {
// A loopback backend has no gate; sending credentials it never issued is
// meaningless, and an unknown mode must not invent a credential.
assert.deepEqual(resolveReadinessProbeAuth('local', 'native-at', 'session-token'), { kind: 'public' })
assert.deepEqual(resolveReadinessProbeAuth(undefined, 'native-at', 'session-token'), { kind: 'public' })
assert.deepEqual(resolveReadinessProbeAuth('something-new', null, null), { kind: 'public' })
})
// --- 5. oauth guard vs password gateways (guards the false "not signed in") ---
test('oauthGuardMayHardFail is false only when EVERY provider is password-based', () => {
assert.equal(oauthGuardMayHardFail([{ name: 'basic', supportsPassword: true }]), false)
assert.equal(
oauthGuardMayHardFail([
{ name: 'basic', supportsPassword: true },
{ name: 'ldap', supportsPassword: true }
]),
false
)
})
test('oauthGuardMayHardFail keeps the strict guard for oauth and mixed deployments', () => {
assert.equal(oauthGuardMayHardFail([{ name: 'nous', supportsPassword: false }]), true)
assert.equal(
oauthGuardMayHardFail([
{ name: 'nous', supportsPassword: false },
{ name: 'basic', supportsPassword: true }
]),
true
)
})
test('oauthGuardMayHardFail keeps the strict guard when the list is unusable', () => {
// Backends predating /api/auth/providers, or an unreachable probe, must not
// silently weaken the guard.
assert.equal(oauthGuardMayHardFail([]), true)
assert.equal(oauthGuardMayHardFail(null), true)
assert.equal(oauthGuardMayHardFail(undefined), true)
assert.equal(oauthGuardMayHardFail('nonsense' as any), true)
assert.equal(oauthGuardMayHardFail([{ supportsPassword: true }]), true)
})

View file

@ -21,7 +21,17 @@
* native bearer when present, else the cookie partition. Cookie-only
* routing returns 401 no_cookie for a cookieless native session.
*
* All three are trivial once named; the value is the test that pins the
* 4. resolveReadinessProbeAuth the boot readiness probe must authenticate
* the same way the rest of the connection does. A credential-free probe
* against a gated gateway 401s forever; worse, it cannot tell a missing
* route from a rejected session (see backend-health.ts).
*
* 5. oauthGuardMayHardFail `auth_required: true` means "this gateway is
* gated", NOT "this gateway speaks OAuth". A password-provider gateway
* can satisfy neither the native-bearer nor the OAuth-partition-cookie
* check by design, so the pre-flight guard must not hard-fail it.
*
* All five are trivial once named; the value is the test that pins the
* contract so the god-file call sites can't drift back to the buggy shape.
*/
@ -60,3 +70,75 @@ export function resolveOauthRestAuth(nativeAccessToken: string | null | undefine
return { kind: 'cookie' }
}
export type ReadinessProbeAuth = OauthRestAuth | { kind: 'token'; token: string | null } | { kind: 'public' }
/**
* Decide how the boot readiness probe authenticates.
*
* The probe must present the SAME credentials the rest of the connection
* will use. A credential-free probe against a gated gateway 401s until the
* boot deadline even though the session is perfectly valid and because the
* dashboard auth gate runs ahead of the SPA catch-all, an unknown `/api/*`
* path answers 401 rather than 404, so the probe also cannot detect a backend
* that predates `/api/health`. Sending credentials is what lets a missing
* route surface as a real 404 (see `isMissingHealthEndpointError`).
*
* `oauth` reuses `resolveOauthRestAuth` so the probe and every other oauth
* REST call make the identical bearer-vs-cookie choice. `token` presents the
* connection's session token. `local` (and anything unrecognized) stays
* public: a loopback backend has no gate, and sending credentials it never
* issued would be meaningless.
*/
export function resolveReadinessProbeAuth(
authMode: string | null | undefined,
nativeAccessToken?: string | null,
connectionToken?: string | null
): ReadinessProbeAuth {
if (authMode === 'oauth') {
return resolveOauthRestAuth(nativeAccessToken)
}
if (authMode === 'token') {
return { kind: 'token', token: connectionToken ?? null }
}
return { kind: 'public' }
}
export interface AdvertisedAuthProvider {
name?: string
supportsPassword?: boolean
}
/**
* Whether the oauth pre-flight guard may hard-fail a connection for "not
* signed in".
*
* `authModeFromStatus` maps the gateway's `auth_required: true` onto
* `'oauth'`, but that flag only means the dashboard is GATED it says
* nothing about how you authenticate. A gateway whose providers are all
* username/password cannot satisfy the guard's checks by construction:
* `start_login` raises NotImplementedError, `/auth/native/authorize` rejects
* password providers, and its cookies are set by a plain password-login POST
* rather than the `/auth/callback` redirect the OAuth partition is primed
* for. Hard-failing there rejects a live session one line before the
* ws-ticket mint that would have succeeded against that very partition.
*
* Returns false only when EVERY advertised provider is password-based. An
* unknown or empty list keeps the strict guard, so backends that predate
* `/api/auth/providers` are unaffected.
*/
export function oauthGuardMayHardFail(providers: AdvertisedAuthProvider[] | null | undefined): boolean {
if (!Array.isArray(providers) || providers.length === 0) {
return true
}
const named = providers.filter(provider => provider && typeof provider === 'object' && provider.name)
if (named.length === 0) {
return true
}
return !named.every(provider => provider.supportsPassword)
}

View file

@ -237,6 +237,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
// current snapshot via getBootstrapState() to recover after a devtools
// reload mid-bootstrap.
getBootstrapState: () => ipcRenderer.invoke('hermes:bootstrap:get'),
continueBootstrapLocal: () => ipcRenderer.invoke('hermes:bootstrap:continue-local'),
resetBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:reset'),
repairBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:repair'),
cancelBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:cancel'),

View file

@ -0,0 +1,110 @@
import assert from 'node:assert/strict'
import { test, vi } from 'vitest'
import { createFirstRunSetupGate } from './first-run-setup-gate'
import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup'
const bootstrapBackend = {
activeRoot: '/tmp/hermes-home/hermes-agent',
kind: 'bootstrap-needed',
platform: 'linux'
}
function startupOptions(overrides: Record<string, unknown> = {}) {
return {
connectRemote: vi.fn(async remote => ({ baseUrl: remote.baseUrl, mode: 'remote' as const })),
ensureLocalRuntime: vi.fn(async backend => ({ ...backend, command: 'hermes' })),
prepareLocalBackend: vi.fn(async () => bootstrapBackend),
resolveRemote: vi.fn(async () => null),
waitForDecision: vi.fn(async () => 'continue-local' as const),
waitForLocalStart: vi.fn(async () => {}),
...overrides
}
}
test('remote apply re-resolves the saved connection without ensuring a local runtime', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
let configuredRemote: typeof savedRemote | null = null
const options = startupOptions({
resolveRemote: vi.fn(async () => configuredRemote),
waitForDecision: gate.wait
})
const pending = runPrimaryBackendStartup(options)
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
configuredRemote = savedRemote
assert.equal(gate.abandonForRemoteApply(), true)
assert.deepEqual(await pending, {
kind: 'remote',
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
})
assert.deepEqual(options.resolveRemote.mock.calls, [[], []])
assert.deepEqual(options.connectRemote.mock.calls, [[savedRemote]])
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
})
test('an already-saved remote bypasses every local startup step', async () => {
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
const options = startupOptions({ resolveRemote: vi.fn(async () => savedRemote) })
assert.deepEqual(await runPrimaryBackendStartup(options), {
kind: 'remote',
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
})
assert.equal(options.waitForLocalStart.mock.calls.length, 0)
assert.equal(options.prepareLocalBackend.mock.calls.length, 0)
assert.equal(options.waitForDecision.mock.calls.length, 0)
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
})
test('remote apply fails clearly when no saved remote can be resolved', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const options = startupOptions({ waitForDecision: gate.wait })
const pending = runPrimaryBackendStartup(options)
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
gate.abandonForRemoteApply()
await assert.rejects(pending, /without a saved remote backend/)
assert.equal(options.connectRemote.mock.calls.length, 0)
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
})
test('continue local waits for update exclusion and ensures the prepared runtime exactly once', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const runtimeBackend = { ...bootstrapBackend, command: 'hermes' }
const options = startupOptions({
ensureLocalRuntime: vi.fn(async () => runtimeBackend),
waitForDecision: gate.wait
})
const pending = runPrimaryBackendStartup(options)
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
gate.continueLocal()
assert.deepEqual(await pending, { kind: 'local', backend: runtimeBackend })
assert.deepEqual(options.waitForLocalStart.mock.calls, [[]])
assert.deepEqual(options.prepareLocalBackend.mock.calls, [[]])
assert.deepEqual(options.ensureLocalRuntime.mock.calls, [[bootstrapBackend]])
assert.deepEqual(options.resolveRemote.mock.calls, [[]])
})
test('reset rejects with a typed error and never enters either backend', async () => {
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
const options = startupOptions({ waitForDecision: gate.wait })
const pending = runPrimaryBackendStartup(options)
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
gate.resetForRetry()
await assert.rejects(pending, error => error instanceof FirstRunSetupResetError && error.firstRunSetupReset)
assert.equal(options.connectRemote.mock.calls.length, 0)
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
})

View file

@ -0,0 +1,66 @@
import type { FirstRunSetupDecision } from './first-run-setup-gate'
export interface PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection> {
connectRemote: (remote: Remote) => Promise<Connection>
ensureLocalRuntime: (backend: Backend) => Promise<RuntimeBackend>
prepareLocalBackend: () => Backend | Promise<Backend>
resolveRemote: () => Promise<Remote | null>
waitForDecision: (backend: Backend) => Promise<FirstRunSetupDecision>
waitForLocalStart: () => Promise<unknown>
}
export type PrimaryBackendStartupResult<RuntimeBackend, Connection> =
| { kind: 'local'; backend: RuntimeBackend }
| { kind: 'remote'; connection: Connection }
export class FirstRunSetupResetError extends Error {
readonly firstRunSetupReset = true
constructor() {
super('First-run setup was reset before a choice completed.')
this.name = 'FirstRunSetupResetError'
}
}
// Owns the production startHermes path up to the local process spawn. Keeping
// the full ordering here makes the first-run remote boundary executable in a
// test: an already-saved remote wins immediately; otherwise update exclusion
// and local backend resolution happen before the setup gate, and a remote Apply
// re-resolves persisted config without ever entering ensureRuntime/bootstrap.
export async function runPrimaryBackendStartup<Backend, RuntimeBackend, Remote, Connection>({
connectRemote,
ensureLocalRuntime,
prepareLocalBackend,
resolveRemote,
waitForDecision,
waitForLocalStart
}: PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection>): Promise<
PrimaryBackendStartupResult<RuntimeBackend, Connection>
> {
const savedRemote = await resolveRemote()
if (savedRemote) {
return { kind: 'remote', connection: await connectRemote(savedRemote) }
}
await waitForLocalStart()
const backend = await prepareLocalBackend()
const decision = await waitForDecision(backend)
if (decision === 'remote-applied') {
const appliedRemote = await resolveRemote()
if (!appliedRemote) {
throw new Error('First-run remote setup completed without a saved remote backend.')
}
return { kind: 'remote', connection: await connectRemote(appliedRemote) }
}
if (decision === 'reset') {
throw new FirstRunSetupResetError()
}
return { kind: 'local', backend: await ensureLocalRuntime(backend) }
}

View file

@ -0,0 +1,35 @@
export interface PrimaryConnectionRehomeOptions {
clearLocalBootstrapFailure: () => void
mode: string
notifyConnectionApplied: () => void
resumeFirstRunRemote: () => boolean
teardownPrimaryBackend: (options: { soft: boolean }) => Promise<void>
}
// Production seam shared by the connection-config IPC handler and the
// first-run integration test. A remote apply that resumes the active setup
// gate must keep that connection attempt alive; ordinary mode changes tear the
// current backend down before the renderer is told to reconnect.
export async function rehomePrimaryConnection({
clearLocalBootstrapFailure,
mode,
notifyConnectionApplied,
resumeFirstRunRemote,
teardownPrimaryBackend
}: PrimaryConnectionRehomeOptions): Promise<{ resumedFirstRunRemote: boolean }> {
let resumedFirstRunRemote = false
if (mode === 'remote') {
resumedFirstRunRemote = resumeFirstRunRemote()
clearLocalBootstrapFailure()
}
if (resumedFirstRunRemote) {
return { resumedFirstRunRemote: true }
}
await teardownPrimaryBackend({ soft: true })
notifyConnectionApplied()
return { resumedFirstRunRemote: false }
}

View file

@ -8,7 +8,8 @@ import {
needsExecBit,
spawnHelperCandidates,
type SpawnHelperFs,
withExecBits
withExecBits,
writableNodePtyRoot
} from './spawn-helper-perms'
interface FakeFile {
@ -56,6 +57,30 @@ function fakeFs(
}
}
test('rewrites an archived node-pty root to the matching unpacked tree exactly once', () => {
assert.equal(
writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty'),
'/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'
)
assert.equal(
writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'),
'/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'
)
})
test('uses the unpacked helper when resolution reports an app.asar node-pty root', () => {
const archivedRoot = '/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty'
const unpackedRoot = writableNodePtyRoot(archivedRoot)
const helper = join(unpackedRoot, 'prebuilds', 'darwin-arm64', 'spawn-helper')
const fs = fakeFs({ [helper]: { mode: 0o644 } }, { [join(unpackedRoot, 'prebuilds')]: ['darwin-arm64'] })
const result = ensureSpawnHelperExecutable(archivedRoot, fs)
assert.deepEqual(result.fixed, [helper])
assert.deepEqual(result.errors, [])
assert.deepEqual(fs.chmods, [{ path: helper, mode: 0o755 }])
})
test('needsExecBit / withExecBits treat any missing exec bit as non-executable', () => {
assert.equal(needsExecBit(0o644), true)
assert.equal(needsExecBit(0o755), false)

View file

@ -18,6 +18,14 @@ import { join } from 'node:path'
const EXEC_BITS = 0o111
// Electron exposes module paths inside app.asar even when electron-builder has
// unpacked the native payload beside it. `stat` can read an archived path, but
// chmod cannot mutate it (ENOTDIR). Native node-pty helpers belong in the
// writable app.asar.unpacked tree; leave an already-unpacked path unchanged.
export function writableNodePtyRoot(nodePtyRoot: string): string {
return nodePtyRoot.replace(/app\.asar(?!\.unpacked)/, 'app.asar.unpacked')
}
export interface SpawnHelperFs {
existsSync(path: string): boolean
readdirSync(path: string): string[]
@ -81,8 +89,9 @@ export function ensureSpawnHelperExecutable(
fs: SpawnHelperFs = defaultFs
): EnsureSpawnHelperResult {
const result: EnsureSpawnHelperResult = { fixed: [], errors: [] }
const writableRoot = writableNodePtyRoot(nodePtyRoot)
for (const path of spawnHelperCandidates(nodePtyRoot, fs)) {
for (const path of spawnHelperCandidates(writableRoot, fs)) {
if (!fs.existsSync(path)) {
continue
}

View file

@ -39,5 +39,43 @@ export default [
rules: {
'no-restricted-globals': ['warn', 'document']
}
},
{
// Ban mirroring reactive values into refs via useEffect — the "atom-mirrored
// ref" antipattern. A ref synced from a nanostores atom via useEffect lags the
// atom by one render, which creates stale-read bugs in callbacks that read the
// ref (cancelRun sent session.interrupt to the wrong session; steerPrompt,
// restoreToMessage, editMessage all had closure-priority stale reads). The fix
// is to read $atom.get() directly in callbacks instead. This rule catches the
// mirroring effect at lint time so the pattern can't reappear. Legitimate
// non-atom ref writes inside useEffect (DOM instance refs, mount flags, request
// tokens, prop mirrors) get an eslint-disable-next-line with a comment.
files: ['src/**/*.{ts,tsx}'],
rules: {
'no-restricted-syntax': [
'error',
{
// useEffect(() => { someRef.current = value }, [value])
selector:
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="AssignmentExpression"][body.left.type="MemberExpression"][body.left.property.name="current"]',
message:
'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
},
{
// useEffect(() => { someRef.current = value; ... }, [value])
selector:
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(AssignmentExpression[left.type="MemberExpression"][left.property.name="current"])',
message:
'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
},
{
// useEffect(() => { setMutableRef(ref, value) }, [value])
selector:
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(CallExpression[callee.name="setMutableRef"])',
message:
'Do not mirror reactive values into refs via useEffect (setMutableRef included). Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
}
]
}
}
]

View file

@ -111,7 +111,7 @@
"motion": "^12.38.0",
"nanostores": "^1.3.0",
"node-pty": "1.1.0",
"radix-ui": "^1.4.3",
"radix-ui": "^1.6.5",
"react": "^19.2.5",
"react-arborist": "^3.5.0",
"react-dnd-html5-backend": "^14.0.3",

View file

@ -57,7 +57,8 @@
* - electronPlatformName: 'win32' | 'darwin' | 'linux'
* - arch: Arch enum (0=ia32, 1=x64, 2=armv7l, 3=arm64, 4=universal)
*/
import { existsSync, rmSync } from 'node:fs'
import { existsSync, rmSync, renameSync } from 'node:fs'
import path from 'node:path'
import { Arch } from 'electron-builder'
import { stageNodePty } from './stage-native-deps.mjs'
@ -75,10 +76,52 @@ export function cleanStaleAppOutDir(appOutDir) {
return true
}
/**
* Windows rollback material (#69179): before wiping the previous unpacked
* tree, preserve it as `<appOutDir>.bak` but ONLY when it holds the product
* exe (i.e. it is a previously-working build, not the corrupted partial state
* cleanStaleAppOutDir exists to remove). If the fresh pack then produces a
* Hermes.exe that Windows can't load (truncated PE from a corrupt cached
* Electron zip, wrong arch), the updater's integrity gate in
* `hermes desktop --build-only` (hermes_cli/main.py
* `_ensure_desktop_exe_launchable`) restores this .bak instead of leaving the
* user with "This app can't run on your computer".
*
* Returns true when the tree was preserved (appOutDir no longer exists), false
* when there was nothing worth preserving (caller falls through to the wipe).
* A rename failure (AV holding a handle) also returns false the wipe is the
* safe fallback and matches pre-#69179 behavior exactly.
*/
export function preserveRollbackBackup(appOutDir, productExeName = 'Hermes.exe') {
if (!appOutDir || typeof appOutDir !== 'string' || !existsSync(appOutDir)) {
return false
}
if (!existsSync(path.join(appOutDir, productExeName))) {
// Partial/corrupt tree (interrupted prior pack) — not rollback material.
return false
}
const backupDir = `${appOutDir}.bak`
try {
rmSync(backupDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 })
renameSync(appOutDir, backupDir)
return true
} catch {
return false
}
}
export default async function beforePack(context) {
const appOutDir = context && context.appOutDir
const platformName = context && context.electronPlatformName
try {
if (cleanStaleAppOutDir(appOutDir)) {
// Windows: keep the previous working build as rollback material for the
// post-build integrity gate (#69179) instead of destroying it. Falls
// through to the plain wipe when the old tree is partial/corrupt or the
// rename fails.
const productExe = `${(context && context.packager?.appInfo?.productFilename) || 'Hermes'}.exe`
if (platformName === 'win32' && preserveRollbackBackup(appOutDir, productExe)) {
console.log(`[before-pack] preserved previous unpacked dir for rollback: ${appOutDir}.bak`)
} else if (cleanStaleAppOutDir(appOutDir)) {
console.log(`[before-pack] removed stale unpacked dir before staging: ${appOutDir}`)
}
} catch (err) {

View file

@ -4,7 +4,7 @@ import os from 'node:os'
import path from 'node:path'
import { test } from 'vitest'
import beforePack, { cleanStaleAppOutDir } from '../scripts/before-pack.mjs'
import beforePack, { cleanStaleAppOutDir, preserveRollbackBackup } from '../scripts/before-pack.mjs'
test('cleanStaleAppOutDir removes a populated unpacked directory', () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
@ -50,3 +50,106 @@ test('beforePack default export resolves even when cleanup throws', async () =>
// remove; the contract under test is that the hook never rejects.
await assert.doesNotReject(beforePack({ appOutDir: '', electronPlatformName: 'linux' }))
})
// ─── Windows rollback preservation (#69179) ────────────────────────────────
test('preserveRollbackBackup moves a working build to .bak', () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
try {
const appOutDir = path.join(tempRoot, 'win-unpacked')
fs.mkdirSync(appOutDir, { recursive: true })
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-old-build', 'utf8')
fs.writeFileSync(path.join(appOutDir, 'resources.pak'), 'x', 'utf8')
const preserved = preserveRollbackBackup(appOutDir, 'Hermes.exe')
assert.equal(preserved, true)
// Original slot vacated so electron-builder stages into a clean tree...
assert.equal(fs.existsSync(appOutDir), false)
// ...and the previous working build is intact under .bak for rollback.
assert.equal(
fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'),
'MZ-old-build'
)
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true })
}
})
test('preserveRollbackBackup replaces a stale .bak from an older update', () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
try {
const appOutDir = path.join(tempRoot, 'win-unpacked')
fs.mkdirSync(appOutDir, { recursive: true })
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'current', 'utf8')
fs.mkdirSync(`${appOutDir}.bak`, { recursive: true })
fs.writeFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'two-updates-ago', 'utf8')
assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), true)
assert.equal(fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'), 'current')
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true })
}
})
test('preserveRollbackBackup refuses a partial tree missing the product exe', () => {
// The corrupted partial state (interrupted prior pack) must NOT become
// rollback material — it is exactly what cleanStaleAppOutDir exists to wipe.
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
try {
const appOutDir = path.join(tempRoot, 'win-unpacked')
fs.mkdirSync(appOutDir, { recursive: true })
fs.writeFileSync(path.join(appOutDir, 'LICENSE.electron.txt'), 'x', 'utf8')
assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), false)
// Tree untouched; the caller's wipe path handles it.
assert.equal(fs.existsSync(appOutDir), true)
assert.equal(fs.existsSync(`${appOutDir}.bak`), false)
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true })
}
})
test('preserveRollbackBackup ignores missing or invalid input', () => {
assert.equal(preserveRollbackBackup(''), false)
assert.equal(preserveRollbackBackup(undefined), false)
assert.equal(preserveRollbackBackup(null), false)
assert.equal(preserveRollbackBackup(path.join(os.tmpdir(), 'does-not-exist-xyz')), false)
})
test('beforePack on win32 preserves the previous build instead of wiping it', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
try {
const appOutDir = path.join(tempRoot, 'win-unpacked')
fs.mkdirSync(appOutDir, { recursive: true })
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-working', 'utf8')
// No packager info in the context → default 'Hermes.exe' product name.
// node-pty staging is skipped because arch is not a number here.
await beforePack({ appOutDir, electronPlatformName: 'win32' })
assert.equal(fs.existsSync(appOutDir), false)
assert.equal(
fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'),
'MZ-working'
)
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true })
}
})
test('beforePack on linux keeps the plain wipe (no .bak)', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
try {
const appOutDir = path.join(tempRoot, 'linux-unpacked')
fs.mkdirSync(appOutDir, { recursive: true })
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'x', 'utf8')
await beforePack({ appOutDir, electronPlatformName: 'linux' })
assert.equal(fs.existsSync(appOutDir), false)
assert.equal(fs.existsSync(`${appOutDir}.bak`), false)
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true })
}
})

View file

@ -4,16 +4,20 @@
import coldStart from './cold-start.mjs'
import firstToken from './first-token.mjs'
import keystroke from './keystroke.mjs'
import multitab from './multitab.mjs'
import profileSwitch from './profile-switch.mjs'
import sessionSwitch from './session-switch.mjs'
import stream from './stream.mjs'
import streamHistory from './stream-history.mjs'
import submit from './submit.mjs'
import transcript from './transcript.mjs'
export const SCENARIOS = {
[stream.name]: stream,
[streamHistory.name]: streamHistory,
[keystroke.name]: keystroke,
[transcript.name]: transcript,
[multitab.name]: multitab,
[coldStart.name]: coldStart,
[firstToken.name]: firstToken,
[submit.name]: submit,

View file

@ -0,0 +1,246 @@
// Multi-tab working sessions: N session tiles stacked as tabs in the main
// zone, EVERY tab mounted (keep-alive), all streaming concurrently — the
// "5 tabs doing PR review" workload. Measures frame pacing + longtasks while
// the whole stack streams, which is where multitab renderers crawl.
//
// Drives the real pipeline synthetically (no backend, no credits):
// publishSessionState per session per flush — exactly what the gateway's
// delta flush does — via the __HERMES_SESSION_TILES__ hook.
//
// node scripts/perf/run.mjs multitab --spawn [--tiles 5] [--tokens 240]
import { sleep } from '../lib/cdp.mjs'
import { frameHistogram, percentile } from '../lib/stats.mjs'
// Same recorder pattern as stream.mjs (generation-guarded rAF + longtasks).
const RECORDERS = `
(() => {
window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1
const ftGen = window.__FT_GEN__
window.__FT__ = { times: [], stop: false }
let last = performance.now()
const tick = () => {
if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return
const now = performance.now()
window.__FT__.times.push(now - last)
last = now
requestAnimationFrame(tick)
}
requestAnimationFrame(tick)
window.__LT__ = { entries: [], stop: false }
try {
const po = new PerformanceObserver((list) => {
if (window.__LT__.stop) return
for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime })
})
po.observe({ entryTypes: ['longtask'] })
window.__LT__.po = po
} catch {}
return 'armed'
})()
`
const COLLECT = `
(() => {
window.__FT__.stop = true
window.__LT__.stop = true
try { window.__LT__.po && window.__LT__.po.disconnect() } catch {}
return JSON.stringify({ frames: window.__FT__.times, longtasks: window.__LT__.entries })
})()
`
/** Page-side setup: open `tiles` session tiles stacked into the main zone,
* bind fake runtime ids, and seed each with a realistic transcript. */
const setup = (tiles, seedTurns, streamSeed) => `
(() => {
const hook = window.__HERMES_SESSION_TILES__
if (!hook) return 'no-hook'
const turn = (sid, i) => ([
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff in module ' + i + ' handle the error path?' }] },
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
parts: [{ type: 'text', text: [
'## Finding ' + i, '',
'The handler swallows the rejection. Key points for hunk \\\`' + i + '\\\`:', '',
'- The catch block drops the original error.',
'- Retries are unbounded — see [the loop](https://example.com/loop).', '',
'\\\`\\\`\\\`ts',
'async function retry' + i + '(fn: () => Promise<void>) {',
' for (;;) { try { return await fn() } catch {} }',
'}',
'\\\`\\\`\\\`', '',
'| path | covered |', '|---|---|', '| happy | yes |', '| error | no |', ''
].join('\\n') }] }
])
const state = (sid, rid) => {
const messages = []
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
// Streaming tail the driver grows (--code seeds an open fence).
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
parts: [{ type: 'text', text: ${JSON.stringify(streamSeed)} }] })
return {
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
needsInput: false, turnStartedAt: Date.now(), usage: null
}
}
window.__MT__ = { ids: [], timer: null }
for (let n = 1; n <= ${tiles}; n++) {
const sid = 'perf-tile-' + n
const rid = 'perf-rt-' + n
window.__MT__.ids.push({ sid, rid })
hook.open(sid, 'center')
hook.patch(sid, { runtimeId: rid })
hook.publish(rid, state(sid, rid))
}
return 'ok'
})()
`
// Activate every tab once so keep-alive mounts the full stack (lazy mount:
// a never-activated tab stays unmounted, which would understate the cost).
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
/** Page-side driver: grow every tile's streaming tail by `chunk` each
* `intervalMs`, through the same publish path the gateway flush uses. */
const drive = (chunk, intervalMs, totalTokens) => `
(() => {
const hook = window.__HERMES_SESSION_TILES__
let pushed = 0
const tick = () => {
const states = hook.states()
for (const { rid } of window.__MT__.ids) {
const prev = states[rid]
if (!prev) continue
const messages = prev.messages.map(m => {
if (m.id !== prev.streamId) return m
const head = m.parts.slice(0, -1)
const last = m.parts[m.parts.length - 1]
return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] }
})
hook.publish(rid, { ...prev, messages })
}
pushed += 1
if (pushed < ${totalTokens}) window.__MT__.timer = setTimeout(tick, ${intervalMs})
else window.__MT__.done = true
}
window.__MT__.timer = setTimeout(tick, ${intervalMs})
return 'driving'
})()
`
const CLEANUP = `
(() => {
if (window.__MT__) {
clearTimeout(window.__MT__.timer)
for (const { sid, rid } of window.__MT__.ids) {
window.__HERMES_SESSION_TILES__.publish(rid, {
...window.__HERMES_SESSION_TILES__.states()[rid], busy: false, streamId: null
})
window.__HERMES_SESSION_TILES__.close(sid)
}
window.__MT__ = null
}
return 'cleaned'
})()
`
export default {
name: 'multitab',
tier: 'ci',
description: 'N mounted session-tile tabs all streaming: frame pacing + longtasks.',
async run(cdp, opts = {}) {
const tiles = Number(opts.tiles ?? 5)
const seedTurns = Number(opts.turns ?? 20)
const tokens = Number(opts.tokens ?? 240)
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
const intervalMs = Number(opts.intervalMs ?? 33)
// --code: every tile grows ONE giant fenced code block with no settle
// boundaries — what a coding agent streams. The block re-parses and
// re-renders fully every flush (block memoization can't settle it), the
// documented worst case and the "5 tabs all coding" crawl.
const chunk = opts.code
? ' const value = await resolve(ctx, { retry: true }) // step\n'
: (opts.chunk ?? 'A streamed review sentence with **bold**, `code`, and ordinary prose.\n\n')
const streamSeed = opts.code ? '```ts\n' : ''
await cdp.send('Runtime.enable')
const ok = await cdp.eval(setup(tiles, seedTurns, streamSeed))
if (ok !== 'ok') {
throw new Error(`multitab setup failed (${ok}) — dev hooks missing? (needs a dev/probe renderer)`)
}
// Mount every tab (keep-alive mounts on first activation), then settle.
for (let n = 1; n <= tiles; n++) {
await cdp.eval(reveal(`perf-tile-${n}`))
await sleep(350)
}
await sleep(1000)
await cdp.eval(RECORDERS)
await cdp.eval(drive(chunk, intervalMs, tokens))
await sleep(tokens * intervalMs + 1500)
const data = JSON.parse(await cdp.eval(COLLECT))
await cdp.eval(CLEANUP)
// Drop the first 500ms (recorder install + settle).
const frames = []
let acc = 0
for (const f of data.frames) {
acc += f
if (acc >= 500) {
frames.push(f)
}
}
const ltDurations = data.longtasks.map(e => e.duration)
const windowS = frames.reduce((a, b) => a + b, 0) / 1000
// The felt numbers: sustained fps over the window, and the fps of the
// worst 1-second slice (a 333ms frame IS "3fps" even if the average looks
// fine). Worst slice = max summed frame time in any sliding 1s window.
const avgFps = windowS ? frames.length / windowS : 0
let worstFps = avgFps
for (let i = 0, j = 0, sum = 0; j < frames.length; j++) {
sum += frames[j]
while (sum > 1000) {
sum -= frames[i++]
}
// Only a window that actually spans ~1s counts; short prefixes don't.
if (sum >= 900) {
worstFps = Math.min(worstFps, ((j - i + 1) / sum) * 1000)
}
}
return {
metrics: {
longtasks_n: data.longtasks.length,
longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10,
frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10,
frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10,
slow_frames_33: frames.filter(f => f > 33).length
},
detail: {
tiles,
code: Boolean(opts.code),
windowS: Math.round(windowS * 10) / 10,
avgFps: Math.round(avgFps * 10) / 10,
worstSecondFps: Math.round(worstFps * 10) / 10,
frameHistogram: frameHistogram(frames)
}
}
}
}

View file

@ -0,0 +1,18 @@
// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but
// the history is mounted and allowed to settle before the recorders start, so
// what it captures is the per-delta cost that scales with transcript length —
// the regression reported in #69120.
//
// Report-only (tier: manual): the number depends on how much history the host
// can mount, so it is not gated against the committed baseline.
import stream from './stream.mjs'
export default {
name: 'stream-history',
tier: 'manual',
description: 'Streaming cost with a long settled transcript already mounted.',
run(cdp, opts = {}) {
return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) })
}
}

View file

@ -70,7 +70,7 @@ const COLLECT = `
})()
`
function analyze(data, warmupMs) {
function analyze(data, warmupMs, extra = {}) {
// Drop warm-up frames (recorder installs before the stream starts).
const frames = []
let acc = 0
@ -102,6 +102,7 @@ function analyze(data, warmupMs) {
intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10
},
detail: {
...extra,
windowS: Math.round(windowS * 10) / 10,
avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0,
frameHistogram: frameHistogram(frames),
@ -128,13 +129,36 @@ export default {
// noise unrelated to render cost).
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
const real = Boolean(opts.real)
const historyTurns = Number(opts.historyTurns ?? 0)
const historySettleMs = Number(opts.historySettleMs ?? 1500)
await cdp.send('Runtime.enable')
// Mount the settled history BEFORE the recorders start, so the measurement
// window contains only streaming work — not the one-off mount cost.
if (historyTurns > 0) {
if (real) {
throw new Error('--historyTurns is only supported by the synthetic stream path')
}
await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`)
await sleep(historySettleMs)
const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()'))
const expected = historyTurns * 2
if (mounted !== expected) {
throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`)
}
}
await cdp.eval(RECORDERS)
if (real) {
// Backend path: fire a real prompt and wait for the stream to appear.
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
const baseCount = await cdp.eval(
`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`
)
await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 })
await cdp.eval(`(() => {
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
@ -186,6 +210,6 @@ export default {
await cdp.eval('window.__PERF_DRIVE__.reset()')
}
return analyze(data, real ? 0 : 500)
return analyze(data, real ? 0 : 500, { historyTurns })
}
}

View file

@ -0,0 +1,80 @@
// Model picker open latency probe: click the composer model pill, time until
// the dropdown/dialog content paints, repeat. Run with --cpuprofile via the
// harness runner once promoted; standalone for iteration.
// node scripts/perf/probe-model-picker.mjs [--port 9222] [--rounds 5]
import { CDP } from './perf/lib/cdp.mjs'
const args = process.argv.slice(2)
const flag = name => {
const i = args.indexOf(`--${name}`)
return i >= 0 ? args[i + 1] : undefined
}
const port = Number(flag('port') ?? 9222)
const rounds = Number(flag('rounds') ?? 5)
const sleep = ms => new Promise(r => setTimeout(r, ms))
const cdp = await CDP.connect({ port })
await cdp.send('Runtime.enable')
// Find the pill (dropdown path) — the composer model selector button.
const PILL = `(() => {
const btns = [...document.querySelectorAll('button[aria-label]')]
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || '') && b.closest('[data-slot]'))
return pill ? (pill.getAttribute('aria-label') || 'found') : null
})()`
console.log('pill:', await cdp.eval(PILL))
const MEASURE = `
(async () => {
const btns = [...document.querySelectorAll('button[aria-label]')]
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || ''))
if (!pill) return JSON.stringify({ error: 'no pill' })
const t0 = performance.now()
pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true }))
pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true }))
pill.click()
// Wait for menu/dialog content to exist AND paint (double rAF after found).
const found = await new Promise(resolve => {
const deadline = performance.now() + 5000
const check = () => {
const menu = document.querySelector('[role="menu"], [role="dialog"] [cmdk-list]')
if (menu && menu.childElementCount > 0) {
requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now())))
return
}
if (performance.now() > deadline) { resolve(null); return }
requestAnimationFrame(check)
}
check()
})
const openMs = found ? found - t0 : null
const rows = document.querySelectorAll('[role="menu"] [role="menuitem"], [cmdk-item]').length
// Close: Escape.
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
const menu = document.querySelector('[role="menu"], [role="dialog"]')
menu?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
await new Promise(r => setTimeout(r, 300))
return JSON.stringify({ openMs, rows })
})()
`
const samples = []
for (let i = 0; i < rounds; i++) {
const raw = await cdp.eval(MEASURE, { awaitPromise: true })
const r = JSON.parse(raw)
console.log(`round ${i}:`, r)
if (typeof r.openMs === 'number') samples.push(r.openMs)
await sleep(500)
}
samples.sort((a, b) => a - b)
console.log('\nopen latency ms — min/median/max:',
Math.round(samples[0]), '/', Math.round(samples[Math.floor(samples.length / 2)]), '/', Math.round(samples.at(-1)))
cdp.close()

View file

@ -0,0 +1,60 @@
// CPU-profile one model-picker open.
// node scripts/profile-model-picker.mjs [--port 9222]
import { writeFileSync } from 'node:fs'
import { CDP } from './perf/lib/cdp.mjs'
import { cpuProfileTopSelf } from './perf/lib/stats.mjs'
const port = Number(process.argv.includes('--port') ? process.argv[process.argv.indexOf('--port') + 1] : 9222)
const cdp = await CDP.connect({ port })
await cdp.send('Runtime.enable')
await cdp.send('Profiler.enable')
await cdp.send('Profiler.setSamplingInterval', { interval: 100 })
const OPEN = `
(async () => {
// Reset: close any open menu first.
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
await new Promise(r => setTimeout(r, 250))
const btns = [...document.querySelectorAll('button[aria-label]')]
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || ''))
if (!pill) return -1
const t0 = performance.now()
pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true }))
pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true }))
pill.click()
const found = await new Promise(resolve => {
const deadline = performance.now() + 8000
const check = () => {
const menu = document.querySelector('[role="menu"]')
if (menu && menu.childElementCount > 0) {
requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now())))
return
}
if (performance.now() > deadline) { resolve(-1); return }
requestAnimationFrame(check)
}
check()
})
return found < 0 ? -1 : found - t0
})()
`
await cdp.send('Profiler.start')
const openMs = await cdp.eval(OPEN)
const { profile } = await cdp.send('Profiler.stop')
console.log('openMs:', Math.round(openMs))
const out = `/tmp/model-picker-open.cpuprofile`
writeFileSync(out, JSON.stringify(profile))
console.log('wrote', out)
console.log('top self-time (ms):')
for (const r of cpuProfileTopSelf(profile, 20)) {
console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(44)} ${r.url.split('/').slice(-2).join('/')}:${r.line}`)
}
// Close the menu again.
await cdp.eval(`document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))`)
cdp.close()

View file

@ -50,6 +50,9 @@ export function slashChipKindForItem(item: Unstable_TriggerItem): SlashChipKind
return 'command'
}
/** True for a skill completion — the only kind offered mid-message. */
export const isSkillItem = (item: Unstable_TriggerItem) => slashChipKindForItem(item) === 'skill'
/** A `/` query is at its arg stage once it's past the command name. */
export const slashArgStage = (query: string) => query.includes(' ')

View file

@ -0,0 +1,50 @@
import { afterEach, describe, expect, it } from 'vitest'
import { blurComposerInput } from './focus'
import { RICH_INPUT_SLOT } from './rich-editor'
/**
* Inactive tabs keep their composer mounted, so an unscoped lookup can blur a
* background input and leave the one the user is typing in focused.
*/
/** A composer input inside its own pane layer, hidden or not. */
function mountInput(hidden = false) {
const layer = document.createElement('div')
const input = document.createElement('div')
input.dataset.slot = RICH_INPUT_SLOT
input.tabIndex = 0
layer.toggleAttribute('data-pane-hidden', hidden)
layer.append(input)
document.body.append(layer)
return input
}
afterEach(() => {
document.body.innerHTML = ''
})
describe('blurComposerInput', () => {
it('blurs the foreground composer while a hidden tab matches first', () => {
const background = mountInput(true)
const foreground = mountInput()
foreground.focus()
blurComposerInput()
expect(document.activeElement).not.toBe(foreground)
expect(document.activeElement).not.toBe(background)
})
it('leaves focus alone when the composer does not hold it', () => {
const outside = document.createElement('button')
document.body.append(outside)
mountInput()
outside.focus()
blurComposerInput()
expect(document.activeElement).toBe(outside)
})
})

View file

@ -10,6 +10,8 @@
* steal focus from the composer effect.
*/
import { queryVisible } from '@/components/pane-shell/pane-visibility'
import type { InlineRefInput } from './inline-refs'
import { RICH_INPUT_SLOT } from './rich-editor'
@ -175,9 +177,11 @@ export const focusComposerInput = (el: HTMLElement | null) => {
window.setTimeout(focus, 0)
}
/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). */
/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.).
* Skips inactive tabs they stay mounted, so an unscoped lookup can land on a
* background composer and leave the visible one focused. */
export const blurComposerInput = () => {
const el = document.querySelector(`[data-slot="${RICH_INPUT_SLOT}"]`) as HTMLElement | null
const el = queryVisible(`[data-slot="${RICH_INPUT_SLOT}"]`)
if (el && document.activeElement === el) {
el.blur()

View file

@ -0,0 +1,130 @@
import { act, cleanup, render } from '@testing-library/react'
import { useLayoutEffect } from 'react'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { type ComposerAttachment, mainComposerScope, stashSessionDraft } from '@/store/composer'
import type { QueueEditState } from '../composer-utils'
import { useComposerDraft } from './use-composer-draft'
const mockComposerApi = { setText: vi.fn() }
vi.mock('@assistant-ui/react', () => ({
useAui: () => ({ composer: () => mockComposerApi }),
useAuiState: (selector: (state: { composer: { text: string } }) => unknown) => selector({ composer: { text: '' } }),
useComposerRuntime: () => ({
getState: () => ({ text: '' }),
subscribe: () => () => undefined
})
}))
interface ProbeHarnessProps {
activeQueueSessionKey: string | null
onLayoutSnapshot: (attachments: ComposerAttachment[]) => void
sessionId: string
}
function ProbeHarness({ activeQueueSessionKey, onLayoutSnapshot, sessionId }: ProbeHarnessProps) {
useComposerDraft({
activeQueueSessionKey,
focusKey: null,
inputDisabled: false,
queueEditRef: { current: null as QueueEditState | null },
sessionId
})
// useLayoutEffect fires synchronously right after the DOM commit, BEFORE
// the hook's per-thread scope-swap useEffect (a passive effect) has a
// chance to swap attachmentScope.$attachments over to the new session. A
// synchronous read here — the same read ChatBar's `attachments` prop
// performs at render time — observes the OUTGOING session's attachments.
useLayoutEffect(() => {
onLayoutSnapshot(mainComposerScope.$attachments.get())
})
return null
}
describe('useComposerDraft — attachment scope stays coherent with the committed session on switch (#59305)', () => {
afterEach(() => {
cleanup()
mainComposerScope.clear()
})
it('clears the outgoing session attachments by the layout phase right after switching sessions', () => {
const attachmentA: ComposerAttachment = { id: 'url-A', kind: 'url', label: 'A' }
stashSessionDraft('session-A', 'hi from A', [attachmentA])
const snapshots: ComposerAttachment[][] = []
const { rerender } = render(
<ProbeHarness activeQueueSessionKey="session-A" onLayoutSnapshot={s => snapshots.push(s)} sessionId="session-A" />
)
// Mount loads session A's stashed attachment into the (module-level) main
// scope — confirms the fixture actually seeded the leak precondition.
expect(mainComposerScope.$attachments.get()).toEqual([attachmentA])
snapshots.length = 0 // drop the initial-mount snapshot; only the switch matters
act(() => {
rerender(
<ProbeHarness
activeQueueSessionKey="session-B"
onLayoutSnapshot={s => snapshots.push(s)}
sessionId="session-B"
/>
)
})
// By the layout phase the scope must already be B's (empty) — a submit
// fired the instant B renders must never ship session A's attachment.
expect(snapshots[0]).toEqual([])
})
})
describe('useComposerDraft — rehydrate diagnostic log stays redacted', () => {
afterEach(() => {
cleanup()
mainComposerScope.clear()
vi.restoreAllMocks()
})
it('logs counts/kinds/scope on restore but never the raw url, refText, or label', () => {
const secretUrl = 'https://secret.example.com/private-workspace-path'
const attachment: ComposerAttachment = {
id: 'url-secret',
kind: 'url',
label: 'do-not-leak-label',
refText: `@url:${secretUrl}`
}
stashSessionDraft('session-secret', '', [attachment])
const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined)
render(
<ProbeHarness
activeQueueSessionKey="session-secret"
onLayoutSnapshot={() => undefined}
sessionId="session-secret"
/>
)
const rehydrateCalls = debugSpy.mock.calls.filter(call => call[0] === '[composer-rehydrate]')
expect(rehydrateCalls.length).toBeGreaterThan(0)
const serialized = JSON.stringify(rehydrateCalls)
expect(serialized).not.toContain(secretUrl)
expect(serialized).not.toContain(attachment.label)
expect(serialized).not.toContain(attachment.refText)
expect(rehydrateCalls[0]?.[1]).toMatchObject({
attachmentCount: 1,
attachmentKinds: ['url'],
scope: 'session-secret'
})
})
})

View file

@ -1,5 +1,5 @@
import { useAui, useAuiState, useComposerRuntime } from '@assistant-ui/react'
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
import { type RefObject, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
import { SLASH_COMMAND_RE } from '@/lib/chat-runtime'
import { type ComposerAttachment, stashSessionDraft, takeSessionDraft } from '@/store/composer'
@ -20,7 +20,7 @@ import {
onComposerInsertRequest
} from '../focus'
import { type InlineRefInput, insertInlineRefsIntoEditor } from '../inline-refs'
import { composerPlainText, placeCaretEnd, renderComposerContents } from '../rich-editor'
import { composerPlainText, placeCaretEnd, REF_RE, renderComposerContents } from '../rich-editor'
import { useComposerScope } from '../scope'
import type { ChatBarProps } from '../types'
@ -189,6 +189,21 @@ export function useComposerDraft({
stashSessionDraft(scope, text, attachments)
const loadIntoComposer = (text: string, attachments: ComposerAttachment[]) => {
// Diagnostic breadcrumb for #59305-class reports: identifies WHAT kind of
// state got restored into the composer (session switch, queue-edit
// restore, history browse) without logging any raw content. REF_RE has the
// global flag — testing against a throwaway clone avoids mutating the
// shared instance's lastIndex, which would otherwise corrupt this check on
// the next call.
if (attachments.length > 0 || new RegExp(REF_RE.source, REF_RE.flags).test(text)) {
console.debug('[composer-rehydrate]', {
attachmentCount: attachments.length,
attachmentKinds: attachments.map(a => a.kind),
hasTextRefs: new RegExp(REF_RE.source, REF_RE.flags).test(text),
scope: activeQueueSessionKeyRef.current
})
}
attachmentScope.$attachments.set(cloneAttachments(attachments))
paintDraft(text, false)
}
@ -231,6 +246,7 @@ export function useComposerDraft({
// source otherwise), and (3) schedule the debounced per-session stash.
// Browsing history / editing a queued prompt suppress the stash so recalled
// text never clobbers the draft.
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
useEffect(() => {
const sync = () => {
const text = composerRuntime.getState().text
@ -318,7 +334,17 @@ export function useComposerDraft({
// Per-thread draft swap — the composer's only session coupling. Lifecycle
// never clears composer state; this effect alone stashes on leave, restores
// on enter. Keyed writes are idempotent, so no skip-sentinel.
useEffect(() => {
//
// MUST be a layout effect, not a passive one: it swaps attachmentScope's
// module-level $attachments atom, and a passive effect fires only after the
// browser paints the new session's view — leaving a window where the DOM
// already shows session B while $attachments (and therefore ChatBar's
// `attachments` prop) still holds session A's chips. A submit fired in that
// window (e.g. a fast session switch immediately followed by Enter) would
// ship A's attachments into B's turn (#59305). useLayoutEffect closes the
// window by running before paint.
useLayoutEffect(() => {
// A pending debounce timer from the outgoing session is now stale — its
// scope was correct when scheduled, but the authoritative stash below
// (and the cleanup on the way out) already covers that text. Letting it
@ -344,6 +370,7 @@ export function useComposerDraft({
// pagehide is load-bearing: React skips effect cleanups on reload, so Cmd+R
// inside the debounce/rAF window would drop trailing keystrokes without this.
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
useEffect(() => {
const flushPendingDraftPersist = () => {
const scope = draftScopeRef.current

View file

@ -1,6 +1,12 @@
import { useAuiState } from '@assistant-ui/react'
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
import {
clearSurfaceVar,
COMPOSER_HEIGHT_VAR,
COMPOSER_SURFACE_HEIGHT_VAR,
setSurfaceVar
} from '@/app/chat/surface-vars'
import { useMediaQuery } from '@/hooks/use-media-query'
import { useResizeObserver } from '@/hooks/use-resize-observer'
import { $composerPoppedOut } from '@/store/composer-popout'
@ -89,18 +95,16 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
// (Read globals here so the callback stays stable; mirror the popoutAllowed
// gate since secondary windows are forced docked.)
if ($composerPoppedOut.get() && !isSecondaryWindow()) {
const root = document.documentElement
lastBucketedHeightRef.current = 0
lastBucketedSurfaceHeightRef.current = 0
root.style.setProperty('--composer-measured-height', '0px')
root.style.setProperty('--composer-surface-measured-height', '0px')
setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, '0px')
setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, '0px')
return
}
const { height, width } = composer.getBoundingClientRect()
const surfaceHeight = composerSurfaceRef.current?.getBoundingClientRect().height
const root = document.documentElement
if (width > 0) {
const nextTight = width < COMPOSER_STACK_BREAKPOINT_PX
@ -135,7 +139,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
if (bucket !== lastBucketedHeightRef.current) {
lastBucketedHeightRef.current = bucket
root.style.setProperty('--composer-measured-height', `${bucket}px`)
setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, `${bucket}px`)
}
}
@ -144,7 +148,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
if (bucket !== lastBucketedSurfaceHeightRef.current) {
lastBucketedSurfaceHeightRef.current = bucket
root.style.setProperty('--composer-surface-measured-height', `${bucket}px`)
setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, `${bucket}px`)
}
}
}, [composerRef, composerSurfaceRef, editorRef])
@ -160,12 +164,13 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
}, [poppedOut, syncComposerMetrics])
useEffect(() => {
const composer = composerRef.current
return () => {
const root = document.documentElement
root.style.removeProperty('--composer-measured-height')
root.style.removeProperty('--composer-surface-measured-height')
clearSurfaceVar(composer, COMPOSER_HEIGHT_VAR)
clearSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR)
}
}, [])
}, [composerRef])
// Pill compacts on real width (tile/pane), OR when stacked for any reason
// (viewport-narrow / wrapped) so the controls row never over-runs.

View file

@ -29,6 +29,7 @@ export function useComposerPlaceholder({ disabled, reconnecting, sessionId }: Us
const prevSessionIdRef = useRef(sessionId)
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
useEffect(() => {
const prev = prevSessionIdRef.current
prevSessionIdRef.current = sessionId

View file

@ -322,6 +322,7 @@ export function useComposerQueue({
// never churns, so a change there is a real session switch and must NOT
// migrate; only the runtime-derived key (queueSessionKey falsy → key is
// sessionId) churns on a backend bounce/resume of the same conversation.
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
useEffect(() => {
const prev = prevQueueKeyRef.current
prevQueueKeyRef.current = activeQueueSessionKey

View file

@ -113,7 +113,9 @@ describe('useComposerSubmit busy-turn routing', () => {
hook.result.current.submitDraft()
})
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('/compress preserve context'))
await waitFor(() =>
expect(onSubmit).toHaveBeenCalledWith('/compress preserve context', { composerScope: 'stored-session' })
)
expect(clearDraft).toHaveBeenCalledTimes(1)
expect(onSteer).not.toHaveBeenCalled()
expect(queueCurrentDraft).not.toHaveBeenCalled()
@ -159,9 +161,26 @@ describe('useComposerSubmit busy-turn routing', () => {
hook.result.current.submitDraft()
})
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('ordinary question', { attachments: [] }))
await waitFor(() =>
expect(onSubmit).toHaveBeenCalledWith('ordinary question', {
attachments: [],
composerScope: 'stored-session'
})
)
expect(onSteer).not.toHaveBeenCalled()
expect(queueCurrentDraft).not.toHaveBeenCalled()
expect(onCancel).not.toHaveBeenCalled()
})
it('threads the loaded composer scope through onSubmit for the #59305 submit-time guard', async () => {
const { hook, onSubmit } = renderSubmitHook({ text: 'hello' })
act(() => {
hook.result.current.submitDraft()
})
await waitFor(() =>
expect(onSubmit).toHaveBeenCalledWith('hello', expect.objectContaining({ composerScope: 'stored-session' }))
)
})
})

View file

@ -89,7 +89,11 @@ export function useComposerSubmit({
stashAt(submittedScope, text, submittedAttachments)
}
void Promise.resolve(attachments ? onSubmit(text, { attachments }) : onSubmit(text))
void Promise.resolve(
attachments
? onSubmit(text, { attachments, composerScope: submittedScope })
: onSubmit(text, { composerScope: submittedScope })
)
.then(accepted => void (accepted === false ? restore() : clearSessionDraft(submittedScope)))
.catch(restore)
}

View file

@ -0,0 +1,123 @@
import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core'
import { act, renderHook } from '@testing-library/react'
import { createRef } from 'react'
import { describe, expect, it, vi } from 'vitest'
import { composerPlainText, renderComposerContents, RICH_INPUT_SLOT } from '../rich-editor'
import { useComposerTrigger } from './use-composer-trigger'
/** A live contentEditable seeded with `text`, caret parked at the end. */
function mountEditor(text: string) {
const editor = document.createElement('div')
editor.dataset.slot = RICH_INPUT_SLOT
editor.contentEditable = 'true'
document.body.append(editor)
renderComposerContents(editor, text)
const range = document.createRange()
range.selectNodeContents(editor)
range.collapse(false)
const selection = window.getSelection()!
selection.removeAllRanges()
selection.addRange(range)
return editor
}
const item = (command: string, group = 'Skills'): Unstable_TriggerItem => ({
id: command,
type: 'slash',
label: command.slice(1),
metadata: { command, display: command, meta: '', group, action: '', rawText: command }
})
function mountTrigger(editor: HTMLDivElement, items: Unstable_TriggerItem[]) {
const editorRef = createRef<HTMLDivElement>() as { current: HTMLDivElement | null }
editorRef.current = editor
const draftRef = { current: composerPlainText(editor) }
const adapter: Unstable_TriggerAdapter = {
categories: () => [],
categoryItems: () => [],
search: () => items
}
const setComposerText = vi.fn()
const hook = renderHook(() =>
useComposerTrigger({
at: { adapter: null, loading: false },
draftRef,
editorRef,
requestMainFocus: vi.fn(),
setComposerText,
slash: { adapter, loading: false }
})
)
return { draftRef, hook, setComposerText }
}
describe('useComposerTrigger — slash anywhere in the prompt', () => {
it('opens the completion list for a slash typed mid-message', () => {
const editor = mountEditor('please run /cle')
const { hook } = mountTrigger(editor, [item('/clean')])
act(() => hook.result.current.refreshTrigger())
expect(hook.result.current.trigger).toMatchObject({ kind: '/', inline: true, query: 'cle' })
expect(hook.result.current.triggerItems).toHaveLength(1)
})
it('inserts the picked skill inline and keeps the surrounding prose intact', () => {
const editor = mountEditor('please run /cle')
const { hook } = mountTrigger(editor, [item('/clean')])
act(() => hook.result.current.refreshTrigger())
act(() => hook.result.current.replaceTriggerWithChip(item('/clean')))
// The `/cle` the user typed is replaced by the full command; "please run"
// in front of it survives untouched.
expect(composerPlainText(editor)).toBe('please run /clean ')
})
it('offers only skills mid-message, not app commands', () => {
// `/model` and `/new` act on the app — meaningless as a reference in prose.
const editor = mountEditor('please run /')
const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands'), item('/new', 'Commands')])
act(() => hook.result.current.refreshTrigger())
expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean'])
})
it('still offers the full command set at the start of the prompt', () => {
const editor = mountEditor('/')
const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands')])
act(() => hook.result.current.refreshTrigger())
expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean', 'model'])
})
it('still opens the list for a slash at the start of the prompt', () => {
const editor = mountEditor('/cle')
const { hook } = mountTrigger(editor, [item('/clean')])
act(() => hook.result.current.refreshTrigger())
expect(hook.result.current.trigger).toMatchObject({ kind: '/', query: 'cle' })
expect(hook.result.current.trigger?.inline).toBeUndefined()
})
it('leaves a mid-message file path alone', () => {
const editor = mountEditor('open src/foo/bar')
const { hook } = mountTrigger(editor, [item('/clean')])
act(() => hook.result.current.refreshTrigger())
expect(hook.result.current.trigger).toBeNull()
})
})

Some files were not shown because too many files have changed in this diff Show more