mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Merge origin/main into feat/hermes-relay-shared-metrics
Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
commit
45580cc93a
1114 changed files with 96474 additions and 8371 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -1,6 +1,8 @@
|
|||
.DS_Store
|
||||
/venv/
|
||||
/venv.old/
|
||||
/venv.stale.runtime-*/
|
||||
/.hermes-runtime/
|
||||
/_pycache/
|
||||
*.pyc*
|
||||
__pycache__/
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ else:
|
|||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
|
|
@ -251,11 +252,13 @@ def main(argv: list[str] | None = None) -> None:
|
|||
# MCP servers dynamically via asyncio.to_thread inside the event
|
||||
# loop; that path is unaffected.) Moved from model_tools.py module
|
||||
# scope to avoid freezing the gateway's loop on lazy import (#16856).
|
||||
try:
|
||||
from tools.mcp_tool import discover_mcp_tools
|
||||
discover_mcp_tools()
|
||||
except Exception:
|
||||
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
|
||||
# Metadata-only hosts can opt out of unrelated global MCP startup.
|
||||
if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1":
|
||||
try:
|
||||
from tools.mcp_tool import discover_mcp_tools
|
||||
discover_mcp_tools()
|
||||
except Exception:
|
||||
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
|
||||
|
||||
agent = HermesACPAgent()
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,110 @@ from tools.approval import (
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]:
|
||||
"""Return ``(slug, label, [(model_id, description), ...])`` for named endpoints.
|
||||
|
||||
Covers both the v12 ``providers:`` mapping and the legacy
|
||||
``custom_providers:`` list. These endpoints never appear in canonical
|
||||
provider enumeration, so without this the ACP model selector hides every
|
||||
named endpoint that the TUI ``/model`` picker already renders (#47039
|
||||
implemented named-endpoint rows for the TUI surface only).
|
||||
|
||||
Model lists come from the entry's declared models (``default_model`` +
|
||||
``models``), refreshed from the endpoint's live ``/models`` listing when a
|
||||
credential is available and ``discover_models`` is not disabled. Declared
|
||||
models are kept even when live discovery fails — some OpenAI-compatible
|
||||
endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at
|
||||
all yet serve the declared models fine.
|
||||
|
||||
Slugs use the ``custom:<name>`` shape that ``parse_model_input`` and
|
||||
``resolve_runtime_provider`` already resolve, so encoded choice ids
|
||||
(``custom:<name>:<model>``) round-trip through ``set_session_model``
|
||||
unchanged.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import (
|
||||
get_compatible_custom_providers,
|
||||
is_provider_enabled,
|
||||
load_config,
|
||||
)
|
||||
from hermes_cli.models import fetch_api_models
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
try:
|
||||
cfg = load_config()
|
||||
entries = get_compatible_custom_providers(cfg)
|
||||
except Exception:
|
||||
logger.debug("Could not load named custom providers", exc_info=True)
|
||||
return []
|
||||
|
||||
# ``get_compatible_custom_providers`` drops the ``enabled`` flag during
|
||||
# normalization, so collect explicitly disabled provider keys from the
|
||||
# raw config and skip their entries below.
|
||||
disabled_keys: set[str] = set()
|
||||
raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None
|
||||
if isinstance(raw_providers, dict):
|
||||
for raw_key, raw_entry in raw_providers.items():
|
||||
if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry):
|
||||
disabled_keys.add(str(raw_key).strip().lower())
|
||||
|
||||
catalogs: list[tuple[str, str, list[tuple[str, str]]]] = []
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
provider_key = str(entry.get("provider_key", "") or "").strip()
|
||||
if provider_key.lower() in disabled_keys:
|
||||
continue
|
||||
name = str(entry.get("name", "") or "").strip()
|
||||
base_url = str(entry.get("base_url", "") or "").strip()
|
||||
if not name or not base_url:
|
||||
continue
|
||||
slug_source = provider_key or name
|
||||
slug = "custom:" + slug_source.strip().lower().replace(" ", "-")
|
||||
|
||||
api_key = str(entry.get("api_key", "") or "").strip()
|
||||
if not api_key:
|
||||
key_env = str(entry.get("key_env", "") or "").strip()
|
||||
api_key = os.environ.get(key_env, "").strip() if key_env else ""
|
||||
|
||||
declared: list[str] = []
|
||||
default_model = str(entry.get("model", "") or "").strip()
|
||||
if default_model:
|
||||
declared.append(default_model)
|
||||
models_cfg = entry.get("models")
|
||||
if isinstance(models_cfg, dict):
|
||||
for mid in models_cfg:
|
||||
mid = str(mid or "").strip()
|
||||
if mid and mid not in declared:
|
||||
declared.append(mid)
|
||||
|
||||
if not api_key and not declared:
|
||||
# No credential to discover with and nothing declared:
|
||||
# not addressable from the selector.
|
||||
continue
|
||||
|
||||
model_ids = list(declared)
|
||||
discover = entry.get("discover_models", True)
|
||||
if isinstance(discover, str):
|
||||
discover = discover.lower() not in {"false", "no", "0"}
|
||||
if discover and api_key:
|
||||
try:
|
||||
live = fetch_api_models(
|
||||
api_key, base_url, api_mode=entry.get("api_mode")
|
||||
)
|
||||
except Exception:
|
||||
live = None
|
||||
if live:
|
||||
model_ids = declared + [m for m in live if m not in declared]
|
||||
|
||||
if not model_ids:
|
||||
continue
|
||||
catalogs.append((slug, name, [(mid, "") for mid in model_ids]))
|
||||
|
||||
return catalogs
|
||||
|
||||
try:
|
||||
from hermes_cli import __version__ as HERMES_VERSION
|
||||
except Exception:
|
||||
|
|
@ -97,6 +201,13 @@ _executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="acp-agent")
|
|||
# does not expose a client-side limit, so this is a fixed cap that clients
|
||||
# paginate against using `cursor` / `next_cursor`.
|
||||
_LIST_SESSIONS_PAGE_SIZE = 50
|
||||
# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render
|
||||
# the whole `availableModels` array in one dropdown, so an unbounded
|
||||
# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker
|
||||
# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not
|
||||
# the total; aggregator providers stay intentionally uncapped inside the shared
|
||||
# inventory, and the current model is always kept via the fallback insert below.
|
||||
ACP_MAX_MODELS_PER_PROVIDER = 200
|
||||
_MAX_ACP_RESOURCE_BYTES = 512 * 1024
|
||||
_TEXT_RESOURCE_MIME_PREFIXES = ("text/",)
|
||||
_TEXT_RESOURCE_MIME_TYPES = {
|
||||
|
|
@ -585,46 +696,108 @@ class HermesACPAgent(acp.Agent):
|
|||
return f"{raw_provider}:{raw_model}"
|
||||
|
||||
def _build_model_state(self, state: SessionState) -> SessionModelState | None:
|
||||
"""Return the ACP model selector payload for editors like Zed."""
|
||||
"""Return authenticated providers and their models for ACP clients.
|
||||
|
||||
The shared Hermes inventory is also used by ``hermes model``, the TUI,
|
||||
and the dashboard. Keeping ACP on that substrate prevents its selector
|
||||
from silently collapsing to the current provider's curated list.
|
||||
"""
|
||||
model = str(state.model or getattr(state.agent, "model", "") or "").strip()
|
||||
provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter"
|
||||
|
||||
try:
|
||||
from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label
|
||||
from hermes_cli.inventory import build_models_payload, load_picker_context
|
||||
from hermes_cli.models import normalize_provider, provider_label
|
||||
|
||||
normalized_provider = normalize_provider(provider)
|
||||
provider_name = provider_label(normalized_provider)
|
||||
context = load_picker_context().with_overrides(
|
||||
current_provider=normalized_provider,
|
||||
current_model=model,
|
||||
current_base_url=str(getattr(state.agent, "base_url", "") or ""),
|
||||
)
|
||||
payload = build_models_payload(
|
||||
context,
|
||||
explicit_only=True,
|
||||
include_unconfigured=False,
|
||||
picker_hints=False,
|
||||
canonical_order=True,
|
||||
pricing=False,
|
||||
capabilities=False,
|
||||
refresh=False,
|
||||
probe_custom_providers=False,
|
||||
probe_current_custom_provider=False,
|
||||
max_models=ACP_MAX_MODELS_PER_PROVIDER,
|
||||
)
|
||||
|
||||
available_models: list[ModelInfo] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
for model_id, description in curated_models_for_provider(normalized_provider):
|
||||
rendered_model = str(model_id or "").strip()
|
||||
if not rendered_model:
|
||||
for row in payload.get("providers") or []:
|
||||
row_provider = normalize_provider(str(row.get("slug") or "").strip())
|
||||
if not row_provider:
|
||||
continue
|
||||
choice_id = self._encode_model_choice(normalized_provider, rendered_model)
|
||||
if choice_id in seen_ids:
|
||||
continue
|
||||
desc_parts = [f"Provider: {provider_name}"]
|
||||
if description:
|
||||
desc_parts.append(str(description).strip())
|
||||
if rendered_model == model:
|
||||
desc_parts.append("current")
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=choice_id,
|
||||
name=rendered_model,
|
||||
description=" • ".join(part for part in desc_parts if part),
|
||||
)
|
||||
provider_name = str(row.get("name") or "").strip() or provider_label(
|
||||
row_provider
|
||||
)
|
||||
seen_ids.add(choice_id)
|
||||
for model_entry in row.get("models") or []:
|
||||
if isinstance(model_entry, dict):
|
||||
rendered_model = str(
|
||||
model_entry.get("id")
|
||||
or model_entry.get("model")
|
||||
or model_entry.get("name")
|
||||
or ""
|
||||
).strip()
|
||||
else:
|
||||
rendered_model = str(model_entry or "").strip()
|
||||
if not rendered_model:
|
||||
continue
|
||||
choice_id = self._encode_model_choice(row_provider, rendered_model)
|
||||
if choice_id in seen_ids:
|
||||
continue
|
||||
is_current = (
|
||||
row_provider == normalized_provider and rendered_model == model
|
||||
)
|
||||
description = f"Provider: {provider_name}"
|
||||
if is_current:
|
||||
description += " • current"
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=choice_id,
|
||||
name=f"{provider_name} · {rendered_model}",
|
||||
description=description,
|
||||
)
|
||||
)
|
||||
seen_ids.add(choice_id)
|
||||
|
||||
# Named user-defined endpoints (providers: / custom_providers:)
|
||||
# are invisible to canonical provider enumeration — append them
|
||||
# so editor clients can select them like the TUI /model picker.
|
||||
for named_slug, named_label, named_catalog in _named_custom_provider_catalogs():
|
||||
for named_model, named_desc in named_catalog:
|
||||
named_choice = self._encode_model_choice(named_slug, named_model)
|
||||
if not named_choice or named_choice in seen_ids:
|
||||
continue
|
||||
named_parts = [f"Provider: {named_label}"]
|
||||
if named_desc:
|
||||
named_parts.append(str(named_desc).strip())
|
||||
if named_slug == normalized_provider and named_model == model:
|
||||
named_parts.append("current")
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=named_choice,
|
||||
name=named_model,
|
||||
description=" • ".join(part for part in named_parts if part),
|
||||
)
|
||||
)
|
||||
seen_ids.add(named_choice)
|
||||
|
||||
current_model_id = self._encode_model_choice(normalized_provider, model)
|
||||
if current_model_id and current_model_id not in seen_ids:
|
||||
provider_name = provider_label(normalized_provider)
|
||||
available_models.insert(
|
||||
0,
|
||||
ModelInfo(
|
||||
model_id=current_model_id,
|
||||
name=model,
|
||||
name=f"{provider_name} · {model}",
|
||||
description=f"Provider: {provider_name} • current",
|
||||
),
|
||||
)
|
||||
|
|
@ -1588,7 +1761,16 @@ class HermesACPAgent(acp.Agent):
|
|||
clear_session_vars,
|
||||
set_session_vars,
|
||||
)
|
||||
session_tokens = set_session_vars(session_key=session_id)
|
||||
# ``cwd`` pins the logical working directory for this context,
|
||||
# which is what the system prompt's "Current working directory"
|
||||
# line reports (agent/prompt_builder.py -> resolve_agent_cwd).
|
||||
# Without it the prompt advertises the global Hermes workspace
|
||||
# while the tools are rooted at the client's project, so the
|
||||
# model emits absolute paths under ~/.hermes/workspace and the
|
||||
# edit silently lands outside the editor's workspace.
|
||||
session_tokens = set_session_vars(
|
||||
session_key=session_id, cwd=state.cwd,
|
||||
)
|
||||
except Exception:
|
||||
session_tokens = None
|
||||
clear_session_vars = None # type: ignore[assignment]
|
||||
|
|
@ -1875,8 +2057,26 @@ class HermesACPAgent(acp.Agent):
|
|||
if handler is None:
|
||||
return None # not a known command — let the LLM handle it
|
||||
|
||||
try:
|
||||
# Slash handlers run on the event-loop thread, OUTSIDE the per-turn
|
||||
# contextvars.copy_context() that pins the session cwd for the agent
|
||||
# call. ``/compress`` and ``/model`` reach code that REBUILDS the
|
||||
# system prompt (agent._build_system_prompt -> resolve_agent_cwd), so
|
||||
# an unpinned handler bakes the Hermes install tree into the session's
|
||||
# cached prompt — persisted, and therefore poisoning every later turn
|
||||
# even though the turn itself is pinned. Pin inside a fresh context so
|
||||
# the write can't leak into other concurrent ACP sessions and needs no
|
||||
# teardown.
|
||||
def _dispatch() -> str | None:
|
||||
try:
|
||||
from agent.runtime_cwd import set_session_cwd
|
||||
|
||||
set_session_cwd(state.cwd)
|
||||
except Exception:
|
||||
logger.debug("Could not pin ACP session cwd for slash command", exc_info=True)
|
||||
return handler(args, state)
|
||||
|
||||
try:
|
||||
return contextvars.copy_context().run(_dispatch)
|
||||
except Exception as e:
|
||||
logger.error("Slash command /%s error: %s", cmd, e, exc_info=True)
|
||||
return f"Error executing /{cmd}: {e}"
|
||||
|
|
|
|||
|
|
@ -69,6 +69,43 @@ def _ra():
|
|||
return run_agent
|
||||
|
||||
|
||||
def _moa_reference_output_allowed(agent: Any) -> bool:
|
||||
"""Keep MoA display events off only the machine-readable ``-Q`` surface."""
|
||||
return not (
|
||||
getattr(agent, "platform", None) == "cli"
|
||||
and getattr(agent, "tool_progress_mode", "all") == "off"
|
||||
)
|
||||
|
||||
|
||||
def _relay_moa_reference_event(agent: Any, event: str, **kwargs: Any) -> None:
|
||||
"""Relay MoA display events while preserving the ``-Q`` stdout contract."""
|
||||
if not _moa_reference_output_allowed(agent):
|
||||
return
|
||||
cb = getattr(agent, "tool_progress_callback", None)
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
if event == "moa.reference":
|
||||
cb(
|
||||
"moa.reference",
|
||||
str(kwargs.get("label") or ""),
|
||||
str(kwargs.get("text") or ""),
|
||||
None,
|
||||
moa_index=kwargs.get("index"),
|
||||
moa_count=kwargs.get("count"),
|
||||
)
|
||||
elif event == "moa.aggregating":
|
||||
cb(
|
||||
"moa.aggregating",
|
||||
str(kwargs.get("aggregator") or ""),
|
||||
None,
|
||||
None,
|
||||
moa_ref_count=kwargs.get("ref_count"),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _normalize_route_base_url(base_url: Any) -> str:
|
||||
"""Canonicalize an endpoint URL for model-route identity comparisons."""
|
||||
return normalize_route_base_url(base_url)
|
||||
|
|
@ -786,9 +823,10 @@ def init_agent(
|
|||
# Anthropic prompt caching: auto-enabled for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party gateways that speak the
|
||||
# Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces
|
||||
# input costs by ~75% on multi-turn conversations. Uses system_and_3
|
||||
# strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy``
|
||||
# for the layout-vs-transport decision.
|
||||
# input costs by ~75% on multi-turn conversations. Uses four breakpoints:
|
||||
# the static system prefix, full system prompt, and last two messages
|
||||
# (falling back to system-and-3 when no static prefix is available). See
|
||||
# ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision.
|
||||
agent._use_prompt_caching, agent._use_native_cache_layout = (
|
||||
agent._anthropic_prompt_cache_policy()
|
||||
)
|
||||
|
|
@ -1025,49 +1063,20 @@ def init_agent(
|
|||
elif isinstance(effective_key, str) and len(effective_key) > 12:
|
||||
print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}")
|
||||
elif agent.provider == "moa":
|
||||
from agent.moa_loop import MoAClient
|
||||
from agent.moa_loop import build_moa_facade
|
||||
agent.api_mode = "chat_completions"
|
||||
|
||||
# Route reference-model outputs to the agent's tool_progress_callback so
|
||||
# build_moa_facade wires the reference relay that routes
|
||||
# reference-model outputs to the agent's tool_progress_callback so
|
||||
# every surface that already consumes it (CLI spinner/scrollback, TUI,
|
||||
# desktop, gateway) can show each reference's answer as a labelled block
|
||||
# before the aggregator acts. The facade emits "moa.reference" and
|
||||
# "moa.aggregating" events; we forward them through the same callback
|
||||
# the tool lifecycle uses. Best-effort and cache-safe — these are
|
||||
# display-only events, they never touch the message history.
|
||||
def _moa_reference_relay(event: str, **kwargs: Any) -> None:
|
||||
cb = getattr(agent, "tool_progress_callback", None)
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
if event == "moa.reference":
|
||||
label = str(kwargs.get("label") or "")
|
||||
text = str(kwargs.get("text") or "")
|
||||
idx = kwargs.get("index")
|
||||
count = kwargs.get("count")
|
||||
cb(
|
||||
"moa.reference",
|
||||
label,
|
||||
text,
|
||||
None,
|
||||
moa_index=idx,
|
||||
moa_count=count,
|
||||
)
|
||||
elif event == "moa.aggregating":
|
||||
cb(
|
||||
"moa.aggregating",
|
||||
str(kwargs.get("aggregator") or ""),
|
||||
None,
|
||||
None,
|
||||
moa_ref_count=kwargs.get("ref_count"),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
agent.client = MoAClient(
|
||||
agent.model or "default",
|
||||
reference_callback=_moa_reference_relay,
|
||||
)
|
||||
# desktop, gateway) can show each reference's answer as a labelled
|
||||
# block before the aggregator acts. The facade emits "moa.reference",
|
||||
# "moa.progress", "moa.phase", and "moa.aggregating" events, forwarded
|
||||
# through the same callback the tool lifecycle uses. Best-effort and
|
||||
# cache-safe — display-only events, they never touch the message
|
||||
# history. The factory is shared with the fallback-restore/recovery
|
||||
# paths so a restored facade keeps emitting these events (#53802).
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
agent._client_kwargs = {}
|
||||
agent.api_key = api_key or "moa-virtual-provider"
|
||||
agent.base_url = "moa://local"
|
||||
|
|
@ -1333,6 +1342,13 @@ def init_agent(
|
|||
print("⚠️ Warning: API key appears invalid or missing")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to initialize OpenAI client: {e}")
|
||||
|
||||
# Keep a stable identity for the pool entry that supplied this runtime.
|
||||
# OAuth refreshes can replace the runtime token before a failed request is
|
||||
# recovered, so the mutable API-key value alone cannot reliably attribute
|
||||
# the failure to its source entry.
|
||||
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
|
||||
sync_credential_pool_entry_id(agent)
|
||||
|
||||
# Provider fallback chain — ordered list of backup providers tried
|
||||
# when the primary is exhausted (rate-limit, overload, connection
|
||||
|
|
@ -1479,6 +1495,9 @@ def init_agent(
|
|||
|
||||
# Cached system prompt -- built once per session, only rebuilt on compression
|
||||
agent._cached_system_prompt: Optional[str] = None
|
||||
# Cross-session-stable prefix of the cached prompt. It remains separate
|
||||
# from the persisted string and is used only to place an early cache marker.
|
||||
agent._cached_system_prompt_static: Optional[str] = None
|
||||
|
||||
# Filesystem checkpoint manager (transparent — not a tool)
|
||||
from tools.checkpoint_manager import CheckpointManager
|
||||
|
|
@ -1810,6 +1829,28 @@ def init_agent(
|
|||
compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"}
|
||||
compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20))
|
||||
compression_protect_last = int(_compression_cfg.get("protect_last_n", 20))
|
||||
# Minimum REAL (actionable) user messages guaranteed to survive in the
|
||||
# uncompressed tail (compression.min_tail_user_messages). Default 1
|
||||
# preserves current behavior exactly — the existing single-user tail
|
||||
# anchor. Values > 1 extend the guarantee to the last N actionable
|
||||
# user turns. Booleans rejected (bool subclasses int), non-int-like
|
||||
# values fall back to 1, floor at 1.
|
||||
_raw_min_tail_users = _compression_cfg.get("min_tail_user_messages", 1)
|
||||
if isinstance(_raw_min_tail_users, bool):
|
||||
compression_min_tail_users = 1
|
||||
elif isinstance(_raw_min_tail_users, int):
|
||||
compression_min_tail_users = _raw_min_tail_users
|
||||
elif isinstance(_raw_min_tail_users, float):
|
||||
compression_min_tail_users = (
|
||||
int(_raw_min_tail_users) if _raw_min_tail_users.is_integer() else 1
|
||||
)
|
||||
else:
|
||||
try:
|
||||
compression_min_tail_users = int(str(_raw_min_tail_users).strip())
|
||||
except (TypeError, ValueError):
|
||||
compression_min_tail_users = 1
|
||||
if compression_min_tail_users < 1:
|
||||
compression_min_tail_users = 1
|
||||
# Cap on compression retry rounds before a turn gives up with "max
|
||||
# compression attempts reached" (compression.max_attempts). Hardcoding 3
|
||||
# strands sessions that legitimately need more rounds — e.g. a restart
|
||||
|
|
@ -1838,6 +1879,39 @@ def init_agent(
|
|||
if compression_max_attempts < 1:
|
||||
compression_max_attempts = 3
|
||||
compression_max_attempts = min(compression_max_attempts, 10)
|
||||
|
||||
def _parse_prune_int(raw, default):
|
||||
# Same parser semantics as compression.max_attempts above: reject
|
||||
# booleans (bool subclasses int — YAML `true` would coerce to 1),
|
||||
# reject fractional floats rather than truncating them, accept
|
||||
# integral floats and numeric strings, fall back to the default on
|
||||
# anything else.
|
||||
if isinstance(raw, bool):
|
||||
return default
|
||||
if isinstance(raw, int):
|
||||
return raw
|
||||
if isinstance(raw, float):
|
||||
return int(raw) if raw.is_integer() else default
|
||||
try:
|
||||
return int(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
# Opt-in proactive tool-result prune trigger (0 = disabled — the
|
||||
# default, so an unset key is behavior-neutral). Negative values are
|
||||
# treated as disabled rather than erroring.
|
||||
compression_proactive_prune_tokens = max(
|
||||
0, _parse_prune_int(_compression_cfg.get("proactive_prune_tokens", 0), 0)
|
||||
)
|
||||
compression_proactive_prune_min_chars = _parse_prune_int(
|
||||
_compression_cfg.get("proactive_prune_min_result_chars", 8000), 8000
|
||||
)
|
||||
compression_proactive_prune_min_reclaim = max(
|
||||
0,
|
||||
_parse_prune_int(
|
||||
_compression_cfg.get("proactive_prune_min_reclaim_tokens", 4096), 4096
|
||||
),
|
||||
)
|
||||
# protect_first_n is the number of non-system messages to protect at
|
||||
# the head, in addition to the system prompt (which is always
|
||||
# implicitly protected by the compressor). Floor at 0 — a value of
|
||||
|
|
@ -2312,6 +2386,10 @@ def init_agent(
|
|||
max_tokens=agent.max_tokens,
|
||||
model_thresholds=compression_model_thresholds,
|
||||
threshold_tokens_cap=compression_threshold_tokens,
|
||||
proactive_prune_tokens=compression_proactive_prune_tokens,
|
||||
proactive_prune_min_result_chars=compression_proactive_prune_min_chars,
|
||||
proactive_prune_min_reclaim_tokens=compression_proactive_prune_min_reclaim,
|
||||
min_tail_user_messages=compression_min_tail_users,
|
||||
)
|
||||
_bind_session_state = getattr(agent.context_compressor, "bind_session_state", None)
|
||||
if callable(_bind_session_state):
|
||||
|
|
|
|||
|
|
@ -850,6 +850,25 @@ def strip_think_blocks(agent, content: str) -> str:
|
|||
|
||||
|
||||
|
||||
def sync_credential_pool_entry_id(agent) -> None:
|
||||
"""Rebind ``agent._credential_pool_entry_id`` from the current pool + key.
|
||||
|
||||
OAuth refreshes can replace the runtime token before a failed request is
|
||||
recovered, so the mutable API-key value alone cannot reliably attribute
|
||||
the failure to its source entry. This resolves the stable pool-entry ID
|
||||
for the agent's current ``api_key`` and clears it when no pool is bound.
|
||||
"""
|
||||
pool = getattr(agent, "_credential_pool", None)
|
||||
try:
|
||||
agent._credential_pool_entry_id = (
|
||||
pool.entry_id_for_api_key(getattr(agent, "api_key", None))
|
||||
if pool is not None
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
agent._credential_pool_entry_id = None
|
||||
|
||||
|
||||
def recover_with_credential_pool(
|
||||
agent,
|
||||
*,
|
||||
|
|
@ -934,10 +953,30 @@ def recover_with_credential_pool(
|
|||
# failing entry exactly; fall back to current()'s key only when the agent
|
||||
# carries no key at all.
|
||||
_api_key_hint = getattr(agent, "api_key", None) or None
|
||||
_raw_credential_id = getattr(agent, "_credential_pool_entry_id", None)
|
||||
_credential_id = (
|
||||
_raw_credential_id
|
||||
if isinstance(_raw_credential_id, str) and _raw_credential_id
|
||||
else None
|
||||
)
|
||||
if not _api_key_hint:
|
||||
_cur = pool.current()
|
||||
if _cur:
|
||||
_api_key_hint = getattr(_cur, "runtime_api_key", None)
|
||||
if not _credential_id:
|
||||
_current_id = getattr(_cur, "id", None)
|
||||
if isinstance(_current_id, str) and _current_id:
|
||||
_credential_id = _current_id
|
||||
|
||||
def _rotate_failed_credential(rotate_status: int):
|
||||
kwargs = {
|
||||
"status_code": rotate_status,
|
||||
"error_context": error_context,
|
||||
"api_key_hint": _api_key_hint,
|
||||
}
|
||||
if _credential_id:
|
||||
kwargs["credential_id"] = _credential_id
|
||||
return pool.mark_exhausted_and_rotate(**kwargs)
|
||||
|
||||
effective_reason = classified_reason
|
||||
if effective_reason is None:
|
||||
|
|
@ -972,11 +1011,7 @@ def recover_with_credential_pool(
|
|||
# Runtime credentials can be resolved by a separate pool instance,
|
||||
# leaving this recovery pool without ``current_id``. Match the key
|
||||
# that actually failed instead of quarantining a different account.
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (billing) — rotated to pool entry %s",
|
||||
|
|
@ -995,8 +1030,13 @@ def recover_with_credential_pool(
|
|||
# Prefer the entry matching the failing key over the shared current()
|
||||
# pointer, for the same attribution reason as above.
|
||||
current_entry = None
|
||||
if _api_key_hint:
|
||||
if _credential_id:
|
||||
current_entry = next(
|
||||
(e for e in pool.entries() if e.id == _credential_id),
|
||||
None,
|
||||
)
|
||||
if _api_key_hint:
|
||||
current_entry = current_entry or next(
|
||||
(e for e in pool.entries() if e.runtime_api_key == _api_key_hint),
|
||||
None,
|
||||
)
|
||||
|
|
@ -1009,11 +1049,7 @@ def recover_with_credential_pool(
|
|||
current_last_status,
|
||||
)
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s",
|
||||
|
|
@ -1037,11 +1073,7 @@ def recover_with_credential_pool(
|
|||
if not has_retried_429 and not usage_limit_reached:
|
||||
return False, True
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit) — rotated to pool entry %s",
|
||||
|
|
@ -1113,7 +1145,10 @@ def recover_with_credential_pool(
|
|||
# the shared pointer can reference a different, healthy entry, and
|
||||
# refreshing it would consume that entry's single-use refresh token
|
||||
# (or mark it exhausted on failure) for a failure it never had.
|
||||
refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint)
|
||||
refresh_kwargs = {"api_key_hint": _api_key_hint}
|
||||
if _credential_id:
|
||||
refresh_kwargs["credential_id"] = _credential_id
|
||||
refreshed = pool.try_refresh_matching(**refresh_kwargs)
|
||||
if refreshed is not None:
|
||||
# ``try_refresh_matching()`` re-mints a fresh OAuth token and reports
|
||||
# success even when the upstream keeps rejecting it — a single-entry
|
||||
|
|
@ -1145,11 +1180,7 @@ def recover_with_credential_pool(
|
|||
# Refresh failed — rotate to next credential instead of giving up.
|
||||
# The failed entry is already marked exhausted by the refresh attempt.
|
||||
rotate_status = status_code if status_code is not None else 401
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (auth refresh failed) — rotated to pool entry %s",
|
||||
|
|
@ -1194,11 +1225,17 @@ def try_recover_primary_transport(
|
|||
return False
|
||||
|
||||
try:
|
||||
# Close existing client to release stale connections
|
||||
# Retire the existing client to release stale connections. #70773:
|
||||
# never hard-close the shared client here — this runs on the
|
||||
# conversation-loop thread while workers from stale-killed streaming
|
||||
# attempts may still be unwinding their SSL BIOs on the old pool.
|
||||
# ``_retire_shared_openai_client`` shuts the sockets down (FD-safe
|
||||
# from any thread) and defers the FD release to GC, which cannot
|
||||
# complete until every borrowing thread has unwound.
|
||||
if getattr(agent, "client", None) is not None:
|
||||
try:
|
||||
agent._close_openai_client(
|
||||
agent.client, reason="primary_recovery", shared=True,
|
||||
agent._retire_shared_openai_client(
|
||||
agent.client, reason="primary_recovery",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -1225,6 +1262,14 @@ def try_recover_primary_transport(
|
|||
)
|
||||
agent._is_anthropic_oauth = rt["is_anthropic_oauth"]
|
||||
agent.client = None
|
||||
elif (agent.provider or "").strip().lower() == "moa":
|
||||
# MoA is a virtual provider with empty client_kwargs — rebuilding
|
||||
# via _create_openai_client would raise "api_key client option
|
||||
# must be set". Recreate the facade through the shared factory so
|
||||
# the reference_callback relay survives recovery (#53802).
|
||||
from agent.moa_loop import build_moa_facade
|
||||
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
else:
|
||||
agent.client = agent._create_openai_client(
|
||||
dict(rt["client_kwargs"]),
|
||||
|
|
@ -1388,7 +1433,18 @@ def restore_primary_runtime(agent) -> bool:
|
|||
)
|
||||
|
||||
# ── Rebuild client for the primary provider ──
|
||||
if agent.api_mode == "anthropic_messages":
|
||||
if agent.provider == "moa":
|
||||
# MoA is a virtual chat-completions provider. It never has real
|
||||
# OpenAI client kwargs; restoring it after a fallback must recreate
|
||||
# the facade, not call OpenAI() with an empty api_key. Use the
|
||||
# shared factory so the restored facade keeps the reference_callback
|
||||
# relay wired at init — a bare MoAClient() would silently stop
|
||||
# emitting moa.reference/moa.aggregating display events (#53802).
|
||||
from agent.moa_loop import build_moa_facade
|
||||
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
agent._anthropic_client = None
|
||||
elif agent.api_mode == "anthropic_messages":
|
||||
from agent.anthropic_adapter import build_anthropic_client
|
||||
agent._anthropic_api_key = rt["anthropic_api_key"]
|
||||
agent._anthropic_base_url = rt["anthropic_base_url"]
|
||||
|
|
@ -1441,6 +1497,7 @@ def restore_primary_runtime(agent) -> bool:
|
|||
pool_matches_primary = False
|
||||
if pool is not None and pool_provider and not pool_matches_primary:
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
|
||||
|
|
@ -1460,6 +1517,7 @@ def restore_primary_runtime(agent) -> bool:
|
|||
# the pool for its current best entry and swap the live credential in.
|
||||
# When the pool is absent, empty, or the entry has no usable key, we
|
||||
# keep the snapshot key (the existing behavior). Fixes #25205.
|
||||
agent._credential_pool_entry_id = None
|
||||
pool = getattr(agent, "_credential_pool", None)
|
||||
if pool is not None and pool.has_available():
|
||||
entry = pool.select()
|
||||
|
|
@ -2056,6 +2114,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
# restore the original pool (issue #52727: pool reload is part of this
|
||||
# switch and must be reversible on rollback).
|
||||
_snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING)
|
||||
_snapshot["_credential_pool_entry_id"] = getattr(
|
||||
agent, "_credential_pool_entry_id", _MISSING
|
||||
)
|
||||
|
||||
try:
|
||||
# Clear the per-config context_length override so the new model's
|
||||
|
|
@ -2112,6 +2173,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
# A pool bound to the old provider is worse than no pool: the
|
||||
# recovery guard rejects it and every later 401/429 skips rotation.
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
agent._credential_pool = load_pool(new_provider)
|
||||
|
|
@ -2121,10 +2183,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
"continuing without pool rotation this turn",
|
||||
new_provider, _pool_exc,
|
||||
)
|
||||
|
||||
# ── Build new client ──
|
||||
if (new_provider or "").strip().lower() == "moa":
|
||||
from agent.moa_loop import MoAClient
|
||||
from agent.moa_loop import build_moa_facade
|
||||
|
||||
# The MoA virtual provider speaks only chat.completions via the
|
||||
# MoAClient facade — the aggregator's real transport
|
||||
|
|
@ -2141,7 +2202,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
agent.api_key = api_key or "moa-virtual-provider"
|
||||
agent.base_url = "moa://local"
|
||||
agent._client_kwargs = {}
|
||||
agent.client = MoAClient(agent.model or "default")
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
elif api_mode == "anthropic_messages":
|
||||
from agent.anthropic_adapter import (
|
||||
build_anthropic_client,
|
||||
|
|
@ -2217,6 +2278,8 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
reason="switch_model",
|
||||
shared=True,
|
||||
)
|
||||
|
||||
sync_credential_pool_entry_id(agent)
|
||||
except Exception:
|
||||
# Rollback every mutated field to the pre-swap snapshot so the agent
|
||||
# is left consistent (old model + old provider + old client) and the
|
||||
|
|
|
|||
|
|
@ -368,7 +368,7 @@ def _detect_claude_code_version() -> str:
|
|||
try:
|
||||
result = _sp.run(
|
||||
[cmd, "--version"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
# Output is like "2.1.74 (Claude Code)" or just "2.1.74"
|
||||
|
|
@ -914,7 +914,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]:
|
|||
"-s", "Claude Code-credentials",
|
||||
"-w"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=5,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
@ -1920,10 +1920,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|||
return None
|
||||
btype = b.get("type")
|
||||
if btype == "text":
|
||||
# Coerce empty/whitespace-only text to a non-whitespace placeholder;
|
||||
# the Messages input schema rejects blank text blocks (#69512), and a
|
||||
# blank block stored in history replays on every turn → permanent 400.
|
||||
out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))}
|
||||
text_val = b.get("text", "")
|
||||
# Bedrock and strict Anthropic-compatible endpoints reject text
|
||||
# blocks where "text" is empty or whitespace-only (#69512). Drop the
|
||||
# blank block (the caller relocates any cache_control it carried and
|
||||
# falls back to a non-whitespace placeholder when nothing survives)
|
||||
# rather than coercing in place — a coerced "(empty)" block would be
|
||||
# model-visible noise next to surviving thinking/tool_use blocks.
|
||||
# Type-safe: captured blocks can carry text=None from an invalid
|
||||
# upstream payload, which a bare .strip() would crash on.
|
||||
if not isinstance(text_val, str) or not text_val.strip():
|
||||
return None
|
||||
out: Dict[str, Any] = {"type": "text", "text": text_val}
|
||||
# citations is input-valid ONLY when it's a non-empty list; the SDK
|
||||
# emits citations=None on responses, which the input schema rejects.
|
||||
cits = b.get("citations")
|
||||
|
|
@ -2011,9 +2019,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
parsed_args = {}
|
||||
redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args
|
||||
replayed: List[Dict[str, Any]] = []
|
||||
_relocated_replay_cache_control = None
|
||||
_dropped_blank_text = False
|
||||
for b in ordered_blocks:
|
||||
clean = _sanitize_replay_block(b)
|
||||
if clean is None:
|
||||
if isinstance(b, dict) and b.get("type") == "text":
|
||||
_dropped_blank_text = True
|
||||
if isinstance(b, dict) and isinstance(b.get("cache_control"), dict):
|
||||
# A dropped blank text block can still carry the cache
|
||||
# breakpoint marker -- relocate it rather than losing it.
|
||||
_relocated_replay_cache_control = b["cache_control"]
|
||||
continue
|
||||
if clean.get("type") == "tool_use":
|
||||
# Override raw (un-redacted) input with the redacted copy when
|
||||
|
|
@ -2023,20 +2039,68 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
if redacted is not None:
|
||||
clean["input"] = redacted
|
||||
replayed.append(clean)
|
||||
# When every text block was blank and nothing cacheable survived
|
||||
# (e.g. signed thinking + a blank text block, or a SOLE blank
|
||||
# cache-marked block), emit the non-whitespace placeholder so the
|
||||
# replayed message stays schema-valid (#69512) and a relocated cache
|
||||
# marker still has a carrier instead of being silently lost.
|
||||
_has_cacheable_replay = any(
|
||||
isinstance(b, dict) and b.get("type") in {"text", "tool_use"}
|
||||
for b in replayed
|
||||
)
|
||||
if not _has_cacheable_replay and (
|
||||
_dropped_blank_text or _relocated_replay_cache_control is not None
|
||||
):
|
||||
replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER})
|
||||
if replayed:
|
||||
if _relocated_replay_cache_control is not None:
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
replayed, _relocated_replay_cache_control
|
||||
)
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
replayed, m.get("cache_control")
|
||||
)
|
||||
return {"role": "assistant", "content": replayed}
|
||||
|
||||
blocks = _extract_preserved_thinking_blocks(m)
|
||||
# Cache markers dropped along with a blank block are relocated onto the
|
||||
# last surviving cacheable block below (via
|
||||
# _apply_assistant_cache_control_to_last_cacheable_block), rather than
|
||||
# lost -- prompt_caching.py's _apply_cache_marker() sets cache_control
|
||||
# directly on content[-1] for list content, so if that last part happens
|
||||
# to be blank text, dropping it silently would lose the breakpoint.
|
||||
_relocated_cache_control = None
|
||||
if content:
|
||||
if isinstance(content, list):
|
||||
converted_content = _convert_content_to_anthropic(content)
|
||||
if isinstance(converted_content, list):
|
||||
blocks.extend(converted_content)
|
||||
# Bedrock and strict Anthropic-compatible endpoints reject
|
||||
# text blocks where "text" is empty or whitespace-only. The
|
||||
# ordered-replay path enforces the same invariant via
|
||||
# _sanitize_replay_block(). Type-safe against ANY invalid
|
||||
# "text" value from an upstream payload -- None, or a
|
||||
# truthy non-string like an int -- not just None: checking
|
||||
# isinstance() first (rather than `blk.get("text") or ""`)
|
||||
# means a non-string value is treated as blank/invalid
|
||||
# instead of reaching .strip() and raising AttributeError.
|
||||
for blk in converted_content:
|
||||
_blk_text = blk.get("text") if isinstance(blk, dict) else None
|
||||
if (
|
||||
isinstance(blk, dict)
|
||||
and blk.get("type") == "text"
|
||||
and (not isinstance(_blk_text, str) or not _blk_text.strip())
|
||||
):
|
||||
if isinstance(blk.get("cache_control"), dict):
|
||||
_relocated_cache_control = blk["cache_control"]
|
||||
continue
|
||||
blocks.append(blk)
|
||||
else:
|
||||
blocks.append({"type": "text", "text": str(content)})
|
||||
# Scalar (non-list) content: a whitespace-only string is the
|
||||
# same invalid-payload case as an empty list block -- drop it
|
||||
# rather than emitting a blank text block.
|
||||
text_str = str(content)
|
||||
if text_str.strip():
|
||||
blocks.append({"type": "text", "text": text_str})
|
||||
for tc in m.get("tool_calls", []):
|
||||
if not tc or not isinstance(tc, dict):
|
||||
continue
|
||||
|
|
@ -2052,9 +2116,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
"name": fn.get("name", ""),
|
||||
"input": parsed_args,
|
||||
})
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
blocks, m.get("cache_control")
|
||||
)
|
||||
# Kimi's /coding endpoint (Anthropic protocol) requires assistant
|
||||
# tool-call messages to carry reasoning_content when thinking is
|
||||
# enabled server-side. Preserve it as a thinking block so Kimi
|
||||
|
|
@ -2080,19 +2141,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
)
|
||||
if isinstance(reasoning_content, str) and not _already_has_thinking:
|
||||
blocks.insert(0, {"type": "thinking", "thinking": reasoning_content})
|
||||
# Anthropic rejects empty assistant content
|
||||
effective = blocks or content
|
||||
if not effective or effective == "":
|
||||
effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
elif isinstance(effective, list):
|
||||
# The all-empty guard above misses a list that still contains a
|
||||
# whitespace-only text block (e.g. from a content array of blank parts,
|
||||
# or compression). Those also trip "text content blocks must contain
|
||||
# non-whitespace text" (#69512). Coerce text blocks in place; other
|
||||
# block types (thinking/tool_use/image) are left untouched.
|
||||
for blk in effective:
|
||||
if isinstance(blk, dict) and blk.get("type") == "text":
|
||||
blk["text"] = _safe_text(blk.get("text", ""))
|
||||
# Anthropic rejects empty assistant content. IMPORTANT: fall back only
|
||||
# to the placeholder, never to the raw `content` variable -- `content`
|
||||
# is the UNFILTERED original message content, and can itself be exactly
|
||||
# the blank/whitespace-only payload the filtering above just removed
|
||||
# (a sole blank text block, or scalar whitespace with no tool_calls).
|
||||
# `blocks or content` there would silently restore the invalid provider
|
||||
# payload this function exists to prevent (#69512).
|
||||
effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
# Applied here (after the empty-fallback resolution) rather than
|
||||
# earlier against `blocks` directly, so a cache_control relocated from
|
||||
# a dropped blank block that was the ONLY block still lands on the
|
||||
# (empty) placeholder instead of being silently lost when blocks was
|
||||
# empty at the point the marker would otherwise have been applied.
|
||||
if _relocated_cache_control is not None:
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
effective, _relocated_cache_control
|
||||
)
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
effective, m.get("cache_control")
|
||||
)
|
||||
return {"role": "assistant", "content": effective}
|
||||
|
||||
|
||||
|
|
@ -2825,6 +2893,7 @@ def create_anthropic_message(
|
|||
*,
|
||||
log_prefix: str = "",
|
||||
prefer_stream: bool = True,
|
||||
on_stream_event=None,
|
||||
) -> Any:
|
||||
"""Create an Anthropic message, aggregating via stream when available.
|
||||
|
||||
|
|
@ -2834,6 +2903,13 @@ def create_anthropic_message(
|
|||
crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to
|
||||
match the main turn path, falling back to ``create()`` only for providers
|
||||
that explicitly do not support streaming, such as restricted Bedrock roles.
|
||||
|
||||
``on_stream_event``: optional callable invoked once per streamed event
|
||||
(best-effort, exceptions swallowed). Lets callers report forward progress
|
||||
to liveness watchdogs — e.g. the auxiliary compression path ticking its
|
||||
progress hook so a slow-but-generating summary model isn't treated as
|
||||
hung. Only fires on the streaming path; the ``create()`` fallback has no
|
||||
events to report.
|
||||
"""
|
||||
sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix)
|
||||
|
||||
|
|
@ -2844,6 +2920,18 @@ def create_anthropic_message(
|
|||
stream_kwargs.pop("stream", None)
|
||||
try:
|
||||
with stream_fn(**stream_kwargs) as stream:
|
||||
if callable(on_stream_event):
|
||||
# Consume the event stream manually so each event can
|
||||
# tick the caller's progress callback; get_final_message
|
||||
# then returns the accumulated snapshot.
|
||||
for _event in stream:
|
||||
try:
|
||||
on_stream_event(_event)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"%son_stream_event callback failed",
|
||||
log_prefix, exc_info=True,
|
||||
)
|
||||
return stream.get_final_message()
|
||||
except Exception as exc:
|
||||
if not _is_stream_unavailable_error(exc):
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ import time
|
|||
import uuid
|
||||
from pathlib import Path # noqa: F401 — used by test mocks
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
|
||||
from urllib.parse import urlparse, parse_qs, urlunparse
|
||||
|
||||
# NOTE: `from openai import OpenAI` is deliberately NOT at module top — the
|
||||
|
|
@ -249,6 +249,50 @@ def aux_interrupt_protection(active: bool = True):
|
|||
_aux_interrupt_protection.active = prev
|
||||
|
||||
|
||||
# ── Forward-progress hook for streamed auxiliary calls ───────────────────
|
||||
# Long auxiliary calls (context compression is the prime case) are watched by
|
||||
# wall-clock deadlines in their hosts (gateway session hygiene). A fixed
|
||||
# deadline punishes SLOW summary models exactly as hard as HUNG ones: a
|
||||
# reasoning model happily streaming a large summary is killed mid-generation.
|
||||
# This thread-local hook lets the host observe liveness instead: the wire
|
||||
# consumers below tick it on every streamed token/SSE event, and the host
|
||||
# extends its deadline while tokens are moving (see gateway/run.py session
|
||||
# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the
|
||||
# call topology — the aux call and its stream consumption run synchronously
|
||||
# on the thread that installed the hook.
|
||||
_aux_progress = threading.local()
|
||||
|
||||
|
||||
def _notify_aux_progress() -> None:
|
||||
"""Tick the installed forward-progress hook, if any. Never raises."""
|
||||
hook = getattr(_aux_progress, "hook", None)
|
||||
if hook is None:
|
||||
return
|
||||
try:
|
||||
hook()
|
||||
except Exception:
|
||||
logger.debug("aux progress hook failed", exc_info=True)
|
||||
|
||||
|
||||
def _aux_progress_active() -> bool:
|
||||
return getattr(_aux_progress, "hook", None) is not None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def aux_progress_hook(hook):
|
||||
"""Install *hook* as the current thread's aux forward-progress callback.
|
||||
|
||||
``hook=None`` is a no-op passthrough so callers can wire it
|
||||
unconditionally. Re-entrant-safe: restores the previous hook on exit.
|
||||
"""
|
||||
prev = getattr(_aux_progress, "hook", None)
|
||||
_aux_progress.hook = hook if callable(hook) else prev
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_aux_progress.hook = prev
|
||||
|
||||
|
||||
def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
|
||||
"""Return False instead of raising when a patched symbol is not a type."""
|
||||
try:
|
||||
|
|
@ -1060,16 +1104,29 @@ class _CodexCompletionsAdapter:
|
|||
# key in extra_body (not top-level) and GitHub/Copilot Responses opts
|
||||
# out of cache-key routing entirely — for those hosts, skip it here.
|
||||
try:
|
||||
from agent.transports.codex import _content_cache_key
|
||||
from agent.transports.codex import (
|
||||
_content_cache_key,
|
||||
_default_prompt_cache_retention_for_request,
|
||||
)
|
||||
from utils import base_url_host_matches
|
||||
|
||||
_host_src = str(getattr(self._client, "base_url", "") or "")
|
||||
_is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai")
|
||||
_is_github = base_url_host_matches(_host_src, "githubcopilot.com")
|
||||
_is_github = (
|
||||
base_url_host_matches(_host_src, "githubcopilot.com")
|
||||
or base_url_host_matches(_host_src, "models.github.ai")
|
||||
)
|
||||
if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs:
|
||||
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"))
|
||||
if _cache_key:
|
||||
resp_kwargs["prompt_cache_key"] = _cache_key
|
||||
if "prompt_cache_retention" not in resp_kwargs:
|
||||
_cache_retention = _default_prompt_cache_retention_for_request(
|
||||
model,
|
||||
_host_src,
|
||||
)
|
||||
if _cache_retention:
|
||||
resp_kwargs["prompt_cache_retention"] = _cache_retention
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True
|
||||
|
|
@ -1151,6 +1208,10 @@ class _CodexCompletionsAdapter:
|
|||
def _on_each_event(_event: Any) -> None:
|
||||
# Re-check timeout/cancellation per event, matching the
|
||||
# cadence the old in-line ``_check_cancelled()`` used.
|
||||
# Each SSE event is also forward progress for hosts watching
|
||||
# a progress hook (gateway session hygiene): a reasoning
|
||||
# model streaming a long summary must not look hung.
|
||||
_notify_aux_progress()
|
||||
_check_cancelled()
|
||||
|
||||
event_stream = self._client.responses.create(**stream_kwargs)
|
||||
|
|
@ -1392,7 +1453,18 @@ class _AnthropicCompletionsAdapter:
|
|||
existing = {}
|
||||
anthropic_kwargs["extra_body"] = {**existing, **passthrough}
|
||||
|
||||
response = create_anthropic_message(self._client, anthropic_kwargs)
|
||||
response = create_anthropic_message(
|
||||
self._client,
|
||||
anthropic_kwargs,
|
||||
# Tick the aux forward-progress hook per streamed event so hosts
|
||||
# watching liveness (gateway session hygiene) don't kill a
|
||||
# slow-but-generating summary model. No-op when no hook is
|
||||
# installed (None keeps the fast get_final_message path).
|
||||
on_stream_event=(
|
||||
(lambda _event: _notify_aux_progress())
|
||||
if _aux_progress_active() else None
|
||||
),
|
||||
)
|
||||
_transport = get_transport("anthropic_messages")
|
||||
_nr = _transport.normalize_response(
|
||||
response, strip_tool_prefix=self._is_oauth
|
||||
|
|
@ -1705,7 +1777,7 @@ def _read_nous_auth() -> Optional[dict]:
|
|||
try:
|
||||
if not _AUTH_JSON_PATH.is_file():
|
||||
return None
|
||||
data = json.loads(_AUTH_JSON_PATH.read_text())
|
||||
data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8"))
|
||||
if data.get("active_provider") != "nous":
|
||||
return None
|
||||
provider = data.get("providers", {}).get("nous", {})
|
||||
|
|
@ -2489,16 +2561,18 @@ def _relay_sync_completion(
|
|||
*,
|
||||
provider: str | None = None,
|
||||
api_mode: str | None = None,
|
||||
create: Callable[[dict[str, Any]], Any] | None = None,
|
||||
) -> Any:
|
||||
callback = create or (lambda request: client.chat.completions.create(**request))
|
||||
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
|
||||
if route is None:
|
||||
return client.chat.completions.create(**kwargs)
|
||||
return callback(kwargs)
|
||||
provider_name, fallback_model, metadata = route
|
||||
from agent import relay_llm
|
||||
|
||||
return relay_llm.execute_current(
|
||||
kwargs,
|
||||
lambda request: client.chat.completions.create(**request),
|
||||
callback,
|
||||
name=provider_name,
|
||||
model_name=str(kwargs.get("model") or fallback_model),
|
||||
metadata=metadata,
|
||||
|
|
@ -2512,16 +2586,18 @@ async def _relay_async_completion(
|
|||
*,
|
||||
provider: str | None = None,
|
||||
api_mode: str | None = None,
|
||||
create: Callable[[dict[str, Any]], Any] | None = None,
|
||||
) -> Any:
|
||||
callback = create or (lambda request: client.chat.completions.create(**request))
|
||||
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
|
||||
if route is None:
|
||||
return await client.chat.completions.create(**kwargs)
|
||||
return await callback(kwargs)
|
||||
provider_name, fallback_model, metadata = route
|
||||
from agent import relay_llm
|
||||
|
||||
return await relay_llm.execute_current_async(
|
||||
kwargs,
|
||||
lambda request: client.chat.completions.create(**request),
|
||||
callback,
|
||||
name=provider_name,
|
||||
model_name=str(kwargs.get("model") or fallback_model),
|
||||
metadata=metadata,
|
||||
|
|
@ -3807,6 +3883,7 @@ def _retry_same_provider_sync(
|
|||
effective_timeout: float,
|
||||
effective_extra_body: dict,
|
||||
reasoning_config: Optional[dict],
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
) -> Any:
|
||||
if task == "vision":
|
||||
_, retry_client, retry_model = resolve_vision_provider_client(
|
||||
|
|
@ -3842,7 +3919,13 @@ def _retry_same_provider_sync(
|
|||
extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=retry_base or resolved_base_url,
|
||||
task=task,
|
||||
)
|
||||
# Preserve per-request attribution headers (e.g. Copilot's
|
||||
# ``x-initiator: user``) across the rebuilt-client retry — dropping them
|
||||
# here would let a recovery retry silently lose capability gating (#60293).
|
||||
if extra_headers:
|
||||
retry_kwargs["extra_headers"] = dict(extra_headers)
|
||||
if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
|
||||
retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
|
||||
return _validate_llm_response(
|
||||
|
|
@ -3872,6 +3955,7 @@ async def _retry_same_provider_async(
|
|||
effective_timeout: float,
|
||||
effective_extra_body: dict,
|
||||
reasoning_config: Optional[dict],
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
) -> Any:
|
||||
if task == "vision":
|
||||
_, retry_client, retry_model = resolve_vision_provider_client(
|
||||
|
|
@ -3907,7 +3991,12 @@ async def _retry_same_provider_async(
|
|||
extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=retry_base or resolved_base_url,
|
||||
task=task,
|
||||
)
|
||||
# Preserve per-request attribution headers across the rebuilt-client
|
||||
# retry — see the sync variant above (#60293).
|
||||
if extra_headers:
|
||||
retry_kwargs["extra_headers"] = dict(extra_headers)
|
||||
if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
|
||||
retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
|
||||
return _validate_llm_response(
|
||||
|
|
@ -3988,6 +4077,24 @@ def _refresh_provider_credentials(provider: str) -> bool:
|
|||
return False
|
||||
_evict_cached_clients(normalized)
|
||||
return True
|
||||
if normalized == "vertex":
|
||||
# Mirrors run_agent.py's _try_refresh_vertex_client_credentials
|
||||
# for the main conversation loop. Without this branch, an
|
||||
# auxiliary Vertex client (vision, title generation, reflection,
|
||||
# context compression, ...) that 401s on its ~1h token expiry
|
||||
# falls through to the final `return False` below: the stale
|
||||
# client is never evicted from _client_cache (whose cache key
|
||||
# ignores the rotating bearer token), so every subsequent
|
||||
# auxiliary Vertex call keeps 401ing until process restart.
|
||||
from agent.vertex_adapter import get_vertex_config
|
||||
|
||||
token, base_url = get_vertex_config()
|
||||
if not isinstance(token, str) or not token.strip():
|
||||
return False
|
||||
if not isinstance(base_url, str) or not base_url.strip():
|
||||
return False
|
||||
_evict_cached_clients(normalized)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc)
|
||||
return False
|
||||
|
|
@ -4101,7 +4208,7 @@ def _call_fallback_candidate_sync(
|
|||
temperature=temperature, max_tokens=max_tokens,
|
||||
tools=tools, timeout=effective_timeout,
|
||||
extra_body=effective_extra_body, reasoning_config=reasoning_config,
|
||||
base_url=fb_base)
|
||||
base_url=fb_base, task=task)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
_relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task)
|
||||
|
|
@ -4118,7 +4225,7 @@ def _call_fallback_candidate_sync(
|
|||
tools=tools, timeout=effective_timeout,
|
||||
extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
|
||||
base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
_relay_sync_completion(
|
||||
|
|
@ -4173,7 +4280,7 @@ async def _call_fallback_candidate_async(
|
|||
temperature=temperature, max_tokens=max_tokens,
|
||||
tools=tools, timeout=effective_timeout,
|
||||
extra_body=effective_extra_body, reasoning_config=reasoning_config,
|
||||
base_url=fb_base)
|
||||
base_url=fb_base, task=task)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
await _relay_async_completion(
|
||||
|
|
@ -4197,7 +4304,7 @@ async def _call_fallback_candidate_async(
|
|||
tools=tools, timeout=effective_timeout,
|
||||
extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
|
||||
base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
await _relay_async_completion(
|
||||
|
|
@ -6965,6 +7072,7 @@ def _build_call_kwargs(
|
|||
extra_body: Optional[dict] = None,
|
||||
reasoning_config: Optional[dict] = None,
|
||||
base_url: Optional[str] = None,
|
||||
task: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Build kwargs for .chat.completions.create() with model/provider adjustments."""
|
||||
kwargs: Dict[str, Any] = {
|
||||
|
|
@ -7019,11 +7127,32 @@ def _build_call_kwargs(
|
|||
_provider_norm in {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"}
|
||||
or base_url_host_matches(_effective_base, "integrate.api.nvidia.com")
|
||||
)
|
||||
_is_moa = bool(task) and str(task) == "moa_reference"
|
||||
# Gemini's native generateContent maps max_tokens → maxOutputTokens and,
|
||||
# when it is omitted, applies a fixed 65,535-token ceiling rather than
|
||||
# "the model's full budget" (see gemini_native_adapter.build_gemini_request).
|
||||
# So an explicit cap is both safe and the ONLY way to honor it here —
|
||||
# dropping max_tokens silently makes MoA's reference_max_tokens a no-op
|
||||
# for gemini advisors (they run effectively uncapped).
|
||||
_is_gemini_native = _provider_norm in {
|
||||
"gemini", "google", "google-gemini", "google-ai-studio",
|
||||
}
|
||||
if not _is_gemini_native and _effective_base:
|
||||
try:
|
||||
from agent.gemini_native_adapter import is_native_gemini_base_url
|
||||
_is_gemini_native = is_native_gemini_base_url(_effective_base)
|
||||
except Exception:
|
||||
pass
|
||||
if (
|
||||
_is_anthropic_compat_endpoint(provider, _effective_base)
|
||||
or _is_nvidia_nim
|
||||
or _is_moa
|
||||
or _is_gemini_native
|
||||
):
|
||||
kwargs["max_tokens"] = max_tokens
|
||||
# Use auxiliary_max_tokens_param() so models that require
|
||||
# max_completion_tokens (GPT-5 family, Copilot) get the right
|
||||
# parameter name instead of a hardcoded max_tokens that 400s.
|
||||
kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model))
|
||||
|
||||
if tools:
|
||||
# Defensive dedup: providers like Google Vertex, Azure, and Bedrock
|
||||
|
|
@ -7266,6 +7395,346 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any:
|
|||
return value
|
||||
|
||||
|
||||
# ── Streamed aggregation for progress-hooked auxiliary calls ─────────────
|
||||
# When a forward-progress hook is installed (aux_progress_hook — today only
|
||||
# by context compression), the primary chat.completions attempt is upgraded
|
||||
# to a streamed request that is aggregated back into a complete response.
|
||||
# Two effects, both deliberate:
|
||||
# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead
|
||||
# of a total budget (httpx applies the read timeout per stream read), so
|
||||
# a slow-but-generating summary model is never killed mid-generation
|
||||
# while tokens are moving — only a genuinely silent connection dies.
|
||||
# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs
|
||||
# (gateway session hygiene) extend their deadlines on liveness instead
|
||||
# of guessing with a fixed wall clock.
|
||||
# A total ceiling still bounds the pathological 1-token-per-idle-window
|
||||
# stream; see _aux_stream_total_ceiling().
|
||||
|
||||
_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
|
||||
_AUX_STREAM_CEILING_MULTIPLIER = 4.0
|
||||
|
||||
|
||||
def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
|
||||
"""Absolute wall-clock bound for a progress-hooked streamed aux call.
|
||||
|
||||
Generous by design — the idle timeout is the real guard; this only stops
|
||||
a degenerate stream that trickles one token per idle window forever.
|
||||
"""
|
||||
try:
|
||||
timeout = float(effective_timeout) if effective_timeout is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
timeout = 0.0
|
||||
return max(_AUX_STREAM_CEILING_FLOOR_SECONDS,
|
||||
_AUX_STREAM_CEILING_MULTIPLIER * timeout)
|
||||
|
||||
|
||||
def _client_streams_internally(client: Any) -> bool:
|
||||
"""Wire adapters that consume a stream inside .create() already tick the
|
||||
progress hook themselves (Codex per SSE event, Anthropic per stream
|
||||
event); Bedrock's Converse shim cannot stream at all. None of them
|
||||
accept chat-completions ``stream=True`` semantics from us."""
|
||||
return isinstance(client, (
|
||||
CodexAuxiliaryClient,
|
||||
AnthropicAuxiliaryClient,
|
||||
BedrockAuxiliaryClient,
|
||||
))
|
||||
|
||||
|
||||
def _is_streaming_rejected_error(exc: Exception) -> bool:
|
||||
"""Provider explicitly refused a streamed chat.completions request."""
|
||||
err = str(exc).lower()
|
||||
if "stream_options" in err:
|
||||
return True
|
||||
return "stream" in err and (
|
||||
"not supported" in err
|
||||
or "unsupported" in err
|
||||
or "not allowed" in err
|
||||
or "disabled" in err
|
||||
)
|
||||
|
||||
|
||||
def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
|
||||
"""Detect providers that only accept streaming (non-stream = HTTP 400).
|
||||
|
||||
Some OpenAI-compatible endpoints reject non-streaming chat requests
|
||||
outright — e.g. Tencent Copilot returns
|
||||
``{"code": 11101, "msg": "Non-stream chat request is currently not
|
||||
supported"}``. The main conversation loop already streams, so interactive
|
||||
chat works; auxiliary tasks (title generation, compression, web extract)
|
||||
used the non-streaming path and failed on every call. When this returns
|
||||
True the auxiliary client sends ``stream=True`` and aggregates the chunks
|
||||
itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686).
|
||||
|
||||
Beyond the known-host list, users can mark ANY custom endpoint as
|
||||
stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml
|
||||
(list of substrings matched against the endpoint URL).
|
||||
"""
|
||||
_url = str(base_url or "").lower()
|
||||
if not _url:
|
||||
return False
|
||||
# Tencent Copilot — "Non-stream chat request is currently not supported"
|
||||
if base_url_host_matches(_url, "copilot.tencent.com"):
|
||||
return True
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
aux_cfg = (load_config() or {}).get("auxiliary", {})
|
||||
markers = aux_cfg.get("stream_only_base_urls") or []
|
||||
if isinstance(markers, (list, tuple)):
|
||||
for marker in markers:
|
||||
if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url:
|
||||
return True
|
||||
except Exception:
|
||||
# Config read is best-effort; never break an aux call over it.
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _create_with_progress(
|
||||
client: Any,
|
||||
kwargs: Dict[str, Any],
|
||||
task: Optional[str] = None,
|
||||
*,
|
||||
force_stream: bool = False,
|
||||
) -> Any:
|
||||
"""chat.completions.create() that streams when a progress hook is active
|
||||
or the provider only accepts streamed requests.
|
||||
|
||||
Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when
|
||||
neither trigger applies (every existing caller/task) or when the client's
|
||||
wire adapter streams internally. With a hook + a chunk-capable client,
|
||||
the request is sent with ``stream=True`` and aggregated, ticking the hook
|
||||
per chunk — so the configured ``timeout`` acts per stream read (idle)
|
||||
rather than as a total budget, and outer liveness watchdogs see tokens
|
||||
moving. ``force_stream=True`` (stream-only providers such as Tencent
|
||||
Copilot — credit @kudi88, PR #60686) takes the same streamed path even
|
||||
without a hook. Providers that reject the streamed request fall back to
|
||||
the plain non-streaming call — except under ``force_stream``, where a
|
||||
stream-only provider rejects the plain call by definition, so the
|
||||
original error is surfaced to the normal recovery chains instead.
|
||||
"""
|
||||
_notify_aux_progress() # request dispatched counts as progress
|
||||
if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
|
||||
return client.chat.completions.create(**kwargs)
|
||||
|
||||
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
|
||||
stream_kwargs = dict(kwargs)
|
||||
stream_kwargs["stream"] = True
|
||||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||||
try:
|
||||
chunks = client.chat.completions.create(**stream_kwargs)
|
||||
except Exception as exc:
|
||||
# Genuine provider failures (auth, credit, rate limit, network) are
|
||||
# not streaming's fault — surface them unchanged so the existing
|
||||
# recovery chains (credential refresh, pool rotation, provider
|
||||
# fallback) see the same error they would on a plain call.
|
||||
if (
|
||||
force_stream
|
||||
or _is_transient_transport_error(exc)
|
||||
or _is_auth_error(exc)
|
||||
or _is_payment_error(exc)
|
||||
or _is_rate_limit_error(exc)
|
||||
):
|
||||
raise
|
||||
# Anything else may be a streaming-specific rejection (explicit
|
||||
# "stream not supported", stream_options 400, or an idiosyncratic
|
||||
# 4xx). Retry non-streaming once; if the request itself is bad the
|
||||
# plain call reproduces the real error for the normal except-chains.
|
||||
logger.debug(
|
||||
"Auxiliary %s: streamed request failed (%s); retrying "
|
||||
"non-streaming", task or "call", exc,
|
||||
)
|
||||
return client.chat.completions.create(**kwargs)
|
||||
|
||||
# Some shims (MoA virtual provider under quiet mode, defensive adapters)
|
||||
# return a complete response even when stream=True was requested.
|
||||
if hasattr(chunks, "choices"):
|
||||
_notify_aux_progress()
|
||||
return chunks
|
||||
return _aggregate_chat_stream(
|
||||
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
|
||||
)
|
||||
|
||||
|
||||
def _aggregate_chat_stream(
|
||||
chunks: Any,
|
||||
*,
|
||||
model: str = "",
|
||||
total_ceiling: Optional[float] = None,
|
||||
) -> Any:
|
||||
"""Consume a chat.completions chunk stream into a complete response.
|
||||
|
||||
Ticks the thread-local aux progress hook on every chunk. Raises
|
||||
TimeoutError when *total_ceiling* seconds elapse before the stream
|
||||
finishes — phrased with "timed out" so existing timeout classification
|
||||
(``_is_timeout_error``) treats it exactly like a request timeout.
|
||||
Accumulation is shared with the async mirror via
|
||||
:class:`_ChatStreamAccumulator`.
|
||||
"""
|
||||
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
|
||||
try:
|
||||
for chunk in chunks:
|
||||
acc.feed(chunk)
|
||||
finally:
|
||||
close_fn = getattr(chunks, "close", None)
|
||||
if callable(close_fn):
|
||||
try:
|
||||
close_fn()
|
||||
except Exception:
|
||||
pass
|
||||
return acc.finish()
|
||||
|
||||
|
||||
class _ChatStreamAccumulator:
|
||||
"""Shared per-chunk accumulation for sync and async stream aggregation.
|
||||
|
||||
Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async
|
||||
consumer below cannot drift from the sync one (same content/reasoning/
|
||||
tool-call delta reassembly, same "timed out" ceiling phrasing).
|
||||
"""
|
||||
|
||||
def __init__(self, model: str = "", total_ceiling: Optional[float] = None):
|
||||
self._started = time.monotonic()
|
||||
self._total_ceiling = total_ceiling
|
||||
self.content_parts: List[str] = []
|
||||
self.reasoning_parts: List[str] = []
|
||||
self.tool_calls_acc: Dict[int, Dict[str, Any]] = {}
|
||||
self.finish_reason = None
|
||||
self.usage = None
|
||||
self.resp_id = ""
|
||||
self.resp_model = model or ""
|
||||
|
||||
def feed(self, chunk: Any) -> None:
|
||||
_notify_aux_progress()
|
||||
if (
|
||||
self._total_ceiling is not None
|
||||
and (time.monotonic() - self._started) >= self._total_ceiling
|
||||
):
|
||||
raise TimeoutError(
|
||||
f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
|
||||
"total ceiling (stream still open but over budget)"
|
||||
)
|
||||
self.resp_id = getattr(chunk, "id", None) or self.resp_id
|
||||
self.resp_model = getattr(chunk, "model", None) or self.resp_model
|
||||
chunk_usage = getattr(chunk, "usage", None)
|
||||
if chunk_usage:
|
||||
self.usage = chunk_usage
|
||||
choices = getattr(chunk, "choices", None) or []
|
||||
if not choices:
|
||||
return
|
||||
choice = choices[0]
|
||||
self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
|
||||
delta = getattr(choice, "delta", None)
|
||||
if delta is None:
|
||||
return
|
||||
piece = getattr(delta, "content", None)
|
||||
if piece:
|
||||
self.content_parts.append(piece)
|
||||
reasoning_piece = (
|
||||
getattr(delta, "reasoning", None)
|
||||
or getattr(delta, "reasoning_content", None)
|
||||
)
|
||||
if reasoning_piece and isinstance(reasoning_piece, str):
|
||||
self.reasoning_parts.append(reasoning_piece)
|
||||
for tc in (getattr(delta, "tool_calls", None) or []):
|
||||
idx = getattr(tc, "index", 0) or 0
|
||||
acc = self.tool_calls_acc.setdefault(
|
||||
idx, {"id": "", "name": "", "arguments": []}
|
||||
)
|
||||
if getattr(tc, "id", None):
|
||||
acc["id"] = tc.id
|
||||
fn = getattr(tc, "function", None)
|
||||
if fn is not None:
|
||||
if getattr(fn, "name", None):
|
||||
acc["name"] = fn.name
|
||||
if getattr(fn, "arguments", None):
|
||||
acc["arguments"].append(fn.arguments)
|
||||
|
||||
def finish(self) -> Any:
|
||||
tool_calls = None
|
||||
if self.tool_calls_acc:
|
||||
tool_calls = [
|
||||
SimpleNamespace(
|
||||
id=acc["id"],
|
||||
type="function",
|
||||
function=SimpleNamespace(
|
||||
name=acc["name"],
|
||||
arguments="".join(acc["arguments"]),
|
||||
),
|
||||
)
|
||||
for _idx, acc in sorted(self.tool_calls_acc.items())
|
||||
]
|
||||
message = SimpleNamespace(
|
||||
role="assistant",
|
||||
content="".join(self.content_parts),
|
||||
tool_calls=tool_calls,
|
||||
reasoning="".join(self.reasoning_parts) or None,
|
||||
)
|
||||
choice = SimpleNamespace(
|
||||
index=0,
|
||||
message=message,
|
||||
finish_reason=self.finish_reason or "stop",
|
||||
)
|
||||
return SimpleNamespace(
|
||||
id=self.resp_id,
|
||||
model=self.resp_model,
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=self.usage,
|
||||
)
|
||||
|
||||
|
||||
async def _aggregate_chat_stream_async(
|
||||
chunks: Any,
|
||||
*,
|
||||
model: str = "",
|
||||
total_ceiling: Optional[float] = None,
|
||||
) -> Any:
|
||||
"""Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer).
|
||||
|
||||
The AsyncOpenAI stream contract is an async iterator — consuming it with
|
||||
the sync helper raises. Same accumulation and ceiling semantics via
|
||||
:class:`_ChatStreamAccumulator`.
|
||||
"""
|
||||
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
|
||||
try:
|
||||
async for chunk in chunks:
|
||||
acc.feed(chunk)
|
||||
finally:
|
||||
close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None)
|
||||
if callable(close_fn):
|
||||
try:
|
||||
result = close_fn()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
except Exception:
|
||||
pass
|
||||
return acc.finish()
|
||||
|
||||
|
||||
async def _acreate_with_stream(
|
||||
client: Any,
|
||||
kwargs: Dict[str, Any],
|
||||
task: Optional[str] = None,
|
||||
) -> Any:
|
||||
"""Async chat.completions.create() for stream-only providers.
|
||||
|
||||
Sends ``stream=True`` and aggregates the async chunk stream into a
|
||||
complete response (credit @kudi88, PR #60686 — async contract fixed to
|
||||
``async for`` and tool-call deltas preserved per sweeper review).
|
||||
"""
|
||||
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
|
||||
stream_kwargs = dict(kwargs)
|
||||
stream_kwargs["stream"] = True
|
||||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||||
chunks = await client.chat.completions.create(**stream_kwargs)
|
||||
# Defensive: shims may hand back a complete response despite stream=True.
|
||||
if hasattr(chunks, "choices"):
|
||||
return chunks
|
||||
return await _aggregate_chat_stream_async(
|
||||
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
|
||||
)
|
||||
|
||||
|
||||
@_relay_auxiliary_call
|
||||
def call_llm(
|
||||
task: str = None,
|
||||
|
|
@ -7282,6 +7751,7 @@ def call_llm(
|
|||
timeout: float = None,
|
||||
extra_body: dict = None,
|
||||
reasoning_config: Optional[dict] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
api_mode: str = None,
|
||||
stream: bool = False,
|
||||
stream_options: dict = None,
|
||||
|
|
@ -7307,6 +7777,9 @@ def call_llm(
|
|||
extra_body: Additional request body fields.
|
||||
reasoning_config: Optional Hermes reasoning config for direct model calls
|
||||
such as MoA reference/aggregator slots.
|
||||
extra_headers: Additional per-request HTTP headers. These override
|
||||
client-level defaults for providers that gate capabilities on
|
||||
request attribution (for example Copilot's ``x-initiator``).
|
||||
stream: When True, return the raw SDK streaming iterator instead of a
|
||||
validated complete response. The caller is responsible for consuming
|
||||
chunks (and for any fallback). Used by the MoA aggregator so its
|
||||
|
|
@ -7424,7 +7897,9 @@ def call_llm(
|
|||
temperature=temperature, max_tokens=max_tokens,
|
||||
tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=_base_info or resolved_base_url)
|
||||
base_url=_base_info or resolved_base_url, task=task)
|
||||
if extra_headers:
|
||||
kwargs["extra_headers"] = dict(extra_headers)
|
||||
|
||||
# Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
|
||||
_client_base = str(getattr(client, "base_url", "") or "")
|
||||
|
|
@ -7476,7 +7951,16 @@ def call_llm(
|
|||
kwargs,
|
||||
provider=resolved_provider,
|
||||
api_mode=resolved_api_mode,
|
||||
), task,
|
||||
create=lambda request: _create_with_progress(
|
||||
client,
|
||||
request,
|
||||
task,
|
||||
force_stream=_provider_requires_stream(
|
||||
resolved_provider, _base_info or resolved_base_url,
|
||||
),
|
||||
),
|
||||
),
|
||||
task,
|
||||
provider=resolved_provider, base_url=_base_info)
|
||||
except Exception as transient_err:
|
||||
if not _is_transient_transport_error(transient_err):
|
||||
|
|
@ -7514,7 +7998,17 @@ def call_llm(
|
|||
kwargs,
|
||||
provider=resolved_provider,
|
||||
api_mode=resolved_api_mode,
|
||||
), task)
|
||||
create=lambda request: _create_with_progress(
|
||||
client,
|
||||
request,
|
||||
task,
|
||||
force_stream=_provider_requires_stream(
|
||||
resolved_provider,
|
||||
_base_info or resolved_base_url,
|
||||
),
|
||||
),
|
||||
),
|
||||
task)
|
||||
except Exception as retry_transient:
|
||||
if not _is_transient_transport_error(retry_transient):
|
||||
raise
|
||||
|
|
@ -7718,6 +8212,7 @@ def call_llm(
|
|||
effective_timeout=effective_timeout,
|
||||
effective_extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
|
||||
# ── Same-provider credential-pool recovery ─────────────────────
|
||||
|
|
@ -7766,6 +8261,7 @@ def call_llm(
|
|||
effective_timeout=effective_timeout,
|
||||
effective_extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
except Exception as retry2_err:
|
||||
# The rotated key also hit a quota/auth wall. Mark it
|
||||
|
|
@ -8091,7 +8587,7 @@ async def async_call_llm(
|
|||
temperature=temperature, max_tokens=max_tokens,
|
||||
tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url=_client_base or resolved_base_url)
|
||||
base_url=_client_base or resolved_base_url, task=task)
|
||||
|
||||
# Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
|
||||
if _is_anthropic_compat_endpoint(resolved_provider, _client_base):
|
||||
|
|
@ -8101,6 +8597,22 @@ async def async_call_llm(
|
|||
# Retry ONCE on the same provider for a transient transport blip
|
||||
# before the except-chain escalates to fallback — see call_llm()
|
||||
# for the rationale. (PR #16587)
|
||||
_force_stream_async = (
|
||||
_provider_requires_stream(
|
||||
resolved_provider, _client_base or resolved_base_url,
|
||||
)
|
||||
and not isinstance(client, (
|
||||
AsyncCodexAuxiliaryClient,
|
||||
AsyncAnthropicAuxiliaryClient,
|
||||
AsyncBedrockAuxiliaryClient,
|
||||
))
|
||||
)
|
||||
|
||||
async def _acreate(_kwargs: Dict[str, Any]) -> Any:
|
||||
if _force_stream_async:
|
||||
return await _acreate_with_stream(client, _kwargs, task)
|
||||
return await client.chat.completions.create(**_kwargs)
|
||||
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
await _relay_async_completion(
|
||||
|
|
@ -8108,7 +8620,9 @@ async def async_call_llm(
|
|||
kwargs,
|
||||
provider=resolved_provider,
|
||||
api_mode=resolved_api_mode,
|
||||
), task,
|
||||
create=_acreate,
|
||||
),
|
||||
task,
|
||||
provider=resolved_provider, base_url=_client_base)
|
||||
except Exception as transient_err:
|
||||
if not _is_transient_transport_error(transient_err):
|
||||
|
|
@ -8134,7 +8648,9 @@ async def async_call_llm(
|
|||
kwargs,
|
||||
provider=resolved_provider,
|
||||
api_mode=resolved_api_mode,
|
||||
), task)
|
||||
create=_acreate,
|
||||
),
|
||||
task)
|
||||
except Exception as first_err:
|
||||
if "temperature" in kwargs and _is_unsupported_temperature_error(first_err):
|
||||
retry_kwargs = dict(kwargs)
|
||||
|
|
|
|||
|
|
@ -209,7 +209,10 @@ _SKILL_REVIEW_PROMPT = (
|
|||
"conversation for skills the user loaded via /skill-name or you "
|
||||
"read via skill_view. If any of them covers the territory of the "
|
||||
"new learning, PATCH that one first. It is the skill that was in "
|
||||
"play, so it's the right one to extend.\n"
|
||||
"play, so it's the right one to extend — but only if it is "
|
||||
"curator-managed. Bundled, hub, pinned, and user-owned skills are "
|
||||
"off-limits to you no matter how relevant (see Protected skills "
|
||||
"below); for those, fall through to the next option.\n"
|
||||
" 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). "
|
||||
"If no loaded skill fits but an existing class-level skill does, "
|
||||
"patch it. Add a subsection, a pitfall, or broaden a trigger.\n"
|
||||
|
|
@ -251,10 +254,18 @@ _SKILL_REVIEW_PROMPT = (
|
|||
"Protected skills (DO NOT edit these):\n"
|
||||
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
|
||||
" • Hub-installed skills (installed via 'hermes skills install').\n"
|
||||
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
|
||||
"pin only blocks deletion/archive/consolidation by the curator, not "
|
||||
"content updates. Patch them when a pitfall or missing step turns up, "
|
||||
"same as any other agent-created skill.\n"
|
||||
" • Skills in skills.external_dirs (externally owned).\n"
|
||||
" • PINNED skills (marked via 'hermes curator pin'). You are an "
|
||||
"autonomous no-user-present actor, so pin blocks your writes too — "
|
||||
"content updates included. Only the user, in a foreground session, "
|
||||
"can change a pinned skill.\n"
|
||||
" • USER-OWNED skills — anything not curator-managed. A skill the "
|
||||
"user hand-wrote, installed by URL, or asked a foreground agent to "
|
||||
"create is theirs, not yours; your writes to it WILL be refused. "
|
||||
"This includes skills that were loaded or consulted this session: "
|
||||
"being in play does not make one yours to edit. If such a skill is "
|
||||
"wrong or outdated, say so in your reply and recommend "
|
||||
"'hermes curator adopt <name>' — do not try to patch it.\n"
|
||||
"If the only skills that need updating are protected, say\n"
|
||||
"'Nothing to save.' and stop.\n\n"
|
||||
"Do NOT capture (these become persistent self-imposed constraints "
|
||||
|
|
@ -309,7 +320,9 @@ _COMBINED_REVIEW_PROMPT = (
|
|||
" 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were "
|
||||
"loaded via /skill-name or skill_view in the conversation. If one "
|
||||
"of them covers the learning, PATCH it first. It was in play; "
|
||||
"it's the right place.\n"
|
||||
"it's the right place — provided it is curator-managed. Protected "
|
||||
"and user-owned skills are off-limits however relevant; fall "
|
||||
"through when one of those is the best fit.\n"
|
||||
" 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to "
|
||||
"find the right one). Patch it.\n"
|
||||
" 3. ADD A SUPPORT FILE under an existing umbrella via "
|
||||
|
|
@ -337,10 +350,15 @@ _COMBINED_REVIEW_PROMPT = (
|
|||
"Protected skills (DO NOT edit these):\n"
|
||||
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
|
||||
" • Hub-installed skills (installed via 'hermes skills install').\n"
|
||||
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
|
||||
"pin only blocks deletion/archive/consolidation by the curator, not "
|
||||
"content updates. Patch them when a pitfall or missing step turns up, "
|
||||
"same as any other agent-created skill.\n"
|
||||
" • Skills in skills.external_dirs (externally owned).\n"
|
||||
" • PINNED skills (marked via 'hermes curator pin'). Pin blocks "
|
||||
"autonomous writes entirely — content updates included — because no "
|
||||
"user is present to consent. Only a foreground session can change one.\n"
|
||||
" • USER-OWNED skills — anything not curator-managed (hand-written, "
|
||||
"URL-installed, or created by a foreground agent at the user's "
|
||||
"request). Your writes to these WILL be refused, including to skills "
|
||||
"loaded or consulted this session. If one is wrong, say so in your "
|
||||
"reply and recommend 'hermes curator adopt <name>' instead.\n"
|
||||
"If the only skills that need updating are protected, say\n"
|
||||
"'Nothing to save.' and stop.\n\n"
|
||||
"Do NOT capture as skills (these become persistent self-imposed "
|
||||
|
|
|
|||
|
|
@ -1083,6 +1083,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
tools=tools_for_api,
|
||||
reasoning_config=agent.reasoning_config,
|
||||
session_id=getattr(agent, "session_id", None),
|
||||
base_url=agent.base_url,
|
||||
max_tokens=agent.max_tokens,
|
||||
timeout=agent._resolved_api_call_timeout(),
|
||||
request_overrides=agent.request_overrides,
|
||||
|
|
@ -1520,6 +1521,45 @@ def _fallback_entry_key(fb: dict) -> tuple[str, str, str]:
|
|||
)
|
||||
|
||||
|
||||
def _fallback_entry_is_same_backend_by_base_url(
|
||||
*,
|
||||
current_provider: str,
|
||||
fb_provider: str,
|
||||
current_base_url: str,
|
||||
fb_base_url: str,
|
||||
current_model: str,
|
||||
fb_model: str,
|
||||
) -> bool:
|
||||
"""True when base_url+model identity means the fallback is the same backend.
|
||||
|
||||
Issue #22548: two ``custom_providers`` aliases that point at the same shim
|
||||
URL with the same model must be skipped, or failover loops on the dead
|
||||
backend. First-class providers that share a host while using different
|
||||
auth (``xai-oauth`` vs ``xai``, ``openai-codex`` vs ``openai-api``) are
|
||||
distinct credential surfaces — skipping them strands configured failover
|
||||
when primary and fallback reuse the same model slug on that host.
|
||||
"""
|
||||
if not (
|
||||
fb_base_url
|
||||
and current_base_url
|
||||
and fb_base_url == current_base_url
|
||||
and fb_model == current_model
|
||||
):
|
||||
return False
|
||||
if fb_provider == current_provider:
|
||||
return True
|
||||
try:
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
|
||||
# Both sides are registered first-class providers → different auth
|
||||
# identities even when the inference host matches. Allow failover.
|
||||
if current_provider in PROVIDER_REGISTRY and fb_provider in PROVIDER_REGISTRY:
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _fallback_entry_unavailable_without_network(agent, fb: dict) -> Optional[str]:
|
||||
"""Return a skip reason for fallback entries known to be unusable locally."""
|
||||
fb_provider = (fb.get("provider") or "").strip().lower()
|
||||
|
|
@ -1608,7 +1648,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
# Skip entries that resolve to the current (provider, model) — falling
|
||||
# back to the same backend that just failed loops the failure. Compare
|
||||
# base_url too so two distinct custom_providers entries pointing at the
|
||||
# same shim/proxy URL also dedup. See issue #22548.
|
||||
# same shim/proxy URL also dedup. See issue #22548. Do NOT treat
|
||||
# first-class providers that share a host (xai-oauth vs xai) as the same
|
||||
# backend — they use different credentials.
|
||||
current_provider = (getattr(agent, "provider", "") or "").strip().lower()
|
||||
current_model = (getattr(agent, "model", "") or "").strip()
|
||||
current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower()
|
||||
|
|
@ -1619,11 +1661,13 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
fb_provider, fb_model,
|
||||
)
|
||||
return agent._try_activate_fallback(reason)
|
||||
if (
|
||||
fb_base_url_for_dedup
|
||||
and current_base_url
|
||||
and fb_base_url_for_dedup == current_base_url
|
||||
and fb_model == current_model
|
||||
if _fallback_entry_is_same_backend_by_base_url(
|
||||
current_provider=current_provider,
|
||||
fb_provider=fb_provider,
|
||||
current_base_url=current_base_url,
|
||||
fb_base_url=fb_base_url_for_dedup,
|
||||
current_model=current_model,
|
||||
fb_model=fb_model,
|
||||
):
|
||||
logger.warning(
|
||||
"Fallback skip: chain entry base_url %s matches current backend",
|
||||
|
|
@ -1746,6 +1790,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
fb_provider, fb_model, _pool_provider,
|
||||
)
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
if getattr(agent, "_credential_pool", None) is None:
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
|
|
@ -1808,6 +1853,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
# not only after a later credential-rotation rebuild.
|
||||
agent._replace_primary_openai_client(reason="fallback_timeout_apply")
|
||||
|
||||
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
|
||||
sync_credential_pool_entry_id(agent)
|
||||
|
||||
# Re-evaluate prompt caching for the new provider/model
|
||||
agent._use_prompt_caching, agent._use_native_cache_layout = (
|
||||
agent._anthropic_prompt_cache_policy(
|
||||
|
|
@ -1975,7 +2023,17 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str:
|
|||
for internal_key in [k for k in api_msg if isinstance(k, str) and k.startswith("_")]:
|
||||
api_msg.pop(internal_key, None)
|
||||
if _needs_sanitize:
|
||||
agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model)
|
||||
# In MoA mode, agent.model is the virtual preset name,
|
||||
# not the actual aggregator model. Resolve the real
|
||||
# aggregator model so Gemini preserves thought_signature.
|
||||
_sanitize_model = agent.model
|
||||
if agent.provider == "moa":
|
||||
_moa_client = getattr(agent, "client", None)
|
||||
if _moa_client is not None:
|
||||
_agg_slot = getattr(_moa_client, "last_aggregator_slot", None)
|
||||
if _agg_slot and _agg_slot.get("model"):
|
||||
_sanitize_model = _agg_slot["model"]
|
||||
agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model)
|
||||
api_messages.append(api_msg)
|
||||
|
||||
effective_system = agent._cached_system_prompt or ""
|
||||
|
|
@ -2568,7 +2626,11 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
request_client_holder = {"client": None, "diag": None, "owner_tid": None}
|
||||
# Transport kind of the registered request client — see the non-streaming
|
||||
# variant. Routes _close_request_client_once to anthropic vs openai abort/
|
||||
# close helpers (#67142).
|
||||
# close helpers (#67142). ``kind="stream"`` registers a per-request
|
||||
# *stream handle* instead of a client — used under the MoA facade, whose
|
||||
# singleton client has no per-request sockets to abort
|
||||
# (_abort_request_openai_client is a no-op on it), so interrupts must
|
||||
# close the stream object itself (#57354).
|
||||
request_client_kind = {"value": "openai"}
|
||||
request_client_lock = threading.Lock()
|
||||
# Request-local cancellation flag — see interruptible_api_call for the full
|
||||
|
|
@ -2588,6 +2650,44 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
request_client_holder["owner_tid"] = threading.get_ident()
|
||||
return client
|
||||
|
||||
def _stream_close_callable(stream):
|
||||
close = getattr(stream, "close", None)
|
||||
if callable(close):
|
||||
return close
|
||||
response = getattr(stream, "response", None)
|
||||
close = getattr(response, "close", None)
|
||||
if callable(close):
|
||||
return close
|
||||
return None
|
||||
|
||||
def _set_request_stream_handle(stream):
|
||||
# Register the per-request *stream* under kind="stream" so an
|
||||
# interrupt closes the stream handle itself. Under the MoA facade the
|
||||
# registered "client" is the shared facade singleton whose
|
||||
# per-request abort helpers are no-ops, leaving the underlying HTTP
|
||||
# stream open until the provider drained it (#57354).
|
||||
if _stream_close_callable(stream) is None:
|
||||
return stream
|
||||
with request_client_lock:
|
||||
request_client_holder["client"] = stream
|
||||
request_client_kind["value"] = "stream"
|
||||
request_client_holder["owner_tid"] = threading.get_ident()
|
||||
return stream
|
||||
|
||||
def _close_request_stream_handle(stream, reason: str) -> None:
|
||||
close = _stream_close_callable(stream)
|
||||
if close is None:
|
||||
return
|
||||
try:
|
||||
close()
|
||||
logger.info("Streaming response handle closed (%s)", reason)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Streaming response handle close failed (%s): %s",
|
||||
reason,
|
||||
exc,
|
||||
)
|
||||
|
||||
def _close_request_client_once(reason: str) -> None:
|
||||
# See #29507 explanation in the non-streaming variant above. A
|
||||
# stranger thread (the interrupt-check / stale-stream detector loop)
|
||||
|
|
@ -2595,9 +2695,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# so the worker thread retains ownership of the FD release.
|
||||
with request_client_lock:
|
||||
request_client = request_client_holder.get("client")
|
||||
request_kind = request_client_kind.get("value", "openai")
|
||||
owner_tid = request_client_holder.get("owner_tid")
|
||||
# A registered stream handle (kind="stream", MoA facade path) is
|
||||
# safe to close from any thread — closing IS the abort — so the
|
||||
# stranger-thread ownership carve-out only applies to real
|
||||
# per-request clients (#57354).
|
||||
stranger_thread = (
|
||||
request_client is not None
|
||||
request_kind != "stream"
|
||||
and request_client is not None
|
||||
and owner_tid is not None
|
||||
and owner_tid != threading.get_ident()
|
||||
)
|
||||
|
|
@ -2606,8 +2712,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
request_client_holder["owner_tid"] = None
|
||||
if request_client is None:
|
||||
return
|
||||
kind = request_client_kind.get("value", "openai")
|
||||
if kind == "anthropic_messages":
|
||||
if request_kind == "stream":
|
||||
_close_request_stream_handle(request_client, reason)
|
||||
elif request_kind == "anthropic_messages":
|
||||
if stranger_thread:
|
||||
agent._abort_request_anthropic_client(request_client, reason=reason)
|
||||
else:
|
||||
|
|
@ -2897,6 +3004,10 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
defer_logical_completion=True,
|
||||
)
|
||||
)
|
||||
if agent.provider == "moa":
|
||||
# Hermes interrupts the managed stream; Relay retains sole
|
||||
# ownership of closing the underlying provider stream.
|
||||
_set_request_stream_handle(stream)
|
||||
for chunk in stream:
|
||||
last_chunk_time["t"] = time.time()
|
||||
agent._touch_activity("receiving stream response")
|
||||
|
|
@ -3567,13 +3678,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# already worker-owned-closed by _close_request_client_once
|
||||
# above; the next attempt builds a fresh one. The shared
|
||||
# _anthropic_client is never closed from inside a request.
|
||||
if agent.api_mode != "anthropic_messages":
|
||||
try:
|
||||
agent._replace_primary_openai_client(
|
||||
reason="stream_mid_tool_retry_pool_cleanup"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector for OpenAI.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next attempt.
|
||||
continue
|
||||
|
||||
# SSE error events from proxies (e.g. OpenRouter sends
|
||||
|
|
@ -3632,13 +3739,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# above; next attempt builds fresh), so the shared
|
||||
# _anthropic_client is never closed from inside a
|
||||
# request — only the OpenAI-wire primary is refreshed.
|
||||
if agent.api_mode != "anthropic_messages":
|
||||
try:
|
||||
agent._replace_primary_openai_client(
|
||||
reason="stream_retry_pool_cleanup"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector for OpenAI.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next attempt.
|
||||
continue
|
||||
# Retries exhausted. Log the final failure with
|
||||
# full diagnostic detail (chain, headers,
|
||||
|
|
@ -3882,10 +3985,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# FD-recycle corruption vector. Nothing further is needed.
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup")
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector as #67142.
|
||||
# The shared OpenAI client's connection pool must NOT be
|
||||
# closed from this watchdog/poll thread — worker threads
|
||||
# from previous stale-killed attempts may still be
|
||||
# unwinding their SSL BIOs. The request-local client is
|
||||
# already closed above via _close_request_client_once.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next request.
|
||||
pass
|
||||
# Reset the timer so we don't kill repeatedly while
|
||||
# the inner thread processes the closure.
|
||||
last_chunk_time["t"] = time.time()
|
||||
|
|
|
|||
|
|
@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs(
|
|||
allowed_keys = {
|
||||
"model", "instructions", "input", "tools", "store",
|
||||
"reasoning", "include", "max_output_tokens", "temperature",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key",
|
||||
"prompt_cache_retention", "service_tier",
|
||||
"extra_headers", "extra_body", "timeout",
|
||||
}
|
||||
normalized: Dict[str, Any] = {
|
||||
|
|
@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs(
|
|||
if isinstance(temperature, (int, float)):
|
||||
normalized["temperature"] = float(temperature)
|
||||
|
||||
# Pass through tool_choice, parallel_tool_calls, prompt_cache_key
|
||||
for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"):
|
||||
# Pass through cache routing/retention and tool-dispatch hints.
|
||||
for passthrough_key in (
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"prompt_cache_key",
|
||||
"prompt_cache_retention",
|
||||
):
|
||||
val = api_kwargs.get(passthrough_key)
|
||||
if val is not None:
|
||||
normalized[passthrough_key] = val
|
||||
|
|
|
|||
|
|
@ -520,30 +520,46 @@ class RuntimeMode:
|
|||
return None
|
||||
return [self.profile.toolset, *_enabled_mcp_servers(config)]
|
||||
|
||||
def system_blocks(self) -> list[str]:
|
||||
"""Stable system-prompt blocks for this posture (brief + workspace).
|
||||
def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Return prefix, workspace, and trailing posture blocks separately.
|
||||
|
||||
The operating brief carries a model-family edit-format nudge appended
|
||||
to it (one cached string, not a separate block) so the model is steered
|
||||
toward the `patch` mode it handles best — see ``_edit_format_line``.
|
||||
|
||||
The three lists preserve the historical flat prompt order: the brief,
|
||||
the live workspace snapshot, then configured operator instructions.
|
||||
Prompt assembly can therefore put a cache boundary before the snapshot
|
||||
without changing the persisted system-prompt bytes.
|
||||
"""
|
||||
if not self.is_coding:
|
||||
return []
|
||||
blocks: list[str] = []
|
||||
return [], [], []
|
||||
prefix: list[str] = []
|
||||
workspace_parts: list[str] = []
|
||||
trailing: list[str] = []
|
||||
if self.profile.guidance:
|
||||
brief = self.profile.guidance
|
||||
edit_line = _edit_format_line(self.model)
|
||||
if edit_line:
|
||||
brief = f"{brief}\n{edit_line}"
|
||||
blocks.append(brief)
|
||||
prefix.append(brief)
|
||||
workspace = build_coding_workspace_block(self.cwd)
|
||||
if workspace:
|
||||
blocks.append(workspace)
|
||||
workspace_parts.append(workspace)
|
||||
# Operator instructions ride their own block so the brief (block 0) stays
|
||||
# byte-stable and cache-keyed independently of user config.
|
||||
if self.instructions:
|
||||
blocks.append(f"Operator instructions (from config):\n{self.instructions}")
|
||||
return blocks
|
||||
trailing.append(f"Operator instructions (from config):\n{self.instructions}")
|
||||
return prefix, workspace_parts, trailing
|
||||
|
||||
def system_blocks(self) -> list[str]:
|
||||
"""Return posture blocks in their historical display order.
|
||||
|
||||
``system_prompt_parts`` is the cache-aware API. This compatibility
|
||||
helper retains the public flat list for callers outside prompt assembly.
|
||||
"""
|
||||
prefix, workspace, trailing = self.system_prompt_parts()
|
||||
return [*prefix, *workspace, *trailing]
|
||||
|
||||
def compact_skill_categories(self) -> frozenset[str]:
|
||||
"""Skill categories to demote to names-only in the prompt's skill index.
|
||||
|
|
@ -644,6 +660,19 @@ def coding_system_blocks(
|
|||
).system_blocks()
|
||||
|
||||
|
||||
def coding_system_prompt_parts(
|
||||
*,
|
||||
platform: Optional[str] = None,
|
||||
cwd: Optional[str | Path] = None,
|
||||
config: Optional[dict[str, Any]] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Return coding prefix, workspace snapshot, and trailing guidance."""
|
||||
return resolve_runtime_mode(
|
||||
platform=platform, cwd=cwd, config=config, model=model
|
||||
).system_prompt_parts()
|
||||
|
||||
|
||||
def coding_compact_skill_categories(
|
||||
*,
|
||||
platform: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -90,9 +90,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool:
|
|||
|
||||
|
||||
HISTORICAL_TASK_HEADING = "## Historical Task Snapshot"
|
||||
HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State"
|
||||
HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks"
|
||||
HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work"
|
||||
|
||||
|
||||
SUMMARY_PREFIX = (
|
||||
|
|
@ -107,9 +104,7 @@ SUMMARY_PREFIX = (
|
|||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
|
||||
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
|
||||
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
|
||||
f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
|
|
@ -219,8 +214,45 @@ _MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW]
|
|||
# stale directive it carried (e.g. "resume exactly from Active Task") survives
|
||||
# embedded in the body and keeps hijacking replies. Keep newest-first; entries
|
||||
# are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes.
|
||||
# NEVER mutate or reorder an existing entry — each one is the exact wire text a
|
||||
# shipped build persisted, so editing it silently un-normalizes every summary
|
||||
# written by that build generation; prepend only. tests/agent/
|
||||
# test_summary_prefix_semantics.py byte-pins every entry to enforce this.
|
||||
_HISTORICAL_SUMMARY_PREFIXES = (
|
||||
# Jul 2026 (#65848 class): identical to the current prefix except it
|
||||
# Pre-#69619: identical to the current prefix except the stale-item
|
||||
# discard clause named all four historical headings (the three
|
||||
# section headers removed by #69619 were still in the template).
|
||||
# Summaries persisted by builds immediately before #69619 carry this
|
||||
# exact text and must remain detectable/strippable on resume.
|
||||
"[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted "
|
||||
"into the summary below. This is a handoff from a previous context "
|
||||
"window — treat it as background reference, NOT as active instructions. "
|
||||
"Do NOT answer questions or fulfill requests mentioned in this summary; "
|
||||
"they were already addressed. "
|
||||
"Respond ONLY to the latest user message that appears AFTER this "
|
||||
"summary — that message is the single source of truth for what to do "
|
||||
"right now. "
|
||||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
|
||||
"'## Historical Pending User Asks' / "
|
||||
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
"back', 'just verify', 'don't do that anymore', 'never mind', a new "
|
||||
"topic) must immediately end any in-flight work described in the "
|
||||
"summary; do not re-surface it in later turns. "
|
||||
"IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system "
|
||||
"prompt is ALWAYS authoritative and active — never ignore or deprioritize "
|
||||
"memory content due to this compaction note. "
|
||||
"None of the above restricts HOW you work: your tools remain fully "
|
||||
"active — keep calling them normally for the active task (edit files, "
|
||||
"run commands, search) instead of merely narrating what you would do. "
|
||||
"The current session state (files, config, etc.) may reflect work "
|
||||
"described here — avoid repeating it:",
|
||||
# Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it
|
||||
# lacked the explicit "tools remain fully active" clause — the strong
|
||||
# REFERENCE ONLY framing bled into general tool-use suppression
|
||||
# (observed: 7 consecutive narration-only turns immediately after a
|
||||
|
|
@ -236,9 +268,9 @@ _HISTORICAL_SUMMARY_PREFIXES = (
|
|||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
|
||||
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
|
||||
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
|
||||
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
|
||||
"'## Historical Pending User Asks' / "
|
||||
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
|
|
@ -303,9 +335,238 @@ _SUMMARY_RATIO = 0.20
|
|||
# itself a context-pressure source and slows every compaction.
|
||||
_SUMMARY_TOKENS_CEILING = 10_000
|
||||
|
||||
# Aggregate cap on the serialized turn block fed to the summarizer prompt
|
||||
# (chars). Per-message truncation (_CONTENT_MAX / _TOOL_ARGS_MAX) alone is
|
||||
# not enough: a compression window with hundreds of already-truncated turns
|
||||
# can still produce a multi-hundred-KB prompt that blows past slow auxiliary
|
||||
# backends' context limits or timeouts (Codex Responses fallback paths
|
||||
# especially). 160K chars ≈ 40K tokens — comfortably inside every supported
|
||||
# aux model's window while leaving room for the template + previous summary.
|
||||
# Applied AFTER per-message truncation, with head+tail retention and an
|
||||
# explicit omitted-middle marker (see _bound_summary_input). This is a
|
||||
# prompt-side bound only — NEVER add a max_tokens wire cap on the summary
|
||||
# call (see the no-wire-cap contract test in
|
||||
# test_compression_small_ctx_threshold_floor.py).
|
||||
_SUMMARY_INPUT_MAX_CHARS = 160_000
|
||||
|
||||
# Placeholder used when pruning old tool results
|
||||
_PRUNED_TOOL_PLACEHOLDER = "[Old tool output cleared to save context space]"
|
||||
|
||||
# Ghost-skill defense (#32106): when compaction reduces an old ``skill_view``
|
||||
# result to a 1-line metadata summary, the model still believes the skill is
|
||||
# loaded even though its instructions are gone. The marker below is the ONE
|
||||
# canonical prune signal — ``_skill_pruned_marker()`` builds it and every
|
||||
# presence check matches against the same string, so the emit side and the
|
||||
# check side can never drift apart (the original PR #44166 emitted
|
||||
# ``[SKILL_PRUNED:`` but presence-checked ``[SKILL_PRUNED]``, making
|
||||
# re-injection fire even when the marker had survived).
|
||||
SKILL_PRUNED_MARKER_PREFIX = "[SKILL_PRUNED:"
|
||||
# skill_view results at or below this size stay verbatim in pruned
|
||||
# summaries — small skills are cheap to keep and their loss is unlikely to
|
||||
# ghost the model. Shared by the emit site and the summarizer-input scan.
|
||||
_SKILL_VIEW_PRUNE_MIN_CHARS = 5000
|
||||
# Cap for the deterministic marker re-injection list — keeps a very long
|
||||
# session from growing an unbounded "## Pruned Skills" block in every
|
||||
# iterative summary update. Newest-referenced skills win.
|
||||
_MAX_PRUNED_SKILL_MARKERS = 20
|
||||
|
||||
|
||||
def _skill_pruned_marker(skill_name: str) -> str:
|
||||
"""Return the canonical prune marker for *skill_name*.
|
||||
|
||||
Used verbatim by BOTH the emit sites (tool-result summarization,
|
||||
summary re-injection) and the survival check in
|
||||
``_reinject_pruned_skill_markers`` — one string, no drift.
|
||||
"""
|
||||
return (
|
||||
f"{SKILL_PRUNED_MARKER_PREFIX} content lost in compression; "
|
||||
f"reload with skill_view(name='{skill_name}')]"
|
||||
)
|
||||
|
||||
|
||||
# Matches the canonical marker and captures the skill name. Anchored on the
|
||||
# shared prefix constant so a wording change to the marker body updates the
|
||||
# emit helper and this extractor together.
|
||||
_SKILL_PRUNED_MARKER_RE = re.compile(
|
||||
re.escape(SKILL_PRUNED_MARKER_PREFIX)
|
||||
+ r"[^\]]*?reload with skill_view\(name='([^']+)'\)"
|
||||
)
|
||||
|
||||
|
||||
def _extract_pruned_skill_names(text: str) -> list[str]:
|
||||
"""Return skill names referenced by prune markers in *text*, in order."""
|
||||
names: list[str] = []
|
||||
for match in _SKILL_PRUNED_MARKER_RE.finditer(text or ""):
|
||||
name = match.group(1)
|
||||
if name not in names:
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def _collect_ghosted_skill_names(turns: List[Dict[str, Any]]) -> list[str]:
|
||||
"""Skill names whose instructions are about to be lost in compaction.
|
||||
|
||||
Covers BOTH shapes a compacted middle window can carry:
|
||||
|
||||
- a ``skill_view`` result already demoted by Phase-1 pruning — the
|
||||
canonical ``[SKILL_PRUNED: ...]`` marker is in the row content;
|
||||
- a RAW ``skill_view`` body that was never demoted (it sat inside the
|
||||
protected tail of an earlier prune, then aged into the compression
|
||||
window). The summarizer will paraphrase the instructions away, which
|
||||
is exactly the ghost-skill failure — so it needs a marker too.
|
||||
"""
|
||||
names: list[str] = []
|
||||
|
||||
def _add(name: str) -> None:
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
|
||||
call_id_to_skill: dict[str, str] = {}
|
||||
for idx, skill in _skill_view_call_sites(turns):
|
||||
msg = turns[idx]
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
tc_fn = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", None)
|
||||
tc_name = tc_fn.get("name", "") if isinstance(tc_fn, dict) else getattr(tc_fn, "name", "")
|
||||
if tc_name != "skill_view":
|
||||
continue
|
||||
cid = tc.get("id", "") if isinstance(tc, dict) else (getattr(tc, "id", "") or "")
|
||||
if cid:
|
||||
call_id_to_skill[cid] = skill
|
||||
for msg in turns:
|
||||
content = msg.get("content")
|
||||
text = content if isinstance(content, str) else _content_text_for_contains(content)
|
||||
for name in _extract_pruned_skill_names(text):
|
||||
_add(name)
|
||||
if (
|
||||
msg.get("role") == "tool"
|
||||
and isinstance(content, str)
|
||||
and len(content) > _SKILL_VIEW_PRUNE_MIN_CHARS
|
||||
):
|
||||
skill = call_id_to_skill.get(str(msg.get("tool_call_id") or ""))
|
||||
if skill:
|
||||
_add(skill)
|
||||
return names
|
||||
|
||||
|
||||
_PRUNED_SKILLS_SECTION_HEADING = "## Pruned Skills"
|
||||
|
||||
|
||||
def _reinject_pruned_skill_markers(summary: str, skill_names: list[str]) -> str:
|
||||
"""Deterministically restore prune markers the summarizer dropped.
|
||||
|
||||
``skill_names`` was extracted from the summarizer INPUT before the LLM
|
||||
call. For every skill whose canonical marker (``_skill_pruned_marker``)
|
||||
is absent from the model's output, append it under a ``## Pruned
|
||||
Skills`` section. Presence is checked against the SAME canonical string
|
||||
the emit sites produce — a paraphrased or renamed marker counts as
|
||||
dropped and is restored (the original PR checked the literal
|
||||
``[SKILL_PRUNED]``, which never matches the emitted ``[SKILL_PRUNED:``
|
||||
form, so it duplicated markers that HAD survived).
|
||||
|
||||
The appended block is plain body text: it never carries a handoff
|
||||
prefix, the merged-summary delimiter, or a start-of-content scaffolding
|
||||
marker, so ``classify_summary_content`` / todo-snapshot flag handling
|
||||
are unaffected. The block is routed through ``_redact_compaction_text``
|
||||
like every other compaction-boundary text.
|
||||
"""
|
||||
if not skill_names:
|
||||
return summary
|
||||
missing = [
|
||||
name for name in skill_names
|
||||
if _skill_pruned_marker(name) not in summary
|
||||
]
|
||||
if not missing:
|
||||
return summary
|
||||
lines = [_skill_pruned_marker(name) for name in missing]
|
||||
block = (
|
||||
"\n\n" + _PRUNED_SKILLS_SECTION_HEADING + "\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n(The listed skills' instructions were pruned during context "
|
||||
"compression. Reload with the skill_view call in each marker before "
|
||||
"relying on that skill; one reload per skill is enough — ignore any "
|
||||
"older markers for the same skill.)"
|
||||
)
|
||||
return summary + _redact_compaction_text(block)
|
||||
|
||||
|
||||
# A skill_view call within this many trailing messages counts as "just
|
||||
# loaded": its full instruction body must survive the Phase-1 prune even when
|
||||
# the token-budget boundary would otherwise demote it (#32106). Distinct from
|
||||
# the protected-tail boundary, which is token-based and can land immediately
|
||||
# after a bulky just-loaded skill body.
|
||||
_SKILL_PRUNE_RECENT_WINDOW = 10
|
||||
|
||||
|
||||
def _skill_view_call_sites(
|
||||
messages: List[Dict[str, Any]],
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Yield ``(message_index, skill_name)`` for every skill_view tool call."""
|
||||
sites: list[tuple[int, str]] = []
|
||||
for i, msg in enumerate(messages):
|
||||
if msg.get("role") != "assistant":
|
||||
continue
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
if isinstance(tc, dict):
|
||||
fn = tc.get("function", {})
|
||||
name = fn.get("name", "") if isinstance(fn, dict) else ""
|
||||
args_str = fn.get("arguments", "") if isinstance(fn, dict) else ""
|
||||
else:
|
||||
fn = getattr(tc, "function", None)
|
||||
name = getattr(fn, "name", "") if fn else ""
|
||||
args_str = getattr(fn, "arguments", "") if fn else ""
|
||||
if name != "skill_view" or not isinstance(args_str, str) or not args_str:
|
||||
continue
|
||||
try:
|
||||
args = json.loads(args_str)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if isinstance(args, dict):
|
||||
skill = args.get("name", "")
|
||||
if isinstance(skill, str) and skill:
|
||||
sites.append((i, skill))
|
||||
return sites
|
||||
|
||||
|
||||
def _collect_protected_skill_names(
|
||||
messages: List[Dict[str, Any]], prune_boundary: int,
|
||||
) -> set[str]:
|
||||
"""Skill names whose skill_view bodies must survive Phase-1 demotion.
|
||||
|
||||
A skill is protected (lower-cased set) when any of these hold:
|
||||
|
||||
- its most recent ``skill_view`` call sits within the last
|
||||
``_SKILL_PRUNE_RECENT_WINDOW`` messages (just loaded / just reloaded);
|
||||
- its most recent ``skill_view`` call sits inside the protected tail
|
||||
(at or after *prune_boundary*);
|
||||
- its name is mentioned in a user message inside the protected tail
|
||||
(the user is actively steering work that depends on it).
|
||||
|
||||
Protection applies to the ordinary Phase-1/2 prune only. The Pass-4
|
||||
pressure demotion deliberately ignores it: when the protected region
|
||||
itself exceeds the soft budget, exempting skill bodies would recreate
|
||||
the #61932 dead-end shape.
|
||||
"""
|
||||
total = len(messages)
|
||||
if not total:
|
||||
return set()
|
||||
recent_start = max(0, total - _SKILL_PRUNE_RECENT_WINDOW)
|
||||
tail_start = max(0, prune_boundary)
|
||||
tail_user_texts: list[str] = []
|
||||
for msg in messages[tail_start:]:
|
||||
if msg.get("role") != "user":
|
||||
continue
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str) and content:
|
||||
tail_user_texts.append(content.lower())
|
||||
protected: set[str] = set()
|
||||
for idx, skill in _skill_view_call_sites(messages):
|
||||
key = skill.lower()
|
||||
if idx >= recent_start or idx >= tail_start:
|
||||
protected.add(key)
|
||||
elif any(key in text for text in tail_user_texts):
|
||||
protected.add(key)
|
||||
return protected
|
||||
|
||||
# Chars per token rough estimate
|
||||
_CHARS_PER_TOKEN = 4
|
||||
# Flat token cost per attached image part. Real cost varies by provider and
|
||||
|
|
@ -877,7 +1138,19 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten
|
|||
code_preview += "..."
|
||||
return f"[execute_code] `{code_preview}` ({line_count} lines output)"
|
||||
|
||||
if tool_name in {"skill_view", "skills_list", "skill_manage"}:
|
||||
if tool_name == "skill_view":
|
||||
name = args.get("name", "?")
|
||||
if content_len > _SKILL_VIEW_PRUNE_MIN_CHARS:
|
||||
# Ghost-skill defense (#32106): a metadata-only summary makes the
|
||||
# model believe the skill is still loaded. The canonical marker
|
||||
# tells it the instructions are gone AND how to get them back.
|
||||
return (
|
||||
f"[skill_view] name={name} ({content_len:,} chars) "
|
||||
+ _skill_pruned_marker(str(name))
|
||||
)
|
||||
return f"[skill_view] name={name} ({content_len:,} chars)"
|
||||
|
||||
if tool_name in {"skills_list", "skill_manage"}:
|
||||
name = args.get("name", "?")
|
||||
return f"[{tool_name}] name={name} ({content_len:,} chars)"
|
||||
|
||||
|
|
@ -972,6 +1245,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._last_aux_model_failure_model = None
|
||||
self._last_compression_savings_pct = 100.0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self._fallback_compression_streak = 0
|
||||
self._verify_compaction_cleared_threshold = False
|
||||
self._last_compression_made_progress = False
|
||||
|
|
@ -1109,6 +1383,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._last_aux_model_failure_model = None
|
||||
self._last_compression_savings_pct = 100.0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self._fallback_compression_streak = 0
|
||||
self._verify_compaction_cleared_threshold = False
|
||||
self._last_compression_made_progress = False
|
||||
|
|
@ -1135,6 +1410,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._consecutive_timeout_failures = 0
|
||||
self._fallback_compression_streak = 0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self.get_active_compression_failure_cooldown()
|
||||
self._load_fallback_compression_streak()
|
||||
self._load_ineffective_compression_count()
|
||||
|
|
@ -1506,6 +1782,15 @@ class ContextCompressor(ContextEngine):
|
|||
# rationale as the gpt-5.5/Codex 85% autoraise.
|
||||
_MIN_CTX_TRIGGER_RATIO = 0.85
|
||||
|
||||
# Anti-thrash recovery window (#14694): once the ineffective/fallback
|
||||
# breaker trips, automatic compaction stays blocked for this long, then
|
||||
# ONE probe attempt is allowed (counters drop to 1 strike, so another
|
||||
# ineffective pass re-trips immediately). Long enough that a genuinely
|
||||
# incompressible session isn't compacting in a loop; short enough that a
|
||||
# session which has since grown real compressible material recovers well
|
||||
# before it rides into the provider's hard context limit.
|
||||
_ANTI_THRASH_RECOVERY_SECONDS = 300.0
|
||||
|
||||
@staticmethod
|
||||
def _coerce_max_tokens(value: Any) -> int | None:
|
||||
"""Normalize a max_tokens value to a positive int or None.
|
||||
|
|
@ -1629,6 +1914,10 @@ class ContextCompressor(ContextEngine):
|
|||
max_tokens: int | None = None,
|
||||
model_thresholds: dict[str, float] | None = None,
|
||||
threshold_tokens_cap: Any = None,
|
||||
proactive_prune_tokens: int = 0,
|
||||
proactive_prune_min_result_chars: int = 8000,
|
||||
proactive_prune_min_reclaim_tokens: int = 4096,
|
||||
min_tail_user_messages: int = 1,
|
||||
):
|
||||
self.model = model
|
||||
self.base_url = base_url
|
||||
|
|
@ -1658,6 +1947,32 @@ class ContextCompressor(ContextEngine):
|
|||
)
|
||||
self.protect_first_n = protect_first_n
|
||||
self.protect_last_n = protect_last_n
|
||||
# Proactive tool-result pruning (cost-oriented; runs INDEPENDENTLY of the
|
||||
# full-compression trigger, via prune_tool_results_only()). 0 = disabled.
|
||||
self.proactive_prune_tokens = int(proactive_prune_tokens or 0)
|
||||
# Floor the summarize threshold at 200 chars (matching
|
||||
# _prune_old_tool_results' dedup floor). Below ~200 a generated summary
|
||||
# can be longer than the floor it replaces, so Pass 2 would re-summarize
|
||||
# its own output every turn (corrupting it and never converging); a
|
||||
# negative value would strip every non-tail tool result outright. A
|
||||
# configured 0 keeps the 8000 default via `or`. Keep the floor well above
|
||||
# typical summary length (default 8000) to stay idempotent.
|
||||
self.proactive_prune_min_result_chars = max(
|
||||
200, int(proactive_prune_min_result_chars or 8000)
|
||||
)
|
||||
# Minimum estimated token reclaim before a proactive prune COMMITS.
|
||||
# Every commit rewrites messages the provider has already seen, which
|
||||
# invalidates the prompt-cache prefix from the earliest rewritten
|
||||
# message forward. Without this gate a busy tool loop would re-fire
|
||||
# the prune nearly every iteration (each new tool pair ages an old one
|
||||
# out of the protected tail), breaking the cache per turn. Requiring a
|
||||
# meaningful batch of reclaimable tokens makes fires episodic and
|
||||
# amortized — the same way full compression is the one sanctioned
|
||||
# cache break. 0 disables the gate (commit any non-zero prune).
|
||||
self.proactive_prune_min_reclaim_tokens = max(
|
||||
0, int(proactive_prune_min_reclaim_tokens or 0)
|
||||
)
|
||||
self.min_tail_user_messages = min_tail_user_messages
|
||||
self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80))
|
||||
self.quiet_mode = quiet_mode
|
||||
# Output-token reservation: the provider carves max_tokens out of the
|
||||
|
|
@ -1745,6 +2060,12 @@ class ContextCompressor(ContextEngine):
|
|||
# Anti-thrashing: track whether last compression was effective
|
||||
self._last_compression_savings_pct: float = 100.0
|
||||
self._ineffective_compression_count: int = 0
|
||||
# Monotonic deadline after which a tripped anti-thrash guard grants
|
||||
# one probation probe (#14694). 0.0 = clock not armed. Armed lazily on
|
||||
# the first blocked evaluation; deliberately NOT durable, so a process
|
||||
# restart with a persisted tripped counter (#69872) waits a full fresh
|
||||
# window before probing (#54923: restart must never disarm a guard).
|
||||
self._anti_thrash_recovery_deadline: float = 0.0
|
||||
# Consecutive completed deterministic-fallback boundaries. Unlike the
|
||||
# real-usage effectiveness counter, ordinary fitting responses must not
|
||||
# reset this breaker; only a healthy completed summary does.
|
||||
|
|
@ -1856,6 +2177,16 @@ class ContextCompressor(ContextEngine):
|
|||
self._verify_compaction_cleared_threshold = False
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
|
||||
def snapshot_preflight_display_tokens(self) -> int:
|
||||
"""Capture the display token count before a speculative preflight seed."""
|
||||
return self.last_prompt_tokens
|
||||
|
||||
def rollback_interrupted_preflight_display_tokens(self, snapshot: int) -> None:
|
||||
"""Restore a speculative display seed without touching compaction state."""
|
||||
if self.awaiting_real_usage_after_compression and self.last_prompt_tokens == -1:
|
||||
return
|
||||
self.last_prompt_tokens = snapshot
|
||||
|
||||
def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool:
|
||||
"""Return True when a high rough preflight estimate is known-noisy.
|
||||
|
||||
|
|
@ -2025,21 +2356,66 @@ class ContextCompressor(ContextEngine):
|
|||
_cooldown_remaining,
|
||||
)
|
||||
return True
|
||||
# Anti-thrashing: back off if recent compressions were ineffective
|
||||
# Anti-thrashing: back off if recent compressions were ineffective.
|
||||
# The back-off must not be permanent (#14694): the tripped state was
|
||||
# judged against the transcript as it existed THEN (e.g. a middle
|
||||
# region too small to matter), but the conversation keeps growing and
|
||||
# can accumulate plenty of compressible material later. Without a
|
||||
# recovery path the session never auto-compacts again and rides into
|
||||
# the provider's hard context limit. Recovery is a probation probe:
|
||||
# after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE
|
||||
# attempt by dropping the tripped counter(s) to 1 strike (persisted,
|
||||
# so sibling agents on the same session row unblock too). If the probe
|
||||
# is ineffective again the very next verdict re-trips the guard, so
|
||||
# the worst case in the truly-incompressible state is one compaction
|
||||
# attempt per recovery window — bounded, not thrash.
|
||||
#
|
||||
# The clock is armed lazily on the first BLOCKED evaluation rather
|
||||
# than persisted at trip time: a fresh process that loads a durable
|
||||
# tripped counter (#69872) therefore starts a full window blocked,
|
||||
# preserving the restart-must-not-disarm contract (#54923).
|
||||
if (
|
||||
self._ineffective_compression_count >= 2
|
||||
or self._fallback_compression_streak >= 2
|
||||
):
|
||||
_now = time.monotonic()
|
||||
if self._anti_thrash_recovery_deadline <= 0.0:
|
||||
self._anti_thrash_recovery_deadline = (
|
||||
_now + self._ANTI_THRASH_RECOVERY_SECONDS
|
||||
)
|
||||
elif _now >= self._anti_thrash_recovery_deadline:
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
if self._ineffective_compression_count >= 2:
|
||||
self._record_ineffective_compression_verdict(1)
|
||||
if self._fallback_compression_streak >= 2:
|
||||
self._fallback_compression_streak = 1
|
||||
self._persist_fallback_compression_streak()
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Anti-thrashing recovery: %.0fs elapsed since the "
|
||||
"guard tripped — allowing one compaction probe "
|
||||
"(ineffective=%d fallback=%d).",
|
||||
self._ANTI_THRASH_RECOVERY_SECONDS,
|
||||
self._ineffective_compression_count,
|
||||
self._fallback_compression_streak,
|
||||
)
|
||||
return False
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Compression skipped — repeated compaction attempts did not "
|
||||
"restore healthy context. ineffective=%d fallback=%d. "
|
||||
"Consider /new to start fresh, or /compress <topic> for "
|
||||
"focused compression.",
|
||||
"Auto-compaction will retry once in %.0fs. Consider /new "
|
||||
"to start fresh, or /compress <topic> for focused "
|
||||
"compression.",
|
||||
self._ineffective_compression_count,
|
||||
self._fallback_compression_streak,
|
||||
max(0.0, self._anti_thrash_recovery_deadline - _now),
|
||||
)
|
||||
return True
|
||||
# Guard not tripped (counters were cleared by an effective compaction
|
||||
# or a fitting real-usage reading) — disarm any pending recovery clock
|
||||
# so a LATER trip starts its own full window.
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
|
@ -2049,6 +2425,7 @@ class ContextCompressor(ContextEngine):
|
|||
def _prune_old_tool_results(
|
||||
self, messages: List[Dict[str, Any]], protect_tail_count: int,
|
||||
protect_tail_tokens: int | None = None,
|
||||
min_prune_chars: int = 200,
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""Replace old tool result contents with informative 1-line summaries.
|
||||
|
||||
|
|
@ -2158,7 +2535,14 @@ class ContextCompressor(ContextEngine):
|
|||
else:
|
||||
content_hashes[h] = (i, msg.get("tool_call_id", "?"))
|
||||
|
||||
def _demote_tool_result_at(idx: int) -> bool:
|
||||
# Ghost-skill defense (#32106): skills just loaded (or actively
|
||||
# referenced in the protected tail) keep their full skill_view
|
||||
# bodies through the ordinary prune passes. Without this, a skill
|
||||
# loaded moments before a compaction can be demoted to metadata
|
||||
# while the model still believes its instructions are in context.
|
||||
protected_skills = _collect_protected_skill_names(result, prune_boundary)
|
||||
|
||||
def _demote_tool_result_at(idx: int, *, spare_protected_skills: bool = True) -> bool:
|
||||
"""Replace a bulky tool result at ``idx`` with a 1-line summary.
|
||||
|
||||
Returns True when the message was modified.
|
||||
|
|
@ -2191,10 +2575,22 @@ class ContextCompressor(ContextEngine):
|
|||
return False
|
||||
if content.startswith("[screenshot removed"):
|
||||
return False
|
||||
if len(content) <= 200:
|
||||
# Only prune if the content is substantial (default >200 chars; the
|
||||
# proactive path raises this floor via min_prune_chars).
|
||||
if len(content) <= min_prune_chars:
|
||||
return False
|
||||
call_id = msg.get("tool_call_id", "")
|
||||
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
|
||||
if spare_protected_skills and tool_name == "skill_view" and protected_skills:
|
||||
# Just-loaded / actively-referenced skills survive verbatim
|
||||
# (#32106). Pass-4 pressure demotion overrides this.
|
||||
try:
|
||||
_args = json.loads(tool_args) if tool_args else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
_args = {}
|
||||
_skill = _args.get("name", "") if isinstance(_args, dict) else ""
|
||||
if isinstance(_skill, str) and _skill.lower() in protected_skills:
|
||||
return False
|
||||
summary = _summarize_tool_result(tool_name, tool_args, content)
|
||||
result[idx] = {**msg, "content": summary}
|
||||
pruned += 1
|
||||
|
|
@ -2259,7 +2655,10 @@ class ContextCompressor(ContextEngine):
|
|||
if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling:
|
||||
pressure_hits = 0
|
||||
for i in range(max(0, prune_boundary), demote_end):
|
||||
if _demote_tool_result_at(i):
|
||||
# Pressure passes override the just-loaded-skill guard:
|
||||
# when the protected region itself blows the soft budget,
|
||||
# sparing skill bodies would recreate the #61932 dead-end.
|
||||
if _demote_tool_result_at(i, spare_protected_skills=False):
|
||||
pressure_hits += 1
|
||||
if _truncate_tool_call_args_at(i):
|
||||
pressure_hits += 1
|
||||
|
|
@ -2279,7 +2678,7 @@ class ContextCompressor(ContextEngine):
|
|||
if last_tool_idx is not None and i == last_tool_idx:
|
||||
continue
|
||||
if result[i].get("role") == "tool":
|
||||
if _demote_tool_result_at(i):
|
||||
if _demote_tool_result_at(i, spare_protected_skills=False):
|
||||
pressure_hits += 1
|
||||
elif result[i].get("role") == "assistant":
|
||||
if _truncate_tool_call_args_at(i):
|
||||
|
|
@ -2293,7 +2692,9 @@ class ContextCompressor(ContextEngine):
|
|||
and last_tool_idx >= prune_boundary
|
||||
and _protected_region_tokens() > soft_ceiling
|
||||
):
|
||||
if _demote_tool_result_at(last_tool_idx):
|
||||
if _demote_tool_result_at(
|
||||
last_tool_idx, spare_protected_skills=False
|
||||
):
|
||||
pressure_hits += 1
|
||||
if pressure_hits and not self.quiet_mode:
|
||||
logger.info(
|
||||
|
|
@ -2307,6 +2708,75 @@ class ContextCompressor(ContextEngine):
|
|||
|
||||
return result, pruned
|
||||
|
||||
def prune_tool_results_only(
|
||||
self, messages: List[Dict[str, Any]], current_tokens: int | None = None,
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""Deterministic, no-LLM tool-result prune for the cost-oriented path.
|
||||
|
||||
Runs the Phase-1 prune (``_prune_old_tool_results``) WITHOUT the
|
||||
compression summary phase, gated on ``proactive_prune_tokens`` rather
|
||||
than the (much higher) full-compression threshold. On large-window
|
||||
models ``should_compress()`` (≈50% of the window) rarely fires, so old
|
||||
tool outputs otherwise ride in history and are re-sent verbatim on every
|
||||
subsequent turn; this reclaims them early with no quality-risky LLM
|
||||
summarization.
|
||||
|
||||
Protects the recent tail by message COUNT (``protect_last_n``), never by
|
||||
``tail_token_budget`` — the latter is derived from the 50% compression
|
||||
threshold (≈100K tokens on a 1M window) and would protect the entire
|
||||
session, pruning nothing.
|
||||
|
||||
``_prune_old_tool_results`` runs all three deterministic passes:
|
||||
(1) dedup byte-identical tool results — keeps the newest full copy and
|
||||
back-references older exact duplicates ANYWHERE in the list (including
|
||||
the protected tail), so no unique content is ever lost; (2) summarize
|
||||
non-tail tool results larger than ``min_prune_chars``; (3) truncate
|
||||
oversized tool_call arguments on non-tail assistant messages. Only
|
||||
pass (2)'s floor is raised by ``proactive_prune_min_result_chars``;
|
||||
passes (1) and (3) keep their own fixed floors. The recent-tail
|
||||
protection applies to passes (2) and (3); pass (1) is tail-agnostic by
|
||||
design because dedup is lossless.
|
||||
|
||||
PROMPT-CACHE CONTRACT: a committed prune rewrites message bodies the
|
||||
provider has already seen, invalidating the cached prefix from the
|
||||
earliest rewritten message forward — exactly like a compression
|
||||
boundary. To keep that break episodic rather than per-turn, the prune
|
||||
only COMMITS when the estimated reclaim meets
|
||||
``proactive_prune_min_reclaim_tokens`` (measured on the actual pruned
|
||||
output, not guessed up front). Below the gate the INPUT list object is
|
||||
returned unchanged — the standard no-op caller contract (callers gate
|
||||
bookkeeping on ``result is not input``).
|
||||
|
||||
Returns ``(messages, 0)`` — the input object — when disabled, below
|
||||
the trigger, or when the reclaim gate rejects the commit.
|
||||
"""
|
||||
if self.proactive_prune_tokens <= 0:
|
||||
return messages, 0
|
||||
if current_tokens is not None and current_tokens < self.proactive_prune_tokens:
|
||||
return messages, 0
|
||||
# Nothing to reclaim until there are messages outside the protected tail.
|
||||
if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1:
|
||||
return messages, 0
|
||||
pruned_msgs, pruned_count = self._prune_old_tool_results(
|
||||
messages,
|
||||
protect_tail_count=self.protect_last_n,
|
||||
protect_tail_tokens=None,
|
||||
min_prune_chars=self.proactive_prune_min_result_chars,
|
||||
)
|
||||
if not pruned_count:
|
||||
# Standard no-op contract: hand back the INPUT object so callers
|
||||
# can gate bookkeeping on `result is not input`.
|
||||
return messages, 0
|
||||
# Measured-savings gate (prompt-cache hysteresis): only commit when
|
||||
# the prune reclaims a meaningful batch of tokens. Estimated on the
|
||||
# real before/after messages so dedup + arg truncation count too.
|
||||
if self.proactive_prune_min_reclaim_tokens > 0:
|
||||
before = sum(_estimate_msg_budget_tokens(m) for m in messages)
|
||||
after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs)
|
||||
if (before - after) < self.proactive_prune_min_reclaim_tokens:
|
||||
return messages, 0
|
||||
return pruned_msgs, pruned_count
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Summarization
|
||||
# ------------------------------------------------------------------
|
||||
|
|
@ -2330,6 +2800,10 @@ class ContextCompressor(ContextEngine):
|
|||
_CONTENT_TAIL = 1500 # chars kept from the end
|
||||
_TOOL_ARGS_MAX = 1500 # tool call argument chars
|
||||
_TOOL_ARGS_HEAD = 1200 # kept from the start of tool args
|
||||
# Aggregate cap over the whole serialized block, applied AFTER the
|
||||
# per-message limits above. Alias of the module-level constant (which
|
||||
# carries the full rationale) so subclasses/tests can override per-class.
|
||||
_SUMMARY_INPUT_MAX_CHARS = _SUMMARY_INPUT_MAX_CHARS
|
||||
|
||||
def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str:
|
||||
"""Serialize conversation turns into labeled text for the summarizer.
|
||||
|
|
@ -2589,12 +3063,6 @@ Recovered from a deterministic fallback because the LLM context summarizer was u
|
|||
## Active State
|
||||
Unknown from deterministic fallback. Inspect current repository/session state if needed.
|
||||
|
||||
{HISTORICAL_IN_PROGRESS_HEADING}
|
||||
Unknown from deterministic fallback — the latest user ask is recorded once under
|
||||
"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an
|
||||
unfulfilled instruction to re-answer; verify current state and continue from the
|
||||
protected recent messages after this summary.
|
||||
|
||||
## Blocked
|
||||
{_bullets(blockers, limit=5)}
|
||||
|
||||
|
|
@ -2604,27 +3072,62 @@ None recoverable from deterministic fallback.
|
|||
## Resolved Questions
|
||||
None recoverable from deterministic fallback.
|
||||
|
||||
{HISTORICAL_PENDING_ASKS_HEADING}
|
||||
None recoverable from deterministic fallback. (The latest user ask is preserved once
|
||||
under "{HISTORICAL_TASK_HEADING}" as historical context — it is NOT necessarily
|
||||
outstanding.)
|
||||
|
||||
## Relevant Files
|
||||
{_bullets(relevant_files, limit=12)}
|
||||
|
||||
{HISTORICAL_REMAINING_WORK_HEADING}
|
||||
Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims.
|
||||
|
||||
## Last Dropped Turns
|
||||
{_bullets(last_dropped_turns, limit=8)}
|
||||
|
||||
## Critical Context
|
||||
Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}"""
|
||||
# Ghost-skill defense (#32106): the fallback's per-turn truncation
|
||||
# (``_FALLBACK_TURN_MAX_CHARS``) routinely cuts [SKILL_PRUNED: ...]
|
||||
# markers out of the compacted turns. Re-derive the ghosted skills
|
||||
# from the raw turn contents and re-inject deterministically,
|
||||
# exactly like the LLM-summary path.
|
||||
_pruned_names = _collect_ghosted_skill_names(turns_to_summarize)
|
||||
del _pruned_names[_MAX_PRUNED_SKILL_MARKERS:]
|
||||
summary = self._with_summary_prefix(_redact_compaction_text(body.strip()))
|
||||
if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS:
|
||||
summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]"
|
||||
# Re-inject AFTER the size cap: the markers live at the end of the
|
||||
# body, exactly where the truncation above cuts.
|
||||
summary = _reinject_pruned_skill_markers(summary, _pruned_names)
|
||||
return summary
|
||||
|
||||
@classmethod
|
||||
def _bound_summary_input(cls, content: str) -> str:
|
||||
"""Cap total summarizer input while preserving beginning and recent tail.
|
||||
|
||||
Per-message truncation alone is not enough for very long sessions: a
|
||||
compression window with hundreds of messages can still produce a huge
|
||||
single prompt that slow auxiliary backends time out on. Keep both edges
|
||||
because the beginning often has task setup and the tail has the most
|
||||
recent state; explicitly mark the omitted middle so the summarizer knows
|
||||
context was intentionally compressed before it saw the prompt.
|
||||
"""
|
||||
if len(content) <= cls._SUMMARY_INPUT_MAX_CHARS:
|
||||
return content
|
||||
|
||||
marker_template = (
|
||||
"\n\n...[summary input truncated: omitted "
|
||||
"{omitted:,} chars from the middle to keep compression prompt bounded]...\n\n"
|
||||
)
|
||||
# Estimate once, then rebuild with the exact omitted span after the
|
||||
# head/tail split is known. The second marker can differ by a few chars
|
||||
# if the comma-formatted number changes width, so recompute once.
|
||||
marker = marker_template.format(omitted=len(content))
|
||||
remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0)
|
||||
head_chars = int(remaining * 0.45)
|
||||
tail_chars = remaining - head_chars
|
||||
omitted = max(len(content) - head_chars - tail_chars, 0)
|
||||
marker = marker_template.format(omitted=omitted)
|
||||
remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0)
|
||||
head_chars = int(remaining * 0.45)
|
||||
tail_chars = remaining - head_chars
|
||||
tail = content[-tail_chars:].lstrip() if tail_chars else ""
|
||||
return content[:head_chars].rstrip() + marker + tail
|
||||
|
||||
def _fallback_to_main_for_compression(self, e: Exception, reason: str) -> None:
|
||||
"""Switch from a separate ``summary_model`` back to the main model.
|
||||
|
||||
|
|
@ -2698,6 +3201,23 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
|||
|
||||
summary_budget = self._compute_summary_budget(turns_to_summarize)
|
||||
content_to_summarize = self._serialize_for_summary(turns_to_summarize)
|
||||
# P2 ghost-skill defense (#32106): [SKILL_PRUNED: ...] markers entering
|
||||
# the summarizer are prompt INPUT only — LLMs routinely paraphrase them
|
||||
# into vague prose ("some skills were loaded"), which erases the reload
|
||||
# instruction. Collect the ghosted skills deterministically BEFORE the
|
||||
# call (both already-pruned marker rows AND raw skill_view bodies whose
|
||||
# instructions are about to be summarized away);
|
||||
# ``_reinject_pruned_skill_markers`` restores any marker the model
|
||||
# dropped AFTER the call. Markers already carried by the previous
|
||||
# summary must survive iterative rewrites the same way. Collection
|
||||
# walks the turn LIST, so the serialized input bound below cannot
|
||||
# hide a marker in its omitted middle.
|
||||
_pruned_skill_names = _collect_ghosted_skill_names(turns_to_summarize)
|
||||
for _name in _extract_pruned_skill_names(self._previous_summary or ""):
|
||||
if _name not in _pruned_skill_names:
|
||||
_pruned_skill_names.append(_name)
|
||||
del _pruned_skill_names[_MAX_PRUNED_SKILL_MARKERS:]
|
||||
content_to_summarize = self._bound_summary_input(content_to_summarize)
|
||||
_sanitized_memory_context = sanitize_memory_context(memory_context)
|
||||
_serialized_memory_context = json.dumps(
|
||||
_sanitized_memory_context,
|
||||
|
|
@ -2865,9 +3385,6 @@ Be specific with file paths, commands, line numbers, and results.]
|
|||
- Any running processes or servers
|
||||
- Environment details that matter]
|
||||
|
||||
{HISTORICAL_IN_PROGRESS_HEADING}
|
||||
[Work currently underway — what was being done when compaction fired]
|
||||
|
||||
## Blocked
|
||||
[Any blockers, errors, or issues not yet resolved. Include exact error messages.]
|
||||
|
||||
|
|
@ -2877,30 +3394,39 @@ Be specific with file paths, commands, line numbers, and results.]
|
|||
## Resolved Questions
|
||||
{_resolved_questions_instructions}
|
||||
|
||||
{HISTORICAL_PENDING_ASKS_HEADING}
|
||||
{_pending_asks_instructions}
|
||||
|
||||
## Relevant Files
|
||||
[Files read, modified, or created — with brief note on each]
|
||||
|
||||
{HISTORICAL_REMAINING_WORK_HEADING}
|
||||
[What remains to be done — framed as STALE context for reference only. The agent must NOT resume this work unless the latest user message explicitly asks for it.]
|
||||
|
||||
## Critical Context
|
||||
[Any specific values, error messages, configuration details, or data that would be lost without explicit preservation. NEVER include API keys, tokens, passwords, or credentials — write [REDACTED] instead.]
|
||||
|
||||
{_PRUNED_SKILLS_SECTION_HEADING}
|
||||
[If any [SKILL_PRUNED: ...reload with skill_view(...)] markers appear in the input,
|
||||
repeat each one verbatim here — copy the exact text, do NOT paraphrase, summarize,
|
||||
or describe them. These markers tell the agent which skills must be reloaded before
|
||||
use. If none appear, omit this section entirely.]
|
||||
|
||||
Target ~{summary_budget} tokens. Be CONCRETE — include file paths, command outputs, error messages, line numbers, and specific values. Avoid vague descriptions like "made some changes" — say exactly what changed.
|
||||
{_temporal_anchoring_rule}
|
||||
Write only the summary body. Do not include any preamble or prefix."""
|
||||
|
||||
if self._previous_summary:
|
||||
# Iterative update: preserve existing info, add new progress
|
||||
# Iterative update: preserve existing info, add new progress.
|
||||
# Bound the previous-summary block with the same aggregate cap as
|
||||
# the serialized new turns: a normal summary is far below the cap
|
||||
# (the output side is held to a ~10K-token ceiling), but a
|
||||
# pathological handoff rehydrated from a persisted session can be
|
||||
# arbitrarily large — the iterative prompt (previous summary +
|
||||
# new turns) must stay bounded too.
|
||||
_bounded_previous_summary = self._bound_summary_input(
|
||||
self._previous_summary
|
||||
)
|
||||
prompt = f"""{_summarizer_preamble}
|
||||
|
||||
You are updating a context compaction summary. A previous compaction produced the summary below. New conversation turns have occurred since then and need to be incorporated.
|
||||
|
||||
PREVIOUS SUMMARY:
|
||||
{self._previous_summary}
|
||||
{_bounded_previous_summary}
|
||||
|
||||
NEW TURNS TO INCORPORATE:
|
||||
{content_to_summarize}{_memory_section}
|
||||
|
|
@ -3031,6 +3557,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
# Redact the summary output as well — the summarizer LLM may
|
||||
# ignore prompt instructions and echo back secrets verbatim.
|
||||
summary = _redact_compaction_text(content.strip())
|
||||
# P2 ghost-skill defense (#32106): deterministically restore any
|
||||
# [SKILL_PRUNED: ...] marker the summarizer paraphrased away.
|
||||
summary = _reinject_pruned_skill_markers(summary, _pruned_skill_names)
|
||||
summary = self._ground_historical_task_snapshot(summary, turns_to_summarize)
|
||||
self._validate_summary_user_provenance(summary, has_user_turn)
|
||||
# Store for iterative updates on next compaction
|
||||
|
|
@ -4091,6 +4620,69 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
return max(pair_end, head_end + 1)
|
||||
return adjusted
|
||||
|
||||
def _ensure_last_n_user_messages_in_tail(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
cut_idx: int,
|
||||
head_end: int,
|
||||
n: int,
|
||||
) -> int:
|
||||
"""Guarantee the last N actionable user messages are in the protected tail.
|
||||
|
||||
Generalizes ``_ensure_last_user_message_in_tail`` to preserve an
|
||||
arbitrary number of recent user messages. This prevents the token-
|
||||
budget-based tail cut from consuming recent conversation turns
|
||||
when large tool outputs fill the budget.
|
||||
|
||||
When *n* <= 1, delegates directly to the existing single-message
|
||||
method for byte-identical regression safety.
|
||||
|
||||
If the conversation has fewer than *n* user messages, the earliest
|
||||
available user message is used without error.
|
||||
|
||||
Only REAL actionable user turns count toward N — the collector uses
|
||||
the same ``_is_actionable_user_turn`` /
|
||||
``_is_synthetic_compression_user_turn`` pair as
|
||||
``_find_last_user_message_idx``, so blank platform echoes, compaction
|
||||
handoffs, continuation markers, and todo-snapshot rows never consume
|
||||
a slot (#69291 bug class).
|
||||
|
||||
A user message is already a clean boundary — there is no
|
||||
tool_call/result group that spans across it, so
|
||||
``_align_boundary_backward`` is intentionally NOT called.
|
||||
Calling it can pull the cut past the user message into the
|
||||
preceding assistant(tool_calls)→tool group and split it (#22566).
|
||||
"""
|
||||
if n <= 1:
|
||||
return self._ensure_last_user_message_in_tail(messages, cut_idx, head_end)
|
||||
|
||||
# Collect real user message indices walking backward from end.
|
||||
# Mirror _find_last_user_message_idx's filters: compaction handoffs,
|
||||
# blank platform echoes, and synthetic continuation/todo rows are
|
||||
# continuity artifacts, not real user turns.
|
||||
user_indices = []
|
||||
for i in range(len(messages) - 1, head_end - 1, -1):
|
||||
msg = messages[i]
|
||||
if (
|
||||
self._is_actionable_user_turn(msg)
|
||||
and not self._is_synthetic_compression_user_turn(msg)
|
||||
):
|
||||
user_indices.append(i)
|
||||
|
||||
if len(user_indices) == 0:
|
||||
return cut_idx
|
||||
|
||||
if len(user_indices) < n:
|
||||
target_idx = user_indices[-1]
|
||||
else:
|
||||
target_idx = user_indices[n - 1]
|
||||
|
||||
if target_idx >= cut_idx:
|
||||
return cut_idx
|
||||
|
||||
cut_idx = target_idx
|
||||
return max(cut_idx, head_end + 1)
|
||||
|
||||
def _find_turn_pair_end(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
|
|
@ -4220,6 +4812,26 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
# monotonic — the tail can only grow, never shrink.
|
||||
cut_idx = self._ensure_last_assistant_message_in_tail(messages, cut_idx, head_end)
|
||||
|
||||
# Extend to the last N actionable user messages when configured
|
||||
# (compression.min_tail_user_messages > 1). This prevents the
|
||||
# token-budget tail from consuming recent turns when large tool
|
||||
# outputs fill the budget. The anchor only walks ``cut_idx``
|
||||
# backward (monotonic — the tail can only grow, never shrink), and
|
||||
# a user message is a clean boundary, so the forward re-alignment
|
||||
# below remains a no-op for the anchored index. Gated at the call
|
||||
# site so the default (1) path is byte-identical to the historical
|
||||
# single-anchor pipeline — the single-user anchor already ran above,
|
||||
# and re-invoking it here could re-trigger the causal-coupling
|
||||
# forward push (#22523) after the assistant anchor adjusted the cut.
|
||||
# getattr-guarded: bare ``ContextCompressor.__new__`` test doubles
|
||||
# (and plugin engines) skip __init__, so the attribute may be absent
|
||||
# (see the compression-path test-double pitfall).
|
||||
_min_tail_users = getattr(self, "min_tail_user_messages", 1)
|
||||
if isinstance(_min_tail_users, int) and not isinstance(_min_tail_users, bool) and _min_tail_users > 1:
|
||||
cut_idx = self._ensure_last_n_user_messages_in_tail(
|
||||
messages, cut_idx, head_end, _min_tail_users,
|
||||
)
|
||||
|
||||
# The floor guarantees forward progress — compression must always claim
|
||||
# at least one message or the caller's compress_start >= compress_end
|
||||
# guard turns the pass into a no-op that re-runs forever (the same loop
|
||||
|
|
@ -4521,6 +5133,39 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
)
|
||||
telemetry["chunk_count"] = 1 if turns_to_summarize else 0
|
||||
|
||||
if not turns_to_summarize:
|
||||
# The newest handoff summary consumed the entire compressible
|
||||
# window (every window row was a standalone handoff that strips
|
||||
# to None, and nothing follows it before compress_end) — there
|
||||
# is nothing new to summarize. Skip the summary call entirely:
|
||||
# without this guard the empty window still reached
|
||||
# _generate_summary, wasting an aux LLM call that aborts
|
||||
# noisily on empty input (#59496). Mirrors the sibling
|
||||
# "no compressable window" guard above (#40803): record an
|
||||
# ineffective strike through the durable write-through helper
|
||||
# so the anti-thrash breaker in should_compress() can stop the
|
||||
# loop — this shape cannot shrink, so every subsequent turn
|
||||
# would otherwise re-fire the same no-op. The rehydrated
|
||||
# _previous_summary is deliberately KEPT (not rolled back as
|
||||
# the summary-abort path does for #57835): it came from a
|
||||
# handoff genuinely present in this transcript, which is
|
||||
# returned unchanged.
|
||||
telemetry["failure_class"] = "empty_post_handoff_window"
|
||||
self._record_ineffective_compression_verdict(
|
||||
self._ineffective_compression_count + 1,
|
||||
)
|
||||
self._last_compression_savings_pct = 0.0
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Compression skipped: latest context summary leaves no "
|
||||
"new turns to summarize in window %d-%d. "
|
||||
"ineffective_compression_count=%d",
|
||||
compress_start,
|
||||
compress_end,
|
||||
self._ineffective_compression_count,
|
||||
)
|
||||
return messages
|
||||
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Context compression triggered (%d tokens >= %d threshold)",
|
||||
|
|
|
|||
|
|
@ -189,6 +189,144 @@ class ContextEngine(ABC):
|
|||
host filters unsupported optional arguments by signature.
|
||||
"""
|
||||
|
||||
# -- Optional: proactive tool-result prune -----------------------------
|
||||
|
||||
def prune_tool_results_only(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
current_tokens: int | None = None,
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""Deterministically trim old tool-result payloads without an LLM call.
|
||||
|
||||
Runs on a low, cost-oriented trigger independent of ``should_compress``
|
||||
so large-window engines can reclaim re-sent tool output long before full
|
||||
compaction would fire. Returns ``(messages, n_pruned)``.
|
||||
|
||||
Default is a safe no-op: the list is returned unchanged with ``0``
|
||||
pruned. Engines that don't implement a cheap prune — and any engine that
|
||||
predates this hook — inherit this default, so the agent loop's
|
||||
post-tool-call prune path never raises ``AttributeError`` on them. The
|
||||
built-in ContextCompressor overrides this with the real implementation.
|
||||
"""
|
||||
return messages, 0
|
||||
|
||||
# -- Optional: per-turn context selection (distinct from compression) --
|
||||
|
||||
def select_context(
|
||||
self,
|
||||
request_messages: List[Dict[str, Any]],
|
||||
*,
|
||||
conversation_messages: List[Dict[str, Any]] = None,
|
||||
incoming_message: Dict[str, Any] = None,
|
||||
budget_tokens: int = 0,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Optionally choose/replace the context for THIS request, pre-generation.
|
||||
|
||||
Called every turn after the request message list is assembled and
|
||||
before it is dispatched to the provider — independent of
|
||||
``should_compress()``. This lets an engine *select* which context
|
||||
enters the prompt (retrieval, topic routing, role/branch switching)
|
||||
rather than *shrink* context that is already there. The two verbs are
|
||||
orthogonal:
|
||||
|
||||
- ``compress()`` : context is too long -> make it shorter.
|
||||
- ``select_context()``: this turn belongs to a different context
|
||||
-> use that one instead.
|
||||
|
||||
Without this hook, engines that need per-turn access to the message
|
||||
list have to force ``should_compress()`` to return ``True`` so that
|
||||
``compress()`` is invoked every turn purely as a callback — which
|
||||
conflates selection with compression and degrades behaviour when the
|
||||
engine's backend is unavailable. ``select_context()`` removes the need
|
||||
for that workaround.
|
||||
|
||||
The returned list is request-only: it replaces the messages sent to
|
||||
the provider for this single call and MUST NOT be treated as persisted
|
||||
transcript state. The conversation history in the session DB is left
|
||||
untouched, so nothing leaks across turns. Return ``None`` to leave the
|
||||
request unchanged.
|
||||
|
||||
Unlike the ``pre_llm_call`` plugin hook (which appends to the user
|
||||
message and intentionally never rewrites the list, to preserve the
|
||||
cache prefix), ``select_context()`` may *replace* the message list.
|
||||
|
||||
Ordering / cache contract: the host runs this hook **before** prompt
|
||||
cache-control and **before** every request sanitizer (orphaned-tool
|
||||
cleanup, thinking-only/role normalization, whitespace/JSON
|
||||
normalization). So (a) whatever the hook returns still passes through
|
||||
the same validation as any request — a malformed replacement cannot
|
||||
reach the provider — and (b) prompt-cache stability (an AGENTS.md
|
||||
invariant) is preserved: the default no-op leaves the request
|
||||
byte-identical, so cache behaviour is unchanged for the built-in
|
||||
compressor and any non-implementing engine. An engine that *does*
|
||||
replace the list changes its own cache prefix by definition; that is
|
||||
the engine's concern, and cache-control breakpoints are re-derived on
|
||||
the selected list. The hook is evaluated per provider request (so it
|
||||
re-runs on retries within a turn), consistent with "select the context
|
||||
for THIS request".
|
||||
|
||||
Args:
|
||||
request_messages: The assembled request message list (system
|
||||
prompt + history + any ephemeral prefill), in OpenAI format.
|
||||
conversation_messages: The unmodified persisted conversation
|
||||
history, for reference only (do not mutate).
|
||||
incoming_message: The current turn's user message, if available.
|
||||
budget_tokens: The active model's context length, or 0 if unknown.
|
||||
|
||||
Default returns ``None`` (no-op) — zero impact on the built-in
|
||||
compressor or any existing engine.
|
||||
"""
|
||||
return None
|
||||
|
||||
def on_turn_complete(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
usage: Dict[str, Any] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Observe a finished user turn (post-turn ingestion / observation).
|
||||
|
||||
Called from the standard turn-finalization path once the assistant/tool
|
||||
loop completes, with the finalized in-memory transcript snapshot. This
|
||||
is the complement to ``select_context()``: selection happens *before*
|
||||
the request, while observation happens *after* the turn. It lets an
|
||||
engine ingest, index, summarize, or update routing / topic / session
|
||||
state from what actually happened — so the next ``select_context()``
|
||||
can act on it.
|
||||
|
||||
Coverage: this fires from the normal finalization seam. Some abnormal
|
||||
early-return paths in the loop (e.g. a content-policy block or a
|
||||
provider terminal failure) persist and return without routing through
|
||||
finalization, and therefore do not currently emit this hook. Treat it
|
||||
as a best-effort post-turn observation for completed turns, not a
|
||||
guaranteed callback for every possible early exit; unifying all
|
||||
terminal paths behind one finalization seam is a separate follow-up.
|
||||
|
||||
Together the two hooks remove the need to abuse ``should_compress()`` /
|
||||
``compress()`` as a generic per-turn callback just to observe history,
|
||||
and they cover the case where a turn finishes and there may be no next
|
||||
request from which to infer the previous turn.
|
||||
|
||||
``messages`` is a shallow copy and should be treated as read-only:
|
||||
return values are ignored and this hook must not rely on transcript
|
||||
mutation for persistence. ``kwargs`` may include ``turn_id``,
|
||||
``task_id``, ``api_call_count``, ``interrupted``, ``failed``, and
|
||||
``turn_exit_reason``.
|
||||
|
||||
``usage`` carries the completed turn's canonical token usage (the same
|
||||
dict shape passed to ``update_from_response`` — ``prompt_tokens`` /
|
||||
``completion_tokens`` / ``total_tokens`` plus the canonical
|
||||
``input_tokens`` / ``output_tokens`` / ``cache_read_tokens`` /
|
||||
``cache_write_tokens`` / ``reasoning_tokens`` buckets) so an engine can
|
||||
weigh how large/expensive the selected context actually was when
|
||||
deciding the next ``select_context()``. It is ``None`` on finalized
|
||||
turns that never reached a provider response (e.g. interrupt); engines
|
||||
must treat it as optional.
|
||||
|
||||
Default is a no-op.
|
||||
"""
|
||||
return None
|
||||
|
||||
# -- Optional: pre-flight check ----------------------------------------
|
||||
|
||||
def should_compress_preflight(self, messages: List[Dict[str, Any]]) -> bool:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ REFERENCE_PATTERN = re.compile(
|
|||
rf"(?<![\w/])@(?:(?P<simple>diff|staged)\b|(?P<kind>file|folder|git|url):(?P<value>{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))"
|
||||
)
|
||||
TRAILING_PUNCTUATION = ",.;!?"
|
||||
_NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""")
|
||||
_SENSITIVE_HOME_DIRS = (".ssh", ".aws", ".gnupg", ".kube", ".docker", ".azure", ".config/gh")
|
||||
_SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",)
|
||||
_SENSITIVE_HOME_FILES = (
|
||||
|
|
@ -60,6 +61,21 @@ class ContextReferenceResult:
|
|||
blocked: bool = False
|
||||
|
||||
|
||||
def format_reference_value(value: str) -> str:
|
||||
"""Quote a reference value so ``REFERENCE_PATTERN`` reads it back whole.
|
||||
|
||||
The unquoted alternative in the pattern is ``\\S+``, so a path containing a
|
||||
space parses as a truncated ref with the tail left behind as loose text.
|
||||
Mirrors ``formatRefValue`` in the desktop's directive-text.tsx.
|
||||
"""
|
||||
if not _NEEDS_QUOTING.search(value):
|
||||
return value
|
||||
for quote in ("`", '"', "'"):
|
||||
if quote not in value:
|
||||
return f"{quote}{value}{quote}"
|
||||
return value
|
||||
|
||||
|
||||
def parse_context_references(message: str) -> list[ContextReference]:
|
||||
refs: list[ContextReference] = []
|
||||
if not message:
|
||||
|
|
@ -308,7 +324,7 @@ def _expand_git_reference(
|
|||
["git", *args],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=30,
|
||||
stdin=subprocess.DEVNULL,
|
||||
**_popen_kwargs,
|
||||
|
|
@ -534,7 +550,7 @@ def _rg_files(path: Path, cwd: Path, limit: int) -> list[Path] | None:
|
|||
["rg", "--files", str(path.relative_to(cwd))],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=10,
|
||||
stdin=subprocess.DEVNULL,
|
||||
**_popen_kwargs,
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import uuid
|
|||
import threading
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from agent.context_engine import (
|
||||
automatic_compaction_status_message,
|
||||
|
|
@ -221,6 +221,25 @@ class CompressionCommitFence:
|
|||
self._lock = threading.Lock()
|
||||
self._cancelled = False
|
||||
self._commit_started = False
|
||||
# Forward-progress telemetry: the compression worker touches this
|
||||
# whenever the streamed summary call produces a token (see
|
||||
# ContextCompressor._call_summary_llm). Waiters use it to distinguish
|
||||
# a SLOW-but-alive summary model from a HUNG one, so slow models are
|
||||
# not killed by a fixed wall-clock deadline while tokens are moving.
|
||||
self._last_progress = time.monotonic()
|
||||
|
||||
def touch_progress(self) -> None:
|
||||
"""Record forward progress (e.g. a streamed summary token arriving).
|
||||
|
||||
Called from the compression worker thread; read by async waiters via
|
||||
:meth:`seconds_since_progress`. A bare float store is atomic in
|
||||
CPython, so no lock is needed.
|
||||
"""
|
||||
self._last_progress = time.monotonic()
|
||||
|
||||
def seconds_since_progress(self) -> float:
|
||||
"""Seconds since the worker last reported forward progress."""
|
||||
return max(0.0, time.monotonic() - self._last_progress)
|
||||
|
||||
def cancel_before_commit(self) -> bool:
|
||||
"""Cancel a pending commit, or wait for an active commit to finish.
|
||||
|
|
@ -355,6 +374,143 @@ def _emit_compression_attempt_telemetry(
|
|||
logger.debug("failed to emit compression attempt telemetry: %s", exc)
|
||||
|
||||
|
||||
def compression_skipped_due_to_lock(agent: Any) -> bool:
|
||||
"""Type-pinned read of the #69870 lock-skip signal.
|
||||
|
||||
``agent._compression_skipped_due_to_lock`` is set by ``compress_context``
|
||||
when a compression pass no-ops because another path holds the per-session
|
||||
compression lock (holder string when the holder was confirmed, ``True``
|
||||
otherwise) and cleared to ``None`` at the entry of every call.
|
||||
|
||||
The read MUST be type-pinned (``is True or isinstance(x, str)``), never
|
||||
bare truthiness: MagicMock test-double agents auto-create truthy
|
||||
attributes, and a bare ``if getattr(agent, ...)`` would hijack every
|
||||
mocked agent in sibling suites into the lock-skip branch (the
|
||||
#69870 × #69840 type-ahead incident).
|
||||
"""
|
||||
_sig = getattr(agent, "_compression_skipped_due_to_lock", None)
|
||||
return _sig is True or isinstance(_sig, str)
|
||||
|
||||
|
||||
def _adopt_live_compression_child(
|
||||
agent: Any,
|
||||
session_db: Any,
|
||||
parent_session_id: str,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Move a stale compression contender onto the unique durable child.
|
||||
|
||||
Resolve and load first, then mutate the live agent. This ordering keeps the
|
||||
stale contender fail-closed when lineage is ambiguous or the compacted
|
||||
handoff cannot be read.
|
||||
"""
|
||||
finder = getattr(type(session_db), "find_live_compression_child", None)
|
||||
loader = getattr(type(session_db), "get_messages_as_conversation", None)
|
||||
if not callable(finder) or not callable(loader):
|
||||
return None
|
||||
child = finder(session_db, parent_session_id)
|
||||
if not child or not child.get("id"):
|
||||
return None
|
||||
child_session_id = str(child["id"])
|
||||
recovered = loader(session_db, child_session_id)
|
||||
if not isinstance(recovered, list) or not recovered:
|
||||
return None
|
||||
# Revalidate after loading: the child may have rotated or a competing
|
||||
# continuation may have appeared between the two DB reads.
|
||||
confirmed = finder(session_db, parent_session_id)
|
||||
if not confirmed or str(confirmed.get("id") or "") != child_session_id:
|
||||
return None
|
||||
|
||||
agent.session_id = child_session_id
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
|
||||
set_current_session_id(child_session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = child_session_id
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
|
||||
set_session_context(child_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
agent._session_db_created = True
|
||||
if child.get("system_prompt"):
|
||||
agent._cached_system_prompt = child["system_prompt"]
|
||||
agent._last_flushed_db_idx = len(recovered)
|
||||
agent._flushed_db_message_session_id = child_session_id
|
||||
agent._flushed_db_message_ids = {
|
||||
id(message) for message in recovered if isinstance(message, dict)
|
||||
}
|
||||
|
||||
on_session_start = getattr(agent.context_compressor, "on_session_start", None)
|
||||
if callable(on_session_start):
|
||||
try:
|
||||
on_session_start(
|
||||
child_session_id,
|
||||
boundary_reason="compression",
|
||||
old_session_id=parent_session_id,
|
||||
session_db=session_db,
|
||||
platform=getattr(agent, "platform", None) or "cli",
|
||||
conversation_id=getattr(agent, "_gateway_session_key", None),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("context engine compression-child adoption failed: %s", exc)
|
||||
else:
|
||||
bind_state = getattr(agent.context_compressor, "bind_session_state", None)
|
||||
if callable(bind_state):
|
||||
try:
|
||||
bind_state(session_db=session_db, session_id=child_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if agent._memory_manager:
|
||||
agent._memory_manager.on_session_switch(
|
||||
child_session_id,
|
||||
parent_session_id=parent_session_id,
|
||||
reset=False,
|
||||
reason="compression",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("memory manager compression-child adoption failed: %s", exc)
|
||||
|
||||
return recovered
|
||||
|
||||
|
||||
def recover_rotated_compression_session(
|
||||
agent: Any,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Recover a stale live agent before a new turn writes to its old parent."""
|
||||
session_db = getattr(agent, "_session_db", None)
|
||||
session_id = getattr(agent, "session_id", None) or ""
|
||||
if session_db is None or not session_id:
|
||||
return None
|
||||
try:
|
||||
if not _session_was_rotated_by_compression(session_db, session_id):
|
||||
return None
|
||||
# Rotation publication holds the parent compression lease until the
|
||||
# child handoff is durable. A concurrent turn waits briefly rather than
|
||||
# observing the intentional parent-ended/child-empty intermediate state.
|
||||
holder_getter = getattr(session_db, "get_compression_lock_holder", None)
|
||||
for attempt in range(21):
|
||||
recovered = _adopt_live_compression_child(agent, session_db, session_id)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
holder = holder_getter(session_id) if callable(holder_getter) else None
|
||||
if not holder or attempt == 20:
|
||||
return None
|
||||
time.sleep(0.05)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"compression session recovery failed for session=%s (%s: %s)",
|
||||
session_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _compression_lock_holder(agent: Any) -> str:
|
||||
"""Build a unique holder id for the lock: pid:tid:agent-instance:uuid.
|
||||
|
||||
|
|
@ -515,7 +671,17 @@ class _CompressionLockLeaseRefresher:
|
|||
# by the TTL the acquirer set — the lock can never be held past its TTL
|
||||
# by a stuck refresher.
|
||||
consecutive_failures = 0
|
||||
while not self._stop.wait(self._refresh_interval_seconds):
|
||||
# First refresh happens immediately, not one interval late. Everything
|
||||
# between try_acquire() and start() (the rotation-ownership lookup, the
|
||||
# durable-breaker re-read, thread startup) is charged against the very
|
||||
# first lease, so on a short TTL under load the lock could already be
|
||||
# expired — and reclaimable by a competing path — before tick #1.
|
||||
first = True
|
||||
while first or not self._stop.wait(self._refresh_interval_seconds):
|
||||
if first:
|
||||
first = False
|
||||
if self._stop.is_set():
|
||||
break
|
||||
try:
|
||||
refreshed = self._db.refresh_compression_lock(
|
||||
self._session_id,
|
||||
|
|
@ -874,6 +1040,7 @@ _SYNTHETIC_USER_FLAGS = (
|
|||
"_empty_recovery_synthetic",
|
||||
"_verification_stop_synthetic",
|
||||
"_pre_verify_synthetic",
|
||||
"_dropped_toolcall_nudge",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1143,6 +1310,14 @@ def compress_context(
|
|||
# boundary, so the previous flush baseline remains authoritative.
|
||||
agent._last_compression_attempt_recorded = True
|
||||
agent._last_compression_attempt_in_place = None
|
||||
# Clear the lock-skip signal at the VERY TOP, before the codex route and
|
||||
# the breaker gates below can early-return (per-attempt state rule,
|
||||
# #58630/#69853). A stale ``True``/holder value from a prior lock-skip
|
||||
# must never make a later breaker/codex no-op look like lock contention
|
||||
# to the automatic-path consumers (compression_deferred, #49874) — the
|
||||
# second clear before lock acquisition below stays for the same reason
|
||||
# it was added in #69870 and is simply idempotent now.
|
||||
agent._compression_skipped_due_to_lock = None
|
||||
|
||||
_attempt_started_at = time.monotonic()
|
||||
_attempt_id = uuid.uuid4().hex
|
||||
|
|
@ -1226,7 +1401,7 @@ def compress_context(
|
|||
# parent_session_id child, no
|
||||
# `name #N` renumber, no contextvar/env/logging re-sync, no memory/context-
|
||||
# engine session-switch. The conversation keeps one durable id for life,
|
||||
# eliminating the session-rotation bug cluster. Default False during rollout.
|
||||
# eliminating the session-rotation bug cluster. Default True (2107b86024).
|
||||
in_place = bool(getattr(agent, "compression_in_place", False))
|
||||
# Set True once the in-place DB write actually completes (the DB block can
|
||||
# raise and skip it). Surfaced to the gateway via agent._last_compaction_in_place.
|
||||
|
|
@ -1434,6 +1609,8 @@ def compress_context(
|
|||
if _lock_released:
|
||||
return
|
||||
_lock_released = True
|
||||
if getattr(agent, "_active_compression_lock_holder", None) == _lock_holder:
|
||||
agent._active_compression_lock_holder = None
|
||||
if _lock_refresher is not None:
|
||||
try:
|
||||
_lock_refresher.stop()
|
||||
|
|
@ -1445,6 +1622,9 @@ def compress_context(
|
|||
except Exception as _rel_err:
|
||||
logger.debug("compression lock release failed: %s", _rel_err)
|
||||
|
||||
if _lock_holder is not None:
|
||||
agent._active_compression_lock_holder = _lock_holder
|
||||
|
||||
# A delayed contender can acquire the parent lock after the winning path
|
||||
# has released it and completed rotation. The lock serializes work but does
|
||||
# not by itself prove that this stale agent still owns a live parent.
|
||||
|
|
@ -1467,15 +1647,25 @@ def compress_context(
|
|||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
return messages, _existing_sp
|
||||
if _parent_already_rotated:
|
||||
logger.info(
|
||||
"compression skipped: session=%s was already rotated by "
|
||||
"another compression path",
|
||||
_lock_sid,
|
||||
recovered_messages = _adopt_live_compression_child(
|
||||
agent, _lock_db, _lock_sid
|
||||
)
|
||||
_release_lock()
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
if recovered_messages is not None:
|
||||
logger.warning(
|
||||
"compression recovery: stale session=%s adopted live child=%s",
|
||||
_lock_sid,
|
||||
agent.session_id,
|
||||
)
|
||||
return recovered_messages, _existing_sp
|
||||
logger.warning(
|
||||
"compression skipped: session=%s was already rotated by "
|
||||
"another compression path, but no unique live child could be adopted",
|
||||
_lock_sid,
|
||||
)
|
||||
return messages, _existing_sp
|
||||
|
||||
# The agent may have been constructed before another path completed an
|
||||
|
|
@ -1509,6 +1699,37 @@ def compress_context(
|
|||
)
|
||||
_lock_refresher.start()
|
||||
|
||||
# The caller's history snapshot predates lease acquisition. Reload the
|
||||
# durable parent after the lease is live; MORE durable rows than the
|
||||
# snapshot carries means a frontend/background writer committed a turn
|
||||
# in that window, so publishing from this snapshot would omit it.
|
||||
# Deliberately a LENGTH check, not content equality: in-memory
|
||||
# mutation of past turns is legal (multimodal compression, retry
|
||||
# history replacement, think-tag stripping), and a content-equality
|
||||
# abort would permanently wedge compression on such sessions — the
|
||||
# #14694 failure shape.
|
||||
# Rotation-only: in-place compaction (archive_and_compact) is
|
||||
# non-destructive — pre-compaction rows are soft-archived (active=0,
|
||||
# compacted=1), stay searchable and recoverable, so snapshot/durable
|
||||
# drift cannot lose data there and must not abort compaction.
|
||||
if not in_place and _lock_db is not None and _lock_sid:
|
||||
durable_loader = getattr(
|
||||
type(_lock_db), "get_messages_as_conversation", None
|
||||
)
|
||||
if callable(durable_loader):
|
||||
durable_parent = durable_loader(_lock_db, _lock_sid)
|
||||
if isinstance(durable_parent, list) and len(durable_parent) > len(messages):
|
||||
logger.warning(
|
||||
"compression aborted: session=%s changed before lease "
|
||||
"acquisition; preserving newer durable messages",
|
||||
_lock_sid,
|
||||
)
|
||||
_release_lock()
|
||||
existing_prompt = getattr(agent, "_cached_system_prompt", None)
|
||||
if not existing_prompt:
|
||||
existing_prompt = agent._build_system_prompt(system_message)
|
||||
return messages, existing_prompt
|
||||
|
||||
# Notify external memory provider before compression discards context.
|
||||
# The provider's on_pre_compress() may return a string of insights it
|
||||
# wants surfaced inside the compression summary; capture and forward it
|
||||
|
|
@ -1549,7 +1770,29 @@ def compress_context(
|
|||
|
||||
messages_before_compression = copy.deepcopy(messages)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
# Publish forward progress to the commit fence while the summary LLM
|
||||
# call streams. Async hosts (gateway session hygiene) poll
|
||||
# ``commit_fence.seconds_since_progress()`` to extend their deadline
|
||||
# while tokens are moving — so a SLOW summary model is only killed
|
||||
# when it is actually silent, not merely thorough. The hook is
|
||||
# thread-local and the compress call is synchronous on this thread,
|
||||
# so it cannot leak into unrelated auxiliary calls.
|
||||
#
|
||||
# Fenceless callers (CLI /compress, in-loop auto-compress) install a
|
||||
# no-op hook: nobody polls their progress, but an ACTIVE hook is what
|
||||
# switches the summary call onto the streamed path — giving every
|
||||
# compression path the same two guarantees: the configured timeout
|
||||
# acts on inactivity (slow models finish), and a byte-trickling
|
||||
# provider that keeps the connection alive forever is cut off at the
|
||||
# streamed total ceiling (see _aux_stream_total_ceiling) instead of
|
||||
# outliving the SDK's inactivity timeout indefinitely.
|
||||
from agent.auxiliary_client import aux_progress_hook
|
||||
_progress_hook = (
|
||||
commit_fence.touch_progress if commit_fence is not None
|
||||
else (lambda: None)
|
||||
)
|
||||
with aux_progress_hook(_progress_hook):
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
except BaseException as _compress_exc:
|
||||
# ANY exception after lock acquisition — memory hook, capability
|
||||
# inspection, engine lookup, or compress() — must release the lock so
|
||||
|
|
@ -1777,6 +2020,21 @@ def compress_context(
|
|||
):
|
||||
new_system_prompt = cached_system_prompt
|
||||
agent._cached_system_prompt = cached_system_prompt
|
||||
# _invalidate_system_prompt() above also cleared the
|
||||
# cross-session-stable prefix marker boundary. The kept prompt
|
||||
# is byte-identical, so reconstruct the stable tier and reuse
|
||||
# it ONLY when the kept prompt still literally starts with it
|
||||
# (same startswith gate as the restore path); otherwise the
|
||||
# request layer falls back to the legacy single-breakpoint
|
||||
# layout with the prompt bytes untouched.
|
||||
try:
|
||||
from agent.system_prompt import build_system_prompt_parts as _build_parts
|
||||
|
||||
_static = _build_parts(agent, system_message=system_message)["stable"]
|
||||
if _static and cached_system_prompt.startswith(_static):
|
||||
agent._cached_system_prompt_static = _static
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
new_system_prompt = agent._build_system_prompt(system_message)
|
||||
agent._cached_system_prompt = new_system_prompt
|
||||
|
|
@ -1853,81 +2111,54 @@ def compress_context(
|
|||
)
|
||||
except Exception:
|
||||
pass # best-effort — don't block compression on a flush error
|
||||
# Propagate title to the new session with auto-numbering
|
||||
# Publish parent closure + child row + compacted handoff in
|
||||
# one transaction. No reader can observe a missing/empty child.
|
||||
# The rotation child must stay on the parent's profile —
|
||||
# mirror _ensure_db_session's stamp ("default" persists as
|
||||
# NULL). publish_compression_child additionally COALESCEs
|
||||
# from the parent row, covering app-global remote sessions
|
||||
# whose thread lacks the HERMES_HOME context.
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
|
||||
_profile_for_child = get_active_profile_name()
|
||||
if _profile_for_child == "default":
|
||||
_profile_for_child = None
|
||||
except Exception:
|
||||
_profile_for_child = None
|
||||
old_title = agent._session_db.get_session_title(agent.session_id)
|
||||
agent._session_db.end_session(agent.session_id, "compression")
|
||||
old_session_id = agent.session_id
|
||||
agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
# Ordering contract: the agent thread updates the contextvar here;
|
||||
# the gateway propagates to SessionEntry after run_in_executor returns.
|
||||
new_session_id = (
|
||||
f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_"
|
||||
f"{uuid.uuid4().hex[:6]}"
|
||||
)
|
||||
agent._session_db.publish_compression_child(
|
||||
parent_session_id=old_session_id,
|
||||
child_session_id=new_session_id,
|
||||
source=agent.platform
|
||||
or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=agent.model,
|
||||
model_config=agent._session_init_model_config,
|
||||
system_prompt=new_system_prompt,
|
||||
messages=compressed,
|
||||
cwd=getattr(agent, "working_directory", None),
|
||||
profile_name=_profile_for_child,
|
||||
compression_lock_holder=_lock_holder,
|
||||
require_compression_lease=_lock_holder is not None,
|
||||
)
|
||||
agent.session_id = new_session_id
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
|
||||
set_current_session_id(agent.session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = agent.session_id
|
||||
# The gateway/tools session context (ContextVar + env) and the
|
||||
# logging session context are SEPARATE mechanisms. The call above
|
||||
# moves the former; the ``[session_id]`` tag on log lines comes
|
||||
# from ``hermes_logging._session_context`` (set once per turn in
|
||||
# conversation_loop.py). Without this, post-rotation log lines in
|
||||
# the same turn keep the STALE old id while the message/DB/gateway
|
||||
# state carry the new one — breaking log correlation exactly at the
|
||||
# compaction boundary (see #34089). Guarded separately so a logging
|
||||
# failure can never regress the routing update above.
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
|
||||
set_session_context(agent.session_id)
|
||||
except Exception:
|
||||
pass
|
||||
agent._session_db_created = False
|
||||
try:
|
||||
agent._session_db.create_session(
|
||||
session_id=agent.session_id,
|
||||
source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=agent.model,
|
||||
model_config=agent._session_init_model_config,
|
||||
parent_session_id=old_session_id,
|
||||
)
|
||||
except Exception as _cs_err:
|
||||
# The child row could not be created (e.g. FK constraint,
|
||||
# contended write). Previously the outer handler simply
|
||||
# warned and let the agent continue on the NEW id — which
|
||||
# has no row in state.db, producing an orphan: the parent
|
||||
# is ended, the child is never indexed, and every
|
||||
# subsequent message is attributed to a session that
|
||||
# doesn't exist (#33906/#33907). Roll the live id back to
|
||||
# the parent so the conversation stays attached to a real,
|
||||
# indexed session instead of a phantom.
|
||||
logger.warning(
|
||||
"Compression child session create failed (%s) — "
|
||||
"rolling back to parent session %s to avoid an orphan.",
|
||||
_cs_err, old_session_id,
|
||||
)
|
||||
agent.session_id = old_session_id
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
set_current_session_id(agent.session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = agent.session_id
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
set_session_context(agent.session_id)
|
||||
except Exception:
|
||||
pass
|
||||
# Re-open the parent: it was ended above, but we're
|
||||
# continuing on it, so it must not stay closed.
|
||||
try:
|
||||
agent._session_db.reopen_session(old_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
old_session_id = None # no rotation happened
|
||||
# The parent row already exists in state.db, so mark the
|
||||
# session as created — _ensure_db_session would otherwise
|
||||
# retry a (harmless INSERT OR IGNORE) create next turn.
|
||||
agent._session_db_created = True
|
||||
raise
|
||||
agent._session_db_created = True
|
||||
split_status = "rotated_committed"
|
||||
# Carry a persistent /goal onto the continuation session.
|
||||
|
|
@ -1947,18 +2178,14 @@ def compress_context(
|
|||
except (ValueError, Exception) as e:
|
||||
logger.debug("Could not propagate title on compression: %s", e)
|
||||
|
||||
# Shared post-write steps (both modes target agent.session_id, which
|
||||
# in-place keeps and rotation has already reassigned to the new id):
|
||||
# refresh the stored system prompt and reset the flush cursor so the
|
||||
# next turn re-bases its append diff.
|
||||
agent._session_db.update_system_prompt(agent.session_id, new_system_prompt)
|
||||
# In-place mode still updates/replaces the current row here.
|
||||
# Rotation already published prompt + compacted handoff atomically.
|
||||
if in_place:
|
||||
agent._session_db.update_system_prompt(
|
||||
agent.session_id, new_system_prompt
|
||||
)
|
||||
agent._last_flushed_db_idx = 0
|
||||
else:
|
||||
# A headless turn can be killed before its finalizer. Persist
|
||||
# the rotated child's compacted handoff at the boundary so
|
||||
# the new session is immediately resumable.
|
||||
agent._session_db.replace_messages(agent.session_id, compressed)
|
||||
agent._last_flushed_db_idx = len(compressed)
|
||||
agent._flushed_db_message_session_id = agent.session_id
|
||||
agent._flushed_db_message_ids = {
|
||||
|
|
@ -1968,7 +2195,22 @@ def compress_context(
|
|||
}
|
||||
_session_commit_succeeded = True
|
||||
except Exception as e:
|
||||
split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed"
|
||||
if (
|
||||
not in_place
|
||||
and locals().get("old_session_id")
|
||||
and agent.session_id == old_session_id
|
||||
):
|
||||
# Atomic publication failed (including lease loss): keep the
|
||||
# parent live and discard the stale compacted snapshot.
|
||||
old_session_id = None
|
||||
messages[:] = copy.deepcopy(messages_before_compression)
|
||||
compressed = messages
|
||||
_compression_made_progress = False
|
||||
split_status = (
|
||||
"aborted"
|
||||
if locals().get("old_session_id") is None and not in_place
|
||||
else "failed_not_indexed"
|
||||
)
|
||||
# If the rotation rolled back to the parent (orphan-avoidance
|
||||
# above), agent.session_id is the still-indexed parent and
|
||||
# old_session_id was cleared — so this is recovery, not an
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from agent.conversation_compression import (
|
|||
COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE,
|
||||
COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE,
|
||||
PRE_API_COMPRESSION_STATUS_TEMPLATE,
|
||||
compression_skipped_due_to_lock,
|
||||
conversation_history_after_compression,
|
||||
)
|
||||
from agent.context_engine import automatic_compaction_status_message
|
||||
|
|
@ -47,6 +48,7 @@ from agent.turn_context import (
|
|||
reanchor_current_turn_user_idx,
|
||||
)
|
||||
from agent.turn_retry_state import TurnRetryState
|
||||
from agent.runtime_cwd import resolve_agent_cwd
|
||||
from agent.message_sanitization import (
|
||||
close_interrupted_tool_sequence,
|
||||
_repair_tool_call_arguments,
|
||||
|
|
@ -435,6 +437,37 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
|||
# Continuing session — reuse the exact system prompt from the
|
||||
# previous turn so the Anthropic cache prefix matches.
|
||||
agent._cached_system_prompt = stored_prompt
|
||||
# Reconstruct the cross-session-stable prefix for the early cache
|
||||
# breakpoint. The static prefix is not persisted (only the full
|
||||
# prompt is), so gateway surfaces that build a fresh AIAgent per
|
||||
# turn would otherwise lose the two-block system layout after the
|
||||
# first turn — flip-flopping the wire shape mid-conversation and
|
||||
# silently degrading to the legacy single-breakpoint layout.
|
||||
#
|
||||
# Safety: the rebuilt stable tier is used ONLY when the restored
|
||||
# prompt literally starts with it (checked here AND re-checked by
|
||||
# ``_apply_system_cache_markers``'s ``startswith`` gate). If any
|
||||
# stable-tier input changed since the prompt was persisted (skills
|
||||
# edited, identity changed), the prefix mismatches, ``_static``
|
||||
# stays None, and the request falls back to the legacy layout with
|
||||
# the restored prompt bytes untouched — never a rewritten prompt.
|
||||
#
|
||||
# Gated on ``_use_prompt_caching`` so non-Anthropic routes skip the
|
||||
# rebuild entirely (the static prefix is only consumed by
|
||||
# ``apply_anthropic_cache_control``).
|
||||
if getattr(agent, "_use_prompt_caching", False):
|
||||
try:
|
||||
from agent.system_prompt import build_system_prompt_parts as _build_parts
|
||||
|
||||
_static = _build_parts(agent, system_message=system_message)["stable"]
|
||||
if _static and stored_prompt.startswith(_static):
|
||||
agent._cached_system_prompt_static = _static
|
||||
except Exception:
|
||||
# Fail-open: restore continues with the legacy cache layout.
|
||||
logger.debug(
|
||||
"static system-prefix reconstruction failed on restore",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
if stored_prompt:
|
||||
stored_state = "stale_runtime"
|
||||
|
|
@ -507,9 +540,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
|||
|
||||
|
||||
def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
||||
"""Return False when the persisted Model/Provider lines are stale."""
|
||||
"""Return False when the persisted runtime-identity lines are stale."""
|
||||
|
||||
def line_value(label: str) -> str:
|
||||
"""Last matching line wins.
|
||||
|
||||
Safe ONLY for fields emitted in the volatile tier at the very END of
|
||||
the prompt (Model / Provider / Platform). User-supplied project
|
||||
context (AGENTS.md / CLAUDE.md / .cursorrules) is embedded in the
|
||||
middle context tier, so a last-match scan lets project prose shadow
|
||||
any field emitted EARLIER — see ``host_info_value``.
|
||||
"""
|
||||
prefix = f"{label}:"
|
||||
value = ""
|
||||
for line in prompt.splitlines():
|
||||
|
|
@ -517,6 +558,32 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
|||
value = line[len(prefix):].strip()
|
||||
return value
|
||||
|
||||
def host_info_value(label: str) -> str:
|
||||
"""Read a field from the prompt's own host-info block.
|
||||
|
||||
The host-info block (``build_environment_hints``) sits in the STABLE
|
||||
tier, ahead of the embedded project context files. A bare scan of the
|
||||
whole prompt would therefore match a user's ``AGENTS.md`` that merely
|
||||
contains a line starting with the same label, comparing runtime state
|
||||
against project prose. That mismatch never clears, so the check would
|
||||
reject the stored prompt on EVERY turn — rebuilding the system prompt
|
||||
each message and destroying the prefix cache for the whole session,
|
||||
which is far worse than the staleness this function guards against.
|
||||
|
||||
Anchor on the ``User home directory:`` line that immediately precedes
|
||||
the working-directory line in that block, and take the FIRST such
|
||||
occurrence, so only Hermes' own emitted block can satisfy the read.
|
||||
"""
|
||||
prefix = f"{label}:"
|
||||
lines = prompt.splitlines()
|
||||
for idx, line in enumerate(lines):
|
||||
if not line.startswith("User home directory:"):
|
||||
continue
|
||||
for candidate in lines[idx + 1: idx + 4]:
|
||||
if candidate.startswith(prefix):
|
||||
return candidate[len(prefix):].strip()
|
||||
return ""
|
||||
|
||||
stored_model = line_value("Model")
|
||||
current_model = str(getattr(agent, "model", "") or "").strip()
|
||||
if stored_model and current_model and stored_model != current_model:
|
||||
|
|
@ -527,6 +594,24 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
|||
if stored_provider and current_provider and stored_provider != current_provider:
|
||||
return False
|
||||
|
||||
# Detect cwd drift: if the stored prompt was built in a different working
|
||||
# directory, reuse would silently inject a stale path into the prefix cache.
|
||||
# Compare against resolve_agent_cwd() — the SAME resolver used to build the
|
||||
# prompt — so gateway/TUI sessions that set TERMINAL_CWD are not falsely
|
||||
# rejected (they would always differ from the launch dir's os.getcwd()).
|
||||
stored_cwd = host_info_value("Current working directory")
|
||||
if stored_cwd:
|
||||
if stored_cwd != str(resolve_agent_cwd()):
|
||||
return False
|
||||
|
||||
# Detect runtime-surface drift: the stored prompt records which platform it
|
||||
# was built for (e.g. "desktop" vs "cli"). Reusing a desktop-built prompt on
|
||||
# a terminal session (or vice versa) would inject the wrong runtime hints.
|
||||
stored_platform = line_value("Platform")
|
||||
current_platform = str(getattr(agent, "platform", "") or "").strip()
|
||||
if stored_platform and current_platform and stored_platform != current_platform:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -640,6 +725,55 @@ def _content_policy_blocked_result(
|
|||
}
|
||||
|
||||
|
||||
def _compression_deferred_result(
|
||||
agent,
|
||||
messages: List[Dict],
|
||||
api_call_count: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the soft turn result for a lock-contended compression defer.
|
||||
|
||||
Another path (a sibling turn, a background review fork, a manual
|
||||
``/compress``) holds this session's compression lock, so every
|
||||
compression pass this turn no-oped and the request still does not fit.
|
||||
This is a TEMPORARY condition — the lock winner is actively shrinking
|
||||
the same session — so the turn must end as a soft defer
|
||||
(``compression_deferred``), never as ``compression_exhausted``: the
|
||||
gateway auto-resets (wipes) the session on exhaustion (#9893/#35809),
|
||||
which would destroy a session that the concurrent compressor is about
|
||||
to make healthy again.
|
||||
|
||||
``failed`` stays False so the gateway persists the user turn (transient
|
||||
branch) and retry-next-message semantics apply.
|
||||
"""
|
||||
holder = getattr(agent, "_compression_skipped_due_to_lock", None)
|
||||
logger.info(
|
||||
"turn deferred: compression lock held by another path "
|
||||
"(session=%s holder=%s) — not counting as compression exhaustion",
|
||||
agent.session_id or "none",
|
||||
holder if isinstance(holder, str) else "unconfirmed",
|
||||
)
|
||||
try:
|
||||
agent._flush_status_buffer()
|
||||
except Exception:
|
||||
pass
|
||||
_final = (
|
||||
"Context compression is already running for this session. "
|
||||
"Please retry in a moment — your next message will be processed "
|
||||
"once the concurrent compression finishes."
|
||||
)
|
||||
return {
|
||||
"final_response": _final,
|
||||
"messages": messages,
|
||||
"completed": False,
|
||||
"api_calls": api_call_count,
|
||||
"error": _final,
|
||||
"partial": True,
|
||||
"failed": False,
|
||||
"compression_deferred": True,
|
||||
"session_id": agent.session_id,
|
||||
}
|
||||
|
||||
|
||||
def _sync_failover_system_message(agent, api_messages, active_system_prompt):
|
||||
"""Refresh the in-flight system message after a provider failover.
|
||||
|
||||
|
|
@ -666,6 +800,136 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt):
|
|||
return sp
|
||||
|
||||
|
||||
def _apply_context_engine_selection(
|
||||
agent: Any,
|
||||
api_messages: List[Dict[str, Any]],
|
||||
conversation_messages: List[Dict[str, Any]],
|
||||
incoming_message: Optional[Dict[str, Any]],
|
||||
*,
|
||||
logger: Any,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Run the optional per-turn ``ContextEngine.select_context()`` hook.
|
||||
|
||||
Returns the (possibly replaced) request message list. The hook is for
|
||||
context *selection / routing* (retrieval, topic routing, role switching),
|
||||
which is distinct from compression and fires every turn independent of
|
||||
``should_compress()``.
|
||||
|
||||
Fail-open by design: a missing hook, any exception, or an invalid return
|
||||
value yields the unmodified ``api_messages``. The result is request-only —
|
||||
persisted conversation history is never mutated here.
|
||||
"""
|
||||
engine = getattr(agent, "context_compressor", None)
|
||||
if engine is None or not hasattr(engine, "select_context"):
|
||||
return api_messages
|
||||
|
||||
# Skip the no-op base implementation so non-implementing engines —
|
||||
# including the built-in ContextCompressor — pay nothing per request:
|
||||
# no history copies below, no call. ``hasattr`` alone is not enough,
|
||||
# because the ABC defines a default ``select_context`` that every engine
|
||||
# inherits. Mirrors the base-method short-circuit in
|
||||
# ``_notify_context_engine_turn_complete``. Lazy import avoids any import
|
||||
# cycle with agent.context_engine.
|
||||
try:
|
||||
from agent.context_engine import ContextEngine as _CE
|
||||
if getattr(engine.select_context, "__func__", None) is _CE.select_context:
|
||||
return api_messages
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
session_label = getattr(agent, "session_id", None) or "-"
|
||||
# Pass shallow copies of the reference-only inputs so an engine that
|
||||
# mutates them in place cannot alter persisted transcript state. Only
|
||||
# ``request_messages`` (the per-call request list) is meant to be acted on,
|
||||
# and it may be replaced wholesale via the return value — never mutated in
|
||||
# place either. ``conversation_messages`` / ``incoming_message`` are
|
||||
# read-only context; copying enforces the request-only contract rather than
|
||||
# merely documenting it.
|
||||
_conv_copy = [dict(m) if isinstance(m, dict) else m for m in conversation_messages] \
|
||||
if conversation_messages is not None else None
|
||||
_incoming_copy = dict(incoming_message) if isinstance(incoming_message, dict) else incoming_message
|
||||
try:
|
||||
selected = engine.select_context(
|
||||
api_messages,
|
||||
conversation_messages=_conv_copy,
|
||||
incoming_message=_incoming_copy,
|
||||
budget_tokens=getattr(engine, "context_length", 0) or 0,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Context engine select_context hook failed; using unmodified "
|
||||
"request messages (session=%s)",
|
||||
session_label,
|
||||
exc_info=True,
|
||||
)
|
||||
return api_messages
|
||||
|
||||
if selected is None:
|
||||
return api_messages
|
||||
# Require a NON-EMPTY list of dicts. An empty list must fall open to the
|
||||
# original request: ``all([])`` is ``True``, so without the emptiness check
|
||||
# a ``[]`` returned by a buggy/failing engine would replace a valid request
|
||||
# with an empty message list that the downstream sanitizers cannot restore,
|
||||
# reaching the provider as an invalid request instead of failing open.
|
||||
if isinstance(selected, list) and selected and all(isinstance(m, dict) for m in selected):
|
||||
return selected
|
||||
|
||||
logger.warning(
|
||||
"Context engine select_context returned an invalid value "
|
||||
"(not a non-empty list of dicts); ignoring (session=%s)",
|
||||
session_label,
|
||||
)
|
||||
return api_messages
|
||||
|
||||
|
||||
def _notify_context_engine_turn_complete(
|
||||
agent: Any,
|
||||
messages: List[Dict[str, Any]],
|
||||
*,
|
||||
usage: Optional[Dict[str, Any]] = None,
|
||||
logger: Any,
|
||||
**meta: Any,
|
||||
) -> None:
|
||||
"""Notify the active context engine that a user turn has finished.
|
||||
|
||||
Calls the optional ``ContextEngine.on_turn_complete()`` observation hook
|
||||
once per turn, after the assistant/tool loop has produced the finalized
|
||||
transcript. The complement to ``select_context()`` (pre-request selection):
|
||||
this lets an engine ingest / index / summarize the completed turn.
|
||||
|
||||
Fail-open: a missing or no-op hook, or any exception, is swallowed.
|
||||
``messages`` is passed as a shallow copy so the engine cannot mutate the
|
||||
persisted transcript.
|
||||
"""
|
||||
engine = getattr(agent, "context_compressor", None)
|
||||
hook = getattr(engine, "on_turn_complete", None)
|
||||
if engine is None or not callable(hook):
|
||||
return
|
||||
|
||||
# Skip the no-op base implementation so non-implementing engines (incl.
|
||||
# the built-in compressor) pay nothing per turn. Lazy import avoids any
|
||||
# import cycle with agent.context_engine.
|
||||
try:
|
||||
from agent.context_engine import ContextEngine as _CE
|
||||
if getattr(hook, "__func__", None) is _CE.on_turn_complete:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
hook(
|
||||
[dict(m) if isinstance(m, dict) else m for m in messages],
|
||||
usage=usage,
|
||||
**meta,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Context engine on_turn_complete hook failed (session=%s)",
|
||||
getattr(agent, "session_id", None) or "-",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
def run_conversation(
|
||||
agent,
|
||||
user_message: Any,
|
||||
|
|
@ -805,6 +1069,13 @@ def run_conversation(
|
|||
# over instead of spinning. Reset here so each turn starts fresh. See #26080.
|
||||
agent._auth_pool_refresh_counts = {}
|
||||
|
||||
# Reset the per-turn usage holder forwarded to the context engine's
|
||||
# on_turn_complete() observation hook. Set after each successful provider
|
||||
# response (see below); left as None on turns that never reach a response
|
||||
# (early failure / interrupt) so the hook receives None rather than a
|
||||
# stale prior turn's usage.
|
||||
agent._last_turn_usage = None
|
||||
|
||||
# Optional opt-in runtime: if api_mode == codex_app_server, hand the
|
||||
# turn to the codex app-server subprocess (terminal/file ops/patching
|
||||
# all run inside Codex). Default Hermes path is bypassed entirely.
|
||||
|
|
@ -1053,7 +1324,28 @@ def run_conversation(
|
|||
# Uses new dicts so the internal messages list retains the fields
|
||||
# for Codex Responses compatibility.
|
||||
if agent._should_sanitize_tool_calls():
|
||||
agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model)
|
||||
# In MoA mode, agent.model is the virtual preset name
|
||||
# (e.g. "closed"), not the actual aggregator model. Use
|
||||
# the resolved aggregator model so Gemini aggregators
|
||||
# correctly preserve thought_signature (extra_content).
|
||||
_sanitize_model = agent.model
|
||||
if agent.provider == "moa":
|
||||
if moa_config:
|
||||
_agg = moa_config.get("aggregator") or {}
|
||||
if _agg.get("model"):
|
||||
_sanitize_model = _agg["model"]
|
||||
if _sanitize_model == agent.model:
|
||||
# Virtual-provider mode: no moa_config is threaded
|
||||
# through run_conversation — the facade resolves the
|
||||
# preset internally. Ask the facade for the resolved
|
||||
# aggregator slot from the previous create() instead
|
||||
# (set before any history replay that could carry
|
||||
# thought_signature).
|
||||
_moa_client = getattr(agent, "client", None)
|
||||
_agg_slot = getattr(_moa_client, "last_aggregator_slot", None)
|
||||
if _agg_slot and _agg_slot.get("model"):
|
||||
_sanitize_model = _agg_slot["model"]
|
||||
agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model)
|
||||
# Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context
|
||||
# The signature field helps maintain reasoning continuity
|
||||
api_messages.append(api_msg)
|
||||
|
|
@ -1070,9 +1362,9 @@ def run_conversation(
|
|||
#
|
||||
# Hermes invariant: the system prompt is built ONCE per session
|
||||
# (cached on ``_cached_system_prompt``) and replayed verbatim on
|
||||
# every turn. We send it as a single content string so the
|
||||
# bytes are byte-stable across turns and upstream prompt caches
|
||||
# stay warm.
|
||||
# every turn. ``apply_anthropic_cache_control`` may split its stable
|
||||
# prefix into content blocks on the wire, but the stored string and
|
||||
# its byte-stability remain unchanged.
|
||||
effective_system = active_system_prompt or ""
|
||||
if agent.ephemeral_system_prompt:
|
||||
effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip()
|
||||
|
|
@ -1099,7 +1391,18 @@ def run_conversation(
|
|||
aggregator=moa_config.get("aggregator") or {},
|
||||
temperature=_preset_temperature(moa_config, "reference_temperature"),
|
||||
aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"),
|
||||
max_tokens=moa_config.get("reference_max_tokens"),
|
||||
reference_max_tokens=moa_config.get("reference_max_tokens"),
|
||||
# None = no per-preset override; inherit
|
||||
# auxiliary.moa_reference.timeout via call_llm.
|
||||
reference_timeout=(
|
||||
float(moa_config["reference_timeout"])
|
||||
if moa_config.get("reference_timeout")
|
||||
else None
|
||||
),
|
||||
degraded_reference_policy=str(
|
||||
moa_config.get("degraded_reference_policy") or "loud"
|
||||
),
|
||||
agent=agent,
|
||||
)
|
||||
if _moa_context:
|
||||
for _msg in reversed(api_messages):
|
||||
|
|
@ -1126,18 +1429,25 @@ def run_conversation(
|
|||
for idx, pfm in enumerate(agent.prefill_messages):
|
||||
api_messages.insert(sys_offset + idx, pfm.copy())
|
||||
|
||||
# Apply Anthropic prompt caching for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party Anthropic-compatible
|
||||
# gateways. Auto-detected: if ``_use_prompt_caching`` is set,
|
||||
# inject cache_control breakpoints (system + last 3 messages)
|
||||
# to reduce input token costs by ~75% on multi-turn
|
||||
# conversations.
|
||||
if agent._use_prompt_caching:
|
||||
api_messages = apply_anthropic_cache_control(
|
||||
api_messages,
|
||||
cache_ttl=agent._cache_ttl,
|
||||
native_anthropic=agent._use_native_cache_layout,
|
||||
)
|
||||
# Per-turn context selection hook (additive, no-op by default).
|
||||
# Lets a context engine select/replace which context enters the
|
||||
# prompt for THIS call only — retrieval, topic routing, role/branch
|
||||
# switching — distinct from compression and independent of
|
||||
# should_compress(). Request-only: persisted history is untouched, so
|
||||
# caching/sanitization below operate on whatever the engine selected.
|
||||
# Fail-open (see _apply_context_engine_selection).
|
||||
_sel_incoming = (
|
||||
messages[current_turn_user_idx]
|
||||
if 0 <= current_turn_user_idx < len(messages)
|
||||
else None
|
||||
)
|
||||
api_messages = _apply_context_engine_selection(
|
||||
agent,
|
||||
api_messages,
|
||||
messages,
|
||||
_sel_incoming,
|
||||
logger=request_logger,
|
||||
)
|
||||
|
||||
# Safety net: strip orphaned tool results / add stubs for missing
|
||||
# results before sending to the API. Runs unconditionally — not
|
||||
|
|
@ -1197,6 +1507,39 @@ def run_conversation(
|
|||
# the OpenAI SDK. Sanitizing here prevents the 3-retry cycle.
|
||||
_sanitize_messages_surrogates(api_messages)
|
||||
|
||||
# Apply Anthropic prompt caching for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party Anthropic-compatible
|
||||
# gateways. Auto-detected: if ``_use_prompt_caching`` is set, inject
|
||||
# cache_control breakpoints for the static system prefix, full system
|
||||
# prompt, and last two messages (or the legacy system-and-3 layout
|
||||
# when no static prefix is available).
|
||||
#
|
||||
# Runs LAST, after every message mutation above. Marking earlier
|
||||
# defeats the prefix stability the mutations exist to create:
|
||||
# ``_apply_cache_marker`` rewrites ``content`` from a plain string
|
||||
# into a ``[{"type": "text", ...}]`` block, so the marked messages
|
||||
# no longer match the ``isinstance(content, str)`` test in the
|
||||
# whitespace-normalization pass and silently keep their raw
|
||||
# leading/trailing whitespace. A tool result ending in "\n" is
|
||||
# therefore sent unstripped while it sits in the last-3 window and
|
||||
# stripped once it rolls out of it — the same message, different
|
||||
# bytes on consecutive turns, which breaks the prefix match at
|
||||
# exactly the point the breakpoints were meant to protect. Marking
|
||||
# last also keeps breakpoints off messages that the orphan sweep or
|
||||
# the thinking-only drop is about to remove or merge away.
|
||||
if agent._use_prompt_caching:
|
||||
_static_system_prefix = getattr(agent, "_cached_system_prompt_static", None)
|
||||
api_messages = apply_anthropic_cache_control(
|
||||
api_messages,
|
||||
cache_ttl=agent._cache_ttl,
|
||||
native_anthropic=agent._use_native_cache_layout,
|
||||
static_system_prefix=(
|
||||
_static_system_prefix
|
||||
if isinstance(_static_system_prefix, str)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
# Build a persistent-MoA request before measuring compression pressure.
|
||||
# MoA reference output is injected into the aggregator prompt, but it
|
||||
# is deliberately ephemeral and therefore absent from ``messages``.
|
||||
|
|
@ -1354,36 +1697,52 @@ def run_conversation(
|
|||
if _pre_api_status:
|
||||
agent._emit_status(_pre_api_status)
|
||||
_last_preflight_pressure = request_pressure_tokens
|
||||
_pre_api_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages,
|
||||
system_message,
|
||||
approx_tokens=request_pressure_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
# Reset retry/empty-response state so the compacted request
|
||||
# gets a fresh chance instead of inheriting stale recovery
|
||||
# counters from the pre-compaction history.
|
||||
agent._empty_content_retries = 0
|
||||
agent._thinking_prefill_retries = 0
|
||||
agent._last_content_with_tools = None
|
||||
agent._last_content_tools_all_housekeeping = False
|
||||
agent._mute_post_response = False
|
||||
# Re-baseline the flush cursor for the compaction mode that just
|
||||
# ran. Legacy session-rotation returns None (the child session has
|
||||
# not seen the compacted transcript, so the next flush writes it
|
||||
# whole); in-place compaction returns list(messages) because the
|
||||
# compacted rows are already persisted under the same session id —
|
||||
# leaving None there would re-append them, doubling the active
|
||||
# context and retriggering compression. Mirrors the post-response
|
||||
# and preflight compaction sites; see
|
||||
# conversation_history_after_compression().
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
api_call_count -= 1
|
||||
agent._api_call_count = api_call_count
|
||||
agent.iteration_budget.refund()
|
||||
continue
|
||||
if messages is _pre_api_input and compression_skipped_due_to_lock(agent):
|
||||
# #69870 lock-skip: another path holds this session's
|
||||
# compression lock, so this pass no-oped. That is a temporary
|
||||
# DEFER, not evidence about compressibility — refund the
|
||||
# attempt (it must not burn the shared overflow-recovery
|
||||
# budget toward compression_exhausted → gateway auto-reset,
|
||||
# #9893/#35809) and leave the insufficient-progress blocker
|
||||
# unarmed. Proceed with the current request: if it truly does
|
||||
# not fit, the provider's 413/overflow handler returns the
|
||||
# soft compression_deferred result with that stronger signal.
|
||||
compression_attempts -= 1
|
||||
_last_preflight_pressure = None
|
||||
if pending_moa_prepared_request is _moa_prepared_request:
|
||||
pending_moa_prepared_request = None
|
||||
else:
|
||||
# Reset retry/empty-response state so the compacted request
|
||||
# gets a fresh chance instead of inheriting stale recovery
|
||||
# counters from the pre-compaction history.
|
||||
agent._empty_content_retries = 0
|
||||
agent._thinking_prefill_retries = 0
|
||||
agent._last_content_with_tools = None
|
||||
agent._last_content_tools_all_housekeeping = False
|
||||
agent._mute_post_response = False
|
||||
# Re-baseline the flush cursor for the compaction mode that just
|
||||
# ran. Legacy session-rotation returns None (the child session has
|
||||
# not seen the compacted transcript, so the next flush writes it
|
||||
# whole); in-place compaction returns list(messages) because the
|
||||
# compacted rows are already persisted under the same session id —
|
||||
# leaving None there would re-append them, doubling the active
|
||||
# context and retriggering compression. Mirrors the post-response
|
||||
# and preflight compaction sites; see
|
||||
# conversation_history_after_compression().
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
api_call_count -= 1
|
||||
agent._api_call_count = api_call_count
|
||||
agent.iteration_budget.refund()
|
||||
continue
|
||||
elif (
|
||||
agent.compression_enabled
|
||||
and len(messages) > 1
|
||||
|
|
@ -2021,6 +2380,8 @@ def run_conversation(
|
|||
)
|
||||
continue # Retry the API call
|
||||
|
||||
agent._turn_received_provider_response = True
|
||||
|
||||
# Check finish_reason before proceeding
|
||||
if agent.api_mode == "codex_responses":
|
||||
status = getattr(response, "status", None)
|
||||
|
|
@ -2542,6 +2903,14 @@ def run_conversation(
|
|||
"reasoning_tokens": canonical_usage.reasoning_tokens,
|
||||
}
|
||||
agent.context_compressor.update_from_response(usage_dict)
|
||||
|
||||
# Stash this response's canonical usage so the post-turn
|
||||
# on_turn_complete() observation hook can forward it (the
|
||||
# same dict shape passed to update_from_response). A turn
|
||||
# may make several API calls; the engine's per-turn signal
|
||||
# of interest is the cost/size of the latest assembled
|
||||
# request, so we keep the most recent call's usage.
|
||||
agent._last_turn_usage = dict(usage_dict)
|
||||
elif getattr(
|
||||
agent.context_compressor,
|
||||
"awaiting_real_usage_after_compression",
|
||||
|
|
@ -3869,10 +4238,23 @@ def run_conversation(
|
|||
|
||||
original_len = len(messages)
|
||||
original_tokens = estimate_messages_tokens_rough(messages)
|
||||
_overflow_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message, approx_tokens=approx_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
if messages is _overflow_input and compression_skipped_due_to_lock(agent):
|
||||
# #69870 lock-skip: the provider proved the request
|
||||
# does not fit, but this compression pass no-oped only
|
||||
# because another path holds the session's compression
|
||||
# lock. Temporary defer, not exhaustion — refund the
|
||||
# attempt and end the turn softly so the gateway does
|
||||
# NOT auto-reset the session (#9893/#35809).
|
||||
compression_attempts -= 1
|
||||
agent._persist_session(messages, conversation_history)
|
||||
return _compression_deferred_result(
|
||||
agent, messages, api_call_count
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
|
|
@ -4110,10 +4492,23 @@ def run_conversation(
|
|||
|
||||
original_len = len(messages)
|
||||
original_tokens = estimate_messages_tokens_rough(messages)
|
||||
_overflow_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message, approx_tokens=approx_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
if messages is _overflow_input and compression_skipped_due_to_lock(agent):
|
||||
# #69870 lock-skip: the provider proved the request
|
||||
# does not fit, but this compression pass no-oped only
|
||||
# because another path holds the session's compression
|
||||
# lock. Temporary defer, not exhaustion — refund the
|
||||
# attempt and end the turn softly so the gateway does
|
||||
# NOT auto-reset the session (#9893/#35809).
|
||||
compression_attempts -= 1
|
||||
agent._persist_session(messages, conversation_history)
|
||||
return _compression_deferred_result(
|
||||
agent, messages, api_call_count
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
|
|
@ -5354,6 +5749,10 @@ def run_conversation(
|
|||
# flag so it can fire again if the model goes empty on
|
||||
# a LATER tool round.
|
||||
agent._post_tool_empty_retried = False
|
||||
# A landed tool call means any earlier dropped-tool-call stall
|
||||
# was recovered — refresh that budget too so it guards each
|
||||
# stall independently rather than capping the whole run.
|
||||
agent._dropped_toolcall_retries = 0
|
||||
|
||||
previous_msg = messages[-1] if messages else None
|
||||
current_interim_visible = agent._interim_assistant_visible_text(assistant_msg)
|
||||
|
|
@ -5516,14 +5915,27 @@ def run_conversation(
|
|||
if callable(_clear_warn):
|
||||
_clear_warn()
|
||||
agent._safe_print(" ⟳ compacting context…")
|
||||
_post_tool_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message,
|
||||
approx_tokens=agent.context_compressor.last_prompt_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
if (
|
||||
messages is _post_tool_input
|
||||
and compression_skipped_due_to_lock(agent)
|
||||
):
|
||||
# #69870 lock-skip: this pass no-oped because another
|
||||
# path holds the session's compression lock — a
|
||||
# temporary defer, not evidence about compressibility.
|
||||
# Refund the attempt so a lock-loser tool loop does not
|
||||
# burn the shared per-turn budget toward
|
||||
# compression_exhausted (#9893/#35809).
|
||||
compression_attempts -= 1
|
||||
else:
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
elif agent.compression_enabled:
|
||||
# Over threshold but compression is blocked (summary-LLM
|
||||
# cooldown or anti-thrashing). Surface a deduped warning so
|
||||
|
|
@ -5543,15 +5955,72 @@ def run_conversation(
|
|||
_real_tokens,
|
||||
int(getattr(_compressor, "threshold_tokens", 0) or 0),
|
||||
)
|
||||
# Proactive tool-result prune: reclaim re-sent history on
|
||||
# large-window models long before should_compress() (≈50% of
|
||||
# the window) would ever fire. Deterministic, no LLM call;
|
||||
# protects the recent tail. No-op unless proactive_prune_tokens
|
||||
# is configured and _real_tokens is above it — and even then
|
||||
# the prune only commits when it reclaims at least
|
||||
# proactive_prune_min_reclaim_tokens, so prompt-cache breaks
|
||||
# stay episodic like compression's (the one sanctioned cache
|
||||
# break) instead of firing every tool iteration. See
|
||||
# ContextCompressor.prune_tool_results_only.
|
||||
# getattr guard: plugin context engines predating the hook and
|
||||
# minimal test doubles (SimpleNamespace compressors) lack the
|
||||
# method — treat absence as a no-op.
|
||||
_prune = getattr(_compressor, "prune_tool_results_only", None)
|
||||
if callable(_prune):
|
||||
try:
|
||||
_pruned_msgs, _pruned_n = _prune(
|
||||
messages, current_tokens=_real_tokens
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"proactive tool-result prune failed; skipping",
|
||||
exc_info=True,
|
||||
)
|
||||
_pruned_msgs, _pruned_n = messages, 0
|
||||
# Standard no-op caller contract: only commit when the
|
||||
# engine returned a NEW list object with a non-zero count.
|
||||
if _pruned_n and _pruned_msgs is not messages:
|
||||
# Do NOT rebuild conversation_history here. Unlike the
|
||||
# compression branch, the prune neither rotates the session
|
||||
# nor calls archive_and_compact(), so there is no new
|
||||
# persistence baseline to establish. _prune_old_tool_results
|
||||
# returns per-message copies that preserve the
|
||||
# _DB_PERSISTED_MARKER, so the marker-based flush dedup (see
|
||||
# _flush_messages_to_session_db) already prevents both
|
||||
# duplicate writes and dropped rows. Calling
|
||||
# conversation_history_after_compression (a compaction-only
|
||||
# helper keyed on the _last_compaction_in_place flag) would be
|
||||
# a no-op at best, and on a stale in-place flag could seed
|
||||
# this turn's fresh, not-yet-persisted rows into history_ids
|
||||
# and skip writing them.
|
||||
messages = _pruned_msgs
|
||||
|
||||
# Save session log incrementally (so progress is visible even if interrupted)
|
||||
agent._session_messages = messages
|
||||
|
||||
# Touch activity before continuing so the gateway's
|
||||
# inactivity monitor never sees a stale timestamp
|
||||
# between tool completion and the start of the next
|
||||
# API call. Without this, a tool-call result (which
|
||||
# takes ~0s to process) followed by slow post-tool
|
||||
# processing (compression, persist) and a slow
|
||||
# follow-up API call can exceed the gateway inactivity
|
||||
# timeout (HERMES_AGENT_TIMEOUT, default 1800s) and the
|
||||
# gateway kills the session before the next activity
|
||||
# touch fires (#69559, #69131).
|
||||
agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}")
|
||||
# Continue loop for next response
|
||||
continue
|
||||
|
||||
else:
|
||||
# No tool calls - this is the final response
|
||||
# No tool calls - this is the final response.
|
||||
# (Dropped tool-call recovery — finish_reason=="tool_calls" with
|
||||
# an empty tool_calls array — is handled at the finalization
|
||||
# chokepoint below, after final_msg is built, so it catches
|
||||
# every path that reaches turn finalization, not just this one.)
|
||||
final_response = assistant_message.content or ""
|
||||
|
||||
# Fix: unmute output when entering the no-tool-call branch
|
||||
|
|
@ -5883,6 +6352,64 @@ def run_conversation(
|
|||
|
||||
final_msg = agent._build_assistant_message(assistant_message, finish_reason)
|
||||
|
||||
# ── Dropped tool-call recovery (copilot/Claude) ────────
|
||||
# Some providers (observed: claude-opus-4.8 / claude-sonnet-4.5
|
||||
# on GitHub Copilot, ~2026-07) return finish_reason="tool_calls"
|
||||
# while the parsed tool_calls array is empty — the model
|
||||
# signalled it wanted to act but the payload shipped no call.
|
||||
# Reaching finalization with that mismatch means the turn is
|
||||
# about to end with the task unstarted (the narration, which may
|
||||
# be in content or only in the reasoning field, gets treated as
|
||||
# the final answer). Re-prompt (bounded to 3 CONSECUTIVE stalls;
|
||||
# the budget resets after any successful tool round) to make the
|
||||
# model emit the call instead of exiting. finish_reason="stop"
|
||||
# text finishes never enter this guard.
|
||||
if (
|
||||
finish_reason == "tool_calls"
|
||||
and not assistant_message.tool_calls
|
||||
and getattr(agent, "_dropped_toolcall_retries", 0) < 3
|
||||
):
|
||||
agent._dropped_toolcall_retries = getattr(agent, "_dropped_toolcall_retries", 0) + 1
|
||||
logger.warning(
|
||||
"finish_reason=tool_calls with empty tool_calls array "
|
||||
"(narration only) — re-prompting to emit the call "
|
||||
"(retry %d/3, model=%s provider=%s)",
|
||||
agent._dropped_toolcall_retries, agent.model, agent.provider,
|
||||
)
|
||||
agent._emit_status(
|
||||
"↻ Model signaled a tool call but sent none — "
|
||||
f"re-prompting ({agent._dropped_toolcall_retries}/3)"
|
||||
)
|
||||
# Both halves of the re-prompt pair are ephemeral recovery
|
||||
# scaffolding (mirrors the empty-response nudge pattern):
|
||||
# the interim narration-only assistant turn exists solely to
|
||||
# keep role alternation valid for the nudge, and the nudge
|
||||
# exists solely to drive the retry. Flag both so the
|
||||
# persistence layer never writes them to the durable
|
||||
# transcript and the finalization pop below can strip an
|
||||
# unanswered tail pair. A recovered (answered) pair stays
|
||||
# buried mid-list in live memory but is skipped by the
|
||||
# flush regardless of position.
|
||||
final_msg["_dropped_toolcall_nudge"] = True
|
||||
messages.append(final_msg)
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Your previous turn indicated a tool call but none was "
|
||||
"included. Do not narrate a plan or restate intent — issue "
|
||||
"the actual tool call now to continue the task."
|
||||
),
|
||||
"_dropped_toolcall_nudge": True,
|
||||
})
|
||||
agent._session_messages = messages
|
||||
final_response = None
|
||||
continue
|
||||
|
||||
# Reached finalization without the dropped-tool-call mismatch —
|
||||
# a genuine turn end. Clear the consecutive-stall budget so the
|
||||
# next turn starts fresh.
|
||||
agent._dropped_toolcall_retries = 0
|
||||
|
||||
# Pop thinking-only prefill and empty-response retry
|
||||
# scaffolding before appending either a final response or a
|
||||
# verification-stop follow-up. These internal turns are only
|
||||
|
|
@ -5895,6 +6422,7 @@ def run_conversation(
|
|||
messages[-1].get("_thinking_prefill")
|
||||
or messages[-1].get("_empty_recovery_synthetic")
|
||||
or messages[-1].get("_empty_terminal_sentinel")
|
||||
or messages[-1].get("_dropped_toolcall_nudge")
|
||||
)
|
||||
):
|
||||
messages.pop()
|
||||
|
|
|
|||
|
|
@ -512,7 +512,7 @@ class CopilotACPClient:
|
|||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
bufsize=1,
|
||||
cwd=self._acp_cwd,
|
||||
env=_build_subprocess_env(),
|
||||
|
|
@ -708,7 +708,7 @@ class CopilotACPClient:
|
|||
if block_error:
|
||||
raise PermissionError(block_error)
|
||||
try:
|
||||
content = path.read_text()
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
content = ""
|
||||
line = params.get("line")
|
||||
|
|
@ -736,7 +736,7 @@ class CopilotACPClient:
|
|||
if denied:
|
||||
raise PermissionError(denied)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(str(params.get("content") or ""))
|
||||
path.write_text(str(params.get("content") or ""), encoding="utf-8")
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message_id,
|
||||
|
|
|
|||
|
|
@ -594,6 +594,14 @@ class CredentialPool:
|
|||
# Re-armed to None on every successful selection so a recover→re-exhaust
|
||||
# transition logs promptly instead of being swallowed by a stale window.
|
||||
self._last_no_entries_log_at: Optional[float] = None
|
||||
# #70401: consecutive mark_exhausted_and_rotate() calls whose supplied
|
||||
# credential identity matched no pool entry (OAuth wrappers whose
|
||||
# runtime key rotates, entries pruned by another process, ...). These
|
||||
# rotations mark nothing exhausted, so without a cap the pool can
|
||||
# never converge to "no available entries" and the caller's 401 retry
|
||||
# loop runs unbounded and non-interruptible. Reset whenever a real
|
||||
# entry is identified or an escape path returns None.
|
||||
self._unmatched_rotation_streak: int = 0
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
with self._lock:
|
||||
|
|
@ -622,6 +630,28 @@ class CredentialPool:
|
|||
with self._lock:
|
||||
return self._current_unlocked()
|
||||
|
||||
def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]:
|
||||
"""Return the stable id for the runtime credential in use.
|
||||
|
||||
Prefer the current selection when it still supplies ``api_key_hint``.
|
||||
If the cursor was cleared, fall back to an unambiguous key match.
|
||||
"""
|
||||
with self._lock:
|
||||
current = self._current_unlocked()
|
||||
if current is not None and (
|
||||
api_key_hint is None
|
||||
or current.runtime_api_key == api_key_hint
|
||||
):
|
||||
return current.id
|
||||
if api_key_hint is None:
|
||||
return None
|
||||
matches = [
|
||||
entry
|
||||
for entry in self._entries
|
||||
if entry.runtime_api_key == api_key_hint
|
||||
]
|
||||
return matches[0].id if len(matches) == 1 else None
|
||||
|
||||
def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
|
||||
"""Swap an entry in-place by id, preserving sort order."""
|
||||
for idx, entry in enumerate(self._entries):
|
||||
|
|
@ -1562,7 +1592,13 @@ class CredentialPool:
|
|||
|
||||
def select(self) -> Optional[PooledCredential]:
|
||||
with self._lock:
|
||||
return self._select_unlocked()
|
||||
entry = self._select_unlocked()
|
||||
if entry is not None:
|
||||
# A normal (non-recovery) selection starts a fresh episode —
|
||||
# don't let a leftover unmatched-rotation streak from an old
|
||||
# failure trip the #70401 bound early next time.
|
||||
self._unmatched_rotation_streak = 0
|
||||
return entry
|
||||
|
||||
def _available_entries(self, *, clear_expired: bool = False, refresh: bool = False) -> List[PooledCredential]:
|
||||
"""Return entries not currently in exhaustion cooldown.
|
||||
|
|
@ -1763,10 +1799,17 @@ class CredentialPool:
|
|||
status_code: Optional[int],
|
||||
error_context: Optional[Dict[str, Any]] = None,
|
||||
api_key_hint: Optional[str] = None,
|
||||
credential_id: Optional[str] = None,
|
||||
) -> Optional[PooledCredential]:
|
||||
with self._lock:
|
||||
entry = None
|
||||
if api_key_hint:
|
||||
identity_supplied = bool(credential_id or api_key_hint)
|
||||
if credential_id:
|
||||
entry = next(
|
||||
(e for e in self._entries if e.id == credential_id),
|
||||
None,
|
||||
)
|
||||
if entry is None and api_key_hint:
|
||||
# Prefer the specific entry whose API key matches the one that
|
||||
# actually failed. When this pool was freshly loaded from disk
|
||||
# (another process already rotated), current() is None and
|
||||
|
|
@ -1775,20 +1818,59 @@ class CredentialPool:
|
|||
(e for e in self._entries if e.runtime_api_key == api_key_hint),
|
||||
None,
|
||||
)
|
||||
if entry is None:
|
||||
# The failed key is identifiable but matches no entry
|
||||
# (rotated away, or a wrapper whose runtime key differs).
|
||||
# Falling through to current()/_select_unlocked() would
|
||||
# mark an INNOCENT healthy key exhausted for the full
|
||||
# cooldown TTL. Don't guess — just hand back a fresh
|
||||
# selection so the caller can retry.
|
||||
logger.info(
|
||||
"credential pool: failed key hint matched no %s entry; "
|
||||
"rotating without marking any credential exhausted",
|
||||
if entry is None and identity_supplied:
|
||||
# The failed credential is identifiable but matches no entry
|
||||
# (rotated away, or a wrapper whose runtime key differs).
|
||||
# Falling through to current()/_select_unlocked() would mark an
|
||||
# innocent healthy key exhausted for the full cooldown TTL.
|
||||
#
|
||||
# #70401: this branch must still be BOUNDED. With OAuth-token
|
||||
# auth the upstream 401's key hint never matches any entry's
|
||||
# ``runtime_api_key``, so every retry lands here, nothing is
|
||||
# ever marked exhausted, and the pool can never reach the
|
||||
# "no available entries" state — the caller retries the same
|
||||
# dead token forever (~6/sec, starving the event loop so chat
|
||||
# interrupts are never processed). The single-entry case
|
||||
# below already escapes; multi-entry pools could still
|
||||
# ping-pong A→B→A indefinitely without marking anything.
|
||||
# Cap consecutive no-mark rotations at one full lap of the
|
||||
# available entries: past that, every candidate has been
|
||||
# handed back at least once without recovery, so stop
|
||||
# guessing and surface the error (no cooldown is written for
|
||||
# anybody — healthy keys stay available for the next turn).
|
||||
self._unmatched_rotation_streak += 1
|
||||
available_count = len(self._available_entries())
|
||||
if self._unmatched_rotation_streak > max(available_count, 1):
|
||||
logger.warning(
|
||||
"credential pool: failed credential identity matched no "
|
||||
"%s entry for %d consecutive rotations (pool size %d) — "
|
||||
"surfacing the error instead of rotating again",
|
||||
self.provider,
|
||||
self._unmatched_rotation_streak,
|
||||
available_count,
|
||||
)
|
||||
self._unmatched_rotation_streak = 0
|
||||
self._current_id = None
|
||||
return self._select_unlocked()
|
||||
return None
|
||||
logger.info(
|
||||
"credential pool: failed credential identity matched no %s "
|
||||
"entry; rotating without marking any credential exhausted",
|
||||
self.provider,
|
||||
)
|
||||
self._current_id = None
|
||||
next_entry = self._select_unlocked()
|
||||
if next_entry is not None and len(self._available_entries()) == 1:
|
||||
# A single-entry pool cannot rotate. Returning its only
|
||||
# entry reports a successful recovery without changing
|
||||
# the credential, so the caller retries the same 401
|
||||
# indefinitely. Let fallback/error propagation proceed.
|
||||
self._unmatched_rotation_streak = 0
|
||||
self._current_id = None
|
||||
return None
|
||||
return next_entry
|
||||
# A real entry was identified — any prior unmatched-rotation
|
||||
# streak is stale (this mark WILL advance pool state).
|
||||
self._unmatched_rotation_streak = 0
|
||||
if entry is None:
|
||||
entry = self._current_unlocked() or self._select_unlocked()
|
||||
if entry is None:
|
||||
|
|
@ -1806,12 +1888,13 @@ class CredentialPool:
|
|||
# disconnects (a ~2.5min hang with no error surfaced to the user).
|
||||
# Mark every entry sharing the failed key so the pool can reach the
|
||||
# "no available entries" state and let the error propagate.
|
||||
if api_key_hint:
|
||||
failed_runtime_key = getattr(entry, "runtime_api_key", None)
|
||||
if identity_supplied and failed_runtime_key:
|
||||
siblings_marked = False
|
||||
for sibling in self._entries:
|
||||
if sibling.id == entry.id:
|
||||
continue
|
||||
if sibling.runtime_api_key == api_key_hint:
|
||||
if sibling.runtime_api_key == failed_runtime_key:
|
||||
self._mark_exhausted(
|
||||
sibling, status_code, error_context, persist=False
|
||||
)
|
||||
|
|
@ -1885,9 +1968,11 @@ class CredentialPool:
|
|||
return self._try_refresh_current_unlocked()
|
||||
|
||||
def try_refresh_matching(
|
||||
self, api_key_hint: Optional[str] = None
|
||||
self,
|
||||
api_key_hint: Optional[str] = None,
|
||||
credential_id: Optional[str] = None,
|
||||
) -> Optional[PooledCredential]:
|
||||
"""Force-refresh the entry that supplied ``api_key_hint``.
|
||||
"""Force-refresh the entry that supplied the failed request.
|
||||
|
||||
Direct provider integrations may reload the pool after a request has
|
||||
already failed, so they cannot rely on ``current_id`` identifying the
|
||||
|
|
@ -1897,17 +1982,29 @@ class CredentialPool:
|
|||
"""
|
||||
with self._lock:
|
||||
entry = None
|
||||
if api_key_hint:
|
||||
if credential_id:
|
||||
entry = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in self._entries
|
||||
if candidate.runtime_api_key == api_key_hint
|
||||
if candidate.id == credential_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
entry = self._current_unlocked() or self._select_unlocked(refresh=False)
|
||||
if entry is None:
|
||||
if api_key_hint:
|
||||
entry = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in self._entries
|
||||
if candidate.runtime_api_key == api_key_hint
|
||||
),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
entry = self._current_unlocked() or self._select_unlocked(
|
||||
refresh=False
|
||||
)
|
||||
if entry is None:
|
||||
return None
|
||||
self._current_id = entry.id
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ def _remove_env_source(provider: str, removed) -> RemovalResult:
|
|||
if env_path.exists():
|
||||
env_in_dotenv = any(
|
||||
line.strip().startswith(f"{env_var}=")
|
||||
for line in env_path.read_text(errors="replace").splitlines()
|
||||
for line in env_path.read_text(errors="replace", encoding="utf-8").splitlines()
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ def apply_automatic_transitions(now: Optional[datetime] = None) -> Dict[str, int
|
|||
|
||||
counts = {"marked_stale": 0, "archived": 0, "reactivated": 0, "checked": 0, "seeded": 0}
|
||||
|
||||
for row in _u.agent_created_report():
|
||||
for row in _u.curated_report():
|
||||
counts["checked"] += 1
|
||||
name = row["name"]
|
||||
if row.get("pinned"):
|
||||
|
|
@ -422,7 +422,9 @@ CURATOR_REVIEW_PROMPT = (
|
|||
"INSTRUCTIONS AND EXPERIENTIAL KNOWLEDGE. A collection of hundreds of "
|
||||
"narrow skills where each one captures one session's specific bug is "
|
||||
"a FAILURE of the library — not a feature. An agent searching skills "
|
||||
"matches on descriptions, not on exact names; one broad umbrella "
|
||||
"matches on descriptions, not on exact names (note: long descriptions "
|
||||
"are truncated to 57 chars in the system prompt skill index — keep the "
|
||||
"trigger class in that window). One broad umbrella "
|
||||
"skill with labeled subsections beats five narrow siblings for "
|
||||
"discoverability, not the other way around.\n\n"
|
||||
"The right target shape is CLASS-LEVEL skills with rich SKILL.md "
|
||||
|
|
@ -1470,15 +1472,16 @@ def _render_report_markdown(p: Dict[str, Any]) -> str:
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _render_candidate_list() -> str:
|
||||
"""Human/agent-readable list of agent-created skills with usage stats."""
|
||||
rows = skill_usage.agent_created_report()
|
||||
"""Human/agent-readable list of curator-managed skills with usage stats."""
|
||||
rows = skill_usage.curated_report()
|
||||
if not rows:
|
||||
return "No agent-created skills to review."
|
||||
return "No curator-managed skills to review."
|
||||
cron_referenced = _cron_referenced_skills()
|
||||
lines = [f"Agent-created skills ({len(rows)}):\n"]
|
||||
lines = [f"Curator-managed skills ({len(rows)}):\n"]
|
||||
for r in rows:
|
||||
lines.append(
|
||||
f"- {r['name']} "
|
||||
f"provenance={r.get('provenance', 'agent')} "
|
||||
f"state={r['state']} "
|
||||
f"pinned={'yes' if r.get('pinned') else 'no'} "
|
||||
f"cron={'yes' if r['name'] in cron_referenced else 'no'} "
|
||||
|
|
@ -1531,7 +1534,7 @@ def run_curator_review(
|
|||
if dry_run:
|
||||
# Count candidates without mutating state.
|
||||
try:
|
||||
report = skill_usage.agent_created_report()
|
||||
report = skill_usage.curated_report()
|
||||
counts = {
|
||||
"checked": len(report),
|
||||
"marked_stale": 0,
|
||||
|
|
@ -1584,7 +1587,7 @@ def run_curator_review(
|
|||
nonlocal auto_summary
|
||||
# Snapshot skill state BEFORE the LLM pass so the report can diff.
|
||||
try:
|
||||
before_report = skill_usage.agent_created_report()
|
||||
before_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
before_report = []
|
||||
before_names = {r.get("name") for r in before_report if isinstance(r, dict)}
|
||||
|
|
@ -1610,7 +1613,7 @@ def run_curator_review(
|
|||
state2["last_run_duration_seconds"] = elapsed
|
||||
state2["last_run_summary"] = final_summary
|
||||
try:
|
||||
after_report = skill_usage.agent_created_report()
|
||||
after_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
after_report = []
|
||||
try:
|
||||
|
|
@ -1697,7 +1700,7 @@ def run_curator_review(
|
|||
try:
|
||||
rename_lines = _build_rename_summary(
|
||||
before_names=before_names,
|
||||
after_report=skill_usage.agent_created_report(),
|
||||
after_report=skill_usage.curated_report(),
|
||||
tool_calls=llm_meta.get("tool_calls", []) or [],
|
||||
model_final=llm_meta.get("final", "") or "",
|
||||
)
|
||||
|
|
@ -1715,7 +1718,7 @@ def run_curator_review(
|
|||
# reporting bug never breaks the curator itself. Report path is
|
||||
# recorded in state so `hermes curator status` can point at it.
|
||||
try:
|
||||
after_report = skill_usage.agent_created_report()
|
||||
after_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
after_report = []
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -270,8 +270,12 @@ def _translate_tool_call_to_gemini(tool_call: Dict[str, Any]) -> Dict[str, Any]:
|
|||
}
|
||||
}
|
||||
thought_signature = _tool_call_extra_signature(tool_call)
|
||||
if thought_signature:
|
||||
part["thoughtSignature"] = thought_signature
|
||||
# Fallback sentinel for cross-provider tool_calls (e.g. fallback from
|
||||
# xAI/Anthropic to Gemini, where the original tool_call carries no
|
||||
# Gemini thoughtSignature). Mirrors gemini_cloudcode_adapter.py:106.
|
||||
# Without this, Gemini 3 thinking models reject replayed history with
|
||||
# 400 INVALID_ARGUMENT on the missing thoughtSignature.
|
||||
part["thoughtSignature"] = thought_signature or "skip_thought_signature_validator"
|
||||
return part
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ Language resolution order:
|
|||
3. ``display.language`` from config.yaml
|
||||
4. ``"en"`` (baseline)
|
||||
|
||||
Supported languages: en, zh, ja, de, es, fr, tr, uk. Unknown values fall back to en.
|
||||
Supported languages: en, zh, zh-hant, ja, de, es, fr, tr, uk, af, ko, it, ga,
|
||||
pt, ru, hu, ar. Unknown values fall back to en.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -41,7 +42,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
SUPPORTED_LANGUAGES: tuple[str, ...] = (
|
||||
"en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk",
|
||||
"af", "ko", "it", "ga", "pt", "ru", "hu",
|
||||
"af", "ko", "it", "ga", "pt", "ru", "hu", "ar",
|
||||
)
|
||||
DEFAULT_LANGUAGE = "en"
|
||||
|
||||
|
|
@ -78,6 +79,9 @@ _LANGUAGE_ALIASES: dict[str, str] = {
|
|||
"russian": "ru", "русский": "ru", "ru-ru": "ru",
|
||||
# Hungarian
|
||||
"hungarian": "hu", "magyar": "hu", "hu-hu": "hu",
|
||||
# Arabic — bare "arabic"/endonym plus the common regional BCP-47 tags.
|
||||
"arabic": "ar", "العربية": "ar",
|
||||
"ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar",
|
||||
}
|
||||
|
||||
_catalog_cache: dict[str, dict[str, str]] = {}
|
||||
|
|
|
|||
|
|
@ -56,6 +56,8 @@ from pathlib import Path
|
|||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
|
||||
from agent.lsp.protocol import (
|
||||
ERROR_CONTENT_MODIFIED,
|
||||
ERROR_METHOD_NOT_FOUND,
|
||||
|
|
@ -294,6 +296,12 @@ class LSPClient:
|
|||
cmd = self._command
|
||||
if sys.platform == "win32":
|
||||
cmd = self._win_wrap_cmd(cmd)
|
||||
# Suppress the cmd.exe console window that would otherwise flash
|
||||
# every time we launch a ``.cmd``-wrapped language server
|
||||
# (e.g. pyright-langserver.CMD) from a console-less host such as
|
||||
# a VS Code/Zed extension running the ACP adapter.
|
||||
# windows_hide_flags() is CREATE_NO_WINDOW on Windows, 0 on POSIX.
|
||||
creationflags = windows_hide_flags()
|
||||
|
||||
try:
|
||||
# start_new_session=True detaches the LSP server into its own
|
||||
|
|
@ -312,6 +320,7 @@ class LSPClient:
|
|||
env=env,
|
||||
cwd=self._cwd,
|
||||
start_new_session=True,
|
||||
creationflags=creationflags,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
raise LSPProtocolError(
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ import threading
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
|
||||
logger = logging.getLogger("agent.lsp.install")
|
||||
|
||||
# Package-name → install-strategy hint registry. Each entry is a
|
||||
|
|
@ -265,9 +267,10 @@ def _install_npm(
|
|||
[npm, "install", "--prefix", str(staging), "--silent", "--no-fund", "--no-audit", *install_targets],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=300,
|
||||
stdin=subprocess.DEVNULL,
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
logger.warning(
|
||||
|
|
@ -313,10 +316,11 @@ def _install_go(pkg: str, bin_name: str) -> Optional[str]:
|
|||
[go, "install", pkg],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=600,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
logger.warning(
|
||||
|
|
|
|||
|
|
@ -80,8 +80,17 @@ def normalize_tool_schema(schema: Any) -> Optional[Dict[str, Any]]:
|
|||
return schema
|
||||
|
||||
|
||||
def memory_provider_tools_enabled(enabled_toolsets: Optional[List[str]]) -> bool:
|
||||
def memory_provider_tools_enabled(
|
||||
enabled_toolsets: Optional[List[str]],
|
||||
disabled_toolsets: Optional[List[str]] = None,
|
||||
*,
|
||||
memory_tool_present: bool = False,
|
||||
) -> bool:
|
||||
"""Return whether external memory-provider tools should be exposed."""
|
||||
if disabled_toolsets and "memory" in disabled_toolsets:
|
||||
return False
|
||||
if memory_tool_present:
|
||||
return True
|
||||
if enabled_toolsets is None:
|
||||
return True
|
||||
if not enabled_toolsets:
|
||||
|
|
@ -110,9 +119,10 @@ def inject_memory_provider_tools(agent: Any) -> int:
|
|||
for tool in tools
|
||||
if isinstance(tool, dict)
|
||||
}
|
||||
if (
|
||||
"memory" not in existing_tool_names
|
||||
and not memory_provider_tools_enabled(getattr(agent, "enabled_toolsets", None))
|
||||
if not memory_provider_tools_enabled(
|
||||
getattr(agent, "enabled_toolsets", None),
|
||||
getattr(agent, "disabled_toolsets", None),
|
||||
memory_tool_present="memory" in existing_tool_names,
|
||||
):
|
||||
return 0
|
||||
|
||||
|
|
|
|||
1031
agent/moa_loop.py
1031
agent/moa_loop.py
File diff suppressed because it is too large
Load diff
|
|
@ -215,6 +215,7 @@ DEFAULT_CONTEXT_LENGTHS = {
|
|||
# OpenRouter-prefixed models resolve via OpenRouter live API or models.dev.
|
||||
"claude-fable-5": 1000000,
|
||||
"claude-fable": 1000000,
|
||||
"claude-opus-5": 1000000,
|
||||
"claude-sonnet-5": 1000000,
|
||||
"claude-opus-4-8": 1000000,
|
||||
"claude-opus-4.8": 1000000,
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ def record_nous_rate_limit(
|
|||
# Atomic write: write to temp file + rename
|
||||
fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "w") as f:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f)
|
||||
atomic_replace(tmp_path, path)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -192,7 +192,13 @@ SKILLS_GUIDANCE = (
|
|||
"skill with skill_manage so you can reuse it next time.\n"
|
||||
"When using a skill and finding it outdated, incomplete, or wrong, "
|
||||
"patch it immediately with skill_manage(action='patch') — don't wait to be asked. "
|
||||
"Skills that aren't maintained become liabilities."
|
||||
"Skills that aren't maintained become liabilities.\n"
|
||||
"\n"
|
||||
"## Skill Safety Rule\n"
|
||||
"1. **UNAVAILABLE** — If a skill placeholder contains `[SKILL_PRUNED]`, the skill content was lost in compression and is inaccessible.\n"
|
||||
"2. **RELOAD** — Before performing any action that depends on a skill, re-check its content with `skill_view(name='...')` if it shows `[SKILL_PRUNED]`.\n"
|
||||
"3. **WAIT** — If a skill is loading or was just pruned, wait for the reload confirmation before proceeding.\n"
|
||||
"4. **DEDUP** — After reloading a pruned skill, **ignore any remaining `[SKILL_PRUNED]` markers for that same skill** — they are historical artifacts from previous compactions and do not need further action."
|
||||
)
|
||||
|
||||
KANBAN_GUIDANCE = (
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
"""Anthropic prompt caching strategy.
|
||||
|
||||
Single layout: ``system_and_3``. 4 cache_control breakpoints — system
|
||||
prompt + last 3 non-system messages, all at the same TTL (5m or 1h).
|
||||
Reduces input token costs by ~75% on multi-turn conversations within a
|
||||
single session.
|
||||
The default layout uses 4 cache_control breakpoints: the static system
|
||||
prefix, the end of the system prompt, and the last 2 non-system messages.
|
||||
When a static system prefix is unavailable, it falls back to one system
|
||||
breakpoint plus the last 3 messages. All markers use the same TTL (5m or 1h).
|
||||
This preserves intra-session caching while allowing new sessions to reuse the
|
||||
stable system-prompt prefix.
|
||||
|
||||
Pure functions -- no class state, no AIAgent dependency.
|
||||
"""
|
||||
|
|
@ -81,15 +83,55 @@ def _build_marker(ttl: str) -> Dict[str, str]:
|
|||
return marker
|
||||
|
||||
|
||||
def _apply_system_cache_markers(
|
||||
message: dict,
|
||||
cache_marker: dict,
|
||||
static_system_prefix: str | None,
|
||||
*,
|
||||
native_anthropic: bool,
|
||||
) -> int:
|
||||
"""Mark the static system prefix and full prompt when they can be split.
|
||||
|
||||
The system prompt remains one stored string. Splitting it only in the
|
||||
outgoing request keeps session persistence and non-Anthropic transports
|
||||
unchanged while making the stable prefix independently cacheable.
|
||||
"""
|
||||
content = message.get("content")
|
||||
if (
|
||||
isinstance(static_system_prefix, str)
|
||||
and static_system_prefix
|
||||
and isinstance(content, str)
|
||||
and content.startswith(static_system_prefix)
|
||||
):
|
||||
suffix = content[len(static_system_prefix):]
|
||||
if suffix:
|
||||
message["content"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": static_system_prefix,
|
||||
"cache_control": cache_marker,
|
||||
},
|
||||
{"type": "text", "text": suffix, "cache_control": cache_marker},
|
||||
]
|
||||
return 2
|
||||
|
||||
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
|
||||
return 1
|
||||
|
||||
|
||||
def apply_anthropic_cache_control(
|
||||
api_messages: List[Dict[str, Any]],
|
||||
cache_ttl: str = "5m",
|
||||
native_anthropic: bool = False,
|
||||
static_system_prefix: str | None = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply system_and_3 caching strategy to messages for Anthropic models.
|
||||
"""Apply Anthropic cache-control markers to API messages.
|
||||
|
||||
Places up to 4 cache_control breakpoints: system prompt + last 3 non-system
|
||||
messages, all at the same TTL.
|
||||
When ``static_system_prefix`` exactly matches the beginning of a string
|
||||
system prompt, it receives an early marker and the full system prompt gets
|
||||
a trailing marker. The remaining two markers target the latest cacheable
|
||||
non-system messages. Without that prefix, the legacy system-and-3 layout
|
||||
is retained.
|
||||
|
||||
Returns:
|
||||
Deep copy of messages with cache_control breakpoints injected.
|
||||
|
|
@ -103,8 +145,12 @@ def apply_anthropic_cache_control(
|
|||
breakpoints_used = 0
|
||||
|
||||
if messages[0].get("role") == "system":
|
||||
_apply_cache_marker(messages[0], marker, native_anthropic=native_anthropic)
|
||||
breakpoints_used += 1
|
||||
breakpoints_used = _apply_system_cache_markers(
|
||||
messages[0],
|
||||
marker,
|
||||
static_system_prefix,
|
||||
native_anthropic=native_anthropic,
|
||||
)
|
||||
|
||||
remaining = 4 - breakpoints_used
|
||||
non_sys = [
|
||||
|
|
|
|||
8
agent/proxy_sources/__init__.py
Normal file
8
agent/proxy_sources/__init__.py
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
"""Egress proxy integrations.
|
||||
|
||||
Currently ships an iron-proxy (ironsh/iron-proxy) wrapper that intercepts
|
||||
outbound traffic from remote terminal sandboxes and swaps proxy tokens
|
||||
for real upstream credentials at the network edge.
|
||||
|
||||
Design notes live in :mod:`agent.proxy_sources.iron_proxy`.
|
||||
"""
|
||||
2494
agent/proxy_sources/iron_proxy.py
Normal file
2494
agent/proxy_sources/iron_proxy.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -102,6 +102,7 @@ _REASONING_STALE_TIMEOUT_FLOORS: tuple[tuple[str, int], ...] = (
|
|||
# ``claude-opus-4`` so non-thinking Claude 3.x or future
|
||||
# non-reasoning Claude variants don't match.
|
||||
("claude-opus-4", 240),
|
||||
("claude-opus-5", 240),
|
||||
("claude-sonnet-5", 180),
|
||||
("claude-sonnet-4.5", 180),
|
||||
("claude-sonnet-4.6", 180),
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ def run_secret_cli(
|
|||
list(argv),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=timeout,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -200,7 +200,7 @@ def _platform_asset_name() -> str:
|
|||
res = subprocess.run(
|
||||
["ldd", "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=2,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
@ -684,7 +684,7 @@ def _run_bws_list(
|
|||
cmd,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=_BWS_RUN_TIMEOUT,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -464,7 +464,7 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
|
|||
input=stdin_json,
|
||||
capture_output=True,
|
||||
timeout=spec.timeout,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
shell=False,
|
||||
**_popen_kwargs,
|
||||
)
|
||||
|
|
@ -632,7 +632,7 @@ def allowlist_path() -> Path:
|
|||
def load_allowlist() -> Dict[str, Any]:
|
||||
"""Return the parsed allowlist, or an empty skeleton if absent."""
|
||||
try:
|
||||
raw = json.loads(allowlist_path().read_text())
|
||||
raw = json.loads(allowlist_path().read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
return {"approvals": []}
|
||||
if not isinstance(raw, dict):
|
||||
|
|
|
|||
|
|
@ -54,6 +54,21 @@ _BUNDLE_MARKER = " skill bundle,"
|
|||
_BUNDLE_USER_INSTRUCTION = "\nUser instruction: "
|
||||
_BUNDLE_FIRST_SKILL_BLOCK = "\n\n[Loaded as part of the "
|
||||
|
||||
# The skill name sits in the first quoted span of the activation note, for both
|
||||
# the single-skill and the bundle header ("work" / "/clean /work").
|
||||
_SKILL_NAME_RE = re.compile(re.escape(_SKILL_INVOCATION_PREFIX) + r'"([^"]*)"')
|
||||
|
||||
# SQL LIKE pattern matching a skill-expanded turn, for listing queries that
|
||||
# have to recognize scaffolding before the row reaches Python. The prefix
|
||||
# contains no LIKE wildcards (`%`, `_`), so it needs no ESCAPE clause.
|
||||
SKILL_SCAFFOLD_SQL_LIKE = _SKILL_INVOCATION_PREFIX + "%"
|
||||
|
||||
# Marks where a preview query joined the head and tail of a long scaffolded
|
||||
# message. ``describe_skill_invocation`` may hand back a span that runs across
|
||||
# the joint (a bundle instruction cut off by the head window); callers cut the
|
||||
# description there rather than show the skill body on the far side.
|
||||
SKILL_EXCERPT_JOINT = "\x1e"
|
||||
|
||||
|
||||
def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
|
||||
"""Recover the user's instruction from a slash-skill-expanded turn.
|
||||
|
|
@ -82,6 +97,41 @@ def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
|
|||
return None
|
||||
|
||||
|
||||
def describe_skill_invocation(content: Any) -> Optional[str]:
|
||||
"""Render a slash-skill-expanded turn the way the user typed it.
|
||||
|
||||
The expanded message embeds the whole skill body, so any surface that
|
||||
summarizes a user turn from its raw content — session titles, sidebar
|
||||
previews, the ``/rewind`` picker — otherwise shows the skill's own prose
|
||||
as if the user had written it. That is how a skill's opening line ends up
|
||||
as a session title.
|
||||
|
||||
Returns ``"/work — fix the title leak"``, or ``"/work"`` for a bare
|
||||
invocation, or ``None`` when *content* is not skill scaffolding (the
|
||||
caller should then summarize it as an ordinary message).
|
||||
"""
|
||||
if not isinstance(content, str) or not content.startswith(_SKILL_INVOCATION_PREFIX):
|
||||
return None
|
||||
|
||||
match = _SKILL_NAME_RE.match(content)
|
||||
name = (match.group(1) if match else "").strip()
|
||||
# Bundle headers already carry their typed "/a /b" keys; a single skill is
|
||||
# a bare name.
|
||||
label = name if name.startswith("/") else f"/{name}"
|
||||
|
||||
instruction = extract_user_instruction_from_skill_message(content)
|
||||
if instruction and instruction is not content:
|
||||
# An excerpted message (head + tail, joined by SKILL_EXCERPT_JOINT) can
|
||||
# put the joint inside the matched span — keep only the side the
|
||||
# instruction marker was found on.
|
||||
instruction = instruction.split(SKILL_EXCERPT_JOINT)[0]
|
||||
instruction = " ".join(instruction.split())
|
||||
if instruction:
|
||||
return f"{label} — {instruction}" if name else instruction
|
||||
|
||||
return label if name else None
|
||||
|
||||
|
||||
def _extract_single_skill_user_instruction(message: str) -> Optional[str]:
|
||||
# Single-skill format appends the user instruction after the skill body, so
|
||||
# the last occurrence is the user-provided one; the body may quote this text.
|
||||
|
|
@ -453,8 +503,8 @@ def reload_skills() -> Dict[str, Any]:
|
|||
}
|
||||
|
||||
``description`` is the skill's full SKILL.md frontmatter
|
||||
``description:`` field — the same string the system prompt renders
|
||||
as `` - name: description`` for pre-existing skills.
|
||||
``description:`` field. Note: the system prompt skill index
|
||||
truncates this to the first 57 chars; see ``extract_skill_description``.
|
||||
"""
|
||||
# Snapshot pre-reload state (name -> description) from the current
|
||||
# slash-command cache. Using dicts lets the post-rescan diff carry
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ def run_inline_shell(command: str, cwd: Path | None, timeout: int) -> str:
|
|||
["bash", "-c", command],
|
||||
cwd=str(cwd) if cwd else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=max(1, int(timeout)),
|
||||
check=False,
|
||||
stdin=subprocess.DEVNULL,
|
||||
|
|
|
|||
|
|
@ -779,18 +779,31 @@ def resolve_skill_config_values(
|
|||
|
||||
# ── Description extraction ────────────────────────────────────────────────
|
||||
|
||||
SKILL_PROMPT_DESC_LIMIT = 60
|
||||
|
||||
|
||||
def _normalize_skill_description(frontmatter: Dict[str, Any]) -> str:
|
||||
"""Normalize a skill's description field for comparison/truncation."""
|
||||
raw_desc = frontmatter.get("description", "")
|
||||
return str(raw_desc).strip().strip("'\"") if raw_desc else ""
|
||||
|
||||
|
||||
def extract_skill_description(frontmatter: Dict[str, Any]) -> str:
|
||||
"""Extract a truncated description from parsed frontmatter."""
|
||||
raw_desc = frontmatter.get("description", "")
|
||||
if not raw_desc:
|
||||
"""Extract a system-prompt-length description from parsed frontmatter."""
|
||||
desc = _normalize_skill_description(frontmatter)
|
||||
if not desc:
|
||||
return ""
|
||||
desc = str(raw_desc).strip().strip("'\"")
|
||||
if len(desc) > 60:
|
||||
return desc[:57] + "..."
|
||||
if len(desc) > SKILL_PROMPT_DESC_LIMIT:
|
||||
return desc[:SKILL_PROMPT_DESC_LIMIT - 3] + "..."
|
||||
return desc
|
||||
|
||||
|
||||
def is_skill_description_truncated_for_prompt(frontmatter: Dict[str, Any]) -> bool:
|
||||
"""True when the description will be truncated in the system prompt skill index."""
|
||||
desc = _normalize_skill_description(frontmatter)
|
||||
return len(desc) > SKILL_PROMPT_DESC_LIMIT
|
||||
|
||||
|
||||
# ── File iteration ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ def _skip_ssl_guard_enabled() -> bool:
|
|||
|
||||
def _repair_hint() -> str:
|
||||
return (
|
||||
"Repair: python -m pip install --force-reinstall certifi openai httpx\n"
|
||||
"Repair: run `hermes doctor --fix` (auto-reinstalls certifi), or "
|
||||
"manually: python -m pip install --force-reinstall certifi openai httpx\n"
|
||||
"If you configured a custom corporate CA bundle, fix or unset the "
|
||||
"broken CA bundle environment variable."
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,9 +12,11 @@ Three tiers are joined with ``\\n\\n``:
|
|||
* ``stable`` — identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool
|
||||
guidance, computer-use guidance, nous subscription block, tool-use
|
||||
enforcement guidance + per-model operational guidance, skills prompt,
|
||||
alibaba model-name workaround, environment hints, platform hints.
|
||||
alibaba model-name workaround, environment hints, coding guidance,
|
||||
platform hints.
|
||||
* ``context`` — caller-supplied ``system_message`` plus context files
|
||||
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``.
|
||||
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``,
|
||||
plus the session's coding-workspace snapshot.
|
||||
* ``volatile`` — memory snapshot, USER.md profile, external memory
|
||||
provider block, timestamp/session/model/provider line.
|
||||
|
||||
|
|
@ -145,14 +147,14 @@ def _tui_embedded_pane_clarifier(hint: str) -> str:
|
|||
|
||||
|
||||
def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]:
|
||||
"""Assemble the system prompt as three ordered parts.
|
||||
"""Assemble the system prompt as three ordered cache tiers.
|
||||
|
||||
Returns a dict with three keys:
|
||||
* ``stable`` — identity, tool guidance, skills prompt,
|
||||
environment hints, platform hints, model-family operational
|
||||
guidance.
|
||||
* ``context`` — context files (AGENTS.md, .cursorrules, etc.)
|
||||
and caller-supplied system_message.
|
||||
* ``stable`` — the cross-session-stable prefix, through the coding
|
||||
operating brief when a workspace snapshot follows.
|
||||
* ``context`` — the workspace snapshot followed by the remaining
|
||||
session-stable guidance, context files, and caller-supplied
|
||||
system_message.
|
||||
* ``volatile`` — memory snapshot, user profile, external
|
||||
memory provider block, timestamp line.
|
||||
|
||||
|
|
@ -345,25 +347,35 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
stable_parts.append(_env_hints)
|
||||
|
||||
# Coding posture (base Hermes, any interactive coding surface in a code
|
||||
# workspace — see agent/coding_context.py). The operating brief + the live
|
||||
# git/workspace snapshot are built once here and cached for the session;
|
||||
# the snapshot is never re-probed per turn (that would break the prompt
|
||||
# cache), so the brief tells the model to re-check git before relying on it.
|
||||
# workspace — see agent/coding_context.py). Keep the operating brief in
|
||||
# the cross-session-stable prefix, while placing the live git/workspace
|
||||
# snapshot behind its own cache boundary. The post-snapshot blocks must
|
||||
# stay in their historical position after the workspace snapshot.
|
||||
coding_workspace_parts: List[str] = []
|
||||
coding_trailing_parts: List[str] = []
|
||||
if agent.valid_tool_names:
|
||||
try:
|
||||
from agent.coding_context import coding_system_blocks
|
||||
from agent.coding_context import coding_system_prompt_parts
|
||||
|
||||
stable_parts.extend(
|
||||
coding_system_blocks(
|
||||
platform=agent.platform,
|
||||
cwd=resolve_context_cwd(),
|
||||
model=agent.model,
|
||||
)
|
||||
coding_prefix_parts, coding_workspace_parts, coding_trailing_parts = coding_system_prompt_parts(
|
||||
platform=agent.platform,
|
||||
cwd=resolve_context_cwd(),
|
||||
model=agent.model,
|
||||
)
|
||||
stable_parts.extend(coding_prefix_parts)
|
||||
except Exception:
|
||||
# Coding-context probing must never block prompt build.
|
||||
pass
|
||||
|
||||
# Guidance assembled after the coding posture historically followed the
|
||||
# workspace snapshot. With no snapshot, the coding tail instead remains
|
||||
# directly after the coding prefix in the cacheable prefix.
|
||||
if coding_workspace_parts:
|
||||
post_workspace_parts: List[str] = []
|
||||
else:
|
||||
stable_parts.extend(coding_trailing_parts)
|
||||
post_workspace_parts = stable_parts
|
||||
|
||||
# Local Python toolchain probe — names python/pip/uv/PEP-668 state when
|
||||
# something is non-default so the model can pick the right install
|
||||
# strategy without discovering by failure. Emits a single line; emits
|
||||
|
|
@ -376,7 +388,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
from tools.env_probe import get_environment_probe_line
|
||||
_probe_line = get_environment_probe_line()
|
||||
if _probe_line:
|
||||
stable_parts.append(_probe_line)
|
||||
post_workspace_parts.append(_probe_line)
|
||||
except Exception:
|
||||
# Probe failure must never block prompt build.
|
||||
pass
|
||||
|
|
@ -394,7 +406,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
except Exception:
|
||||
active_profile = "default"
|
||||
if active_profile == "default":
|
||||
stable_parts.append(
|
||||
post_workspace_parts.append(
|
||||
"Active Hermes profile: default. Other profiles (if any) live "
|
||||
"under " + str(get_hermes_home()) + "/profiles/<name>/. Each profile has its own "
|
||||
"skills/, plugins/, cron/, and memories/ that affect a different "
|
||||
|
|
@ -403,7 +415,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
"you to."
|
||||
)
|
||||
else:
|
||||
stable_parts.append(
|
||||
post_workspace_parts.append(
|
||||
f"Active Hermes profile: {active_profile}. This session reads "
|
||||
f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default "
|
||||
f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, "
|
||||
|
|
@ -449,11 +461,16 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
if platform_key == "tui" and _effective_hint:
|
||||
_effective_hint = _tui_embedded_pane_clarifier(_effective_hint)
|
||||
if _effective_hint:
|
||||
stable_parts.append(_effective_hint)
|
||||
post_workspace_parts.append(_effective_hint)
|
||||
|
||||
# ── Context tier (cwd-dependent, may change between sessions) ─
|
||||
context_parts: List[str] = []
|
||||
|
||||
if coding_workspace_parts:
|
||||
context_parts.extend(coding_workspace_parts)
|
||||
context_parts.extend(coding_trailing_parts)
|
||||
context_parts.extend(post_workspace_parts)
|
||||
|
||||
# Note: ephemeral_system_prompt is NOT included here. It's injected at
|
||||
# API-call time only so it stays out of the cached/stored system prompt.
|
||||
if system_message is not None:
|
||||
|
|
@ -515,6 +532,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
timestamp_line += f"\nModel: {agent.model}"
|
||||
if agent.provider:
|
||||
timestamp_line += f"\nProvider: {agent.provider}"
|
||||
if agent.platform:
|
||||
timestamp_line += f"\nPlatform: {agent.platform}"
|
||||
volatile_parts.append(timestamp_line)
|
||||
|
||||
return {
|
||||
|
|
@ -541,6 +560,7 @@ def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str
|
|||
"""
|
||||
parts = build_system_prompt_parts(agent, system_message=system_message)
|
||||
joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p)
|
||||
agent._cached_system_prompt_static = parts["stable"]
|
||||
|
||||
# Surface context-file truncation warnings through the normal agent status
|
||||
# channel so gateway/CLI users see them in chat instead of only in logs.
|
||||
|
|
@ -557,6 +577,7 @@ def invalidate_system_prompt(agent: Any) -> None:
|
|||
so the rebuilt prompt captures any writes from this session.
|
||||
"""
|
||||
agent._cached_system_prompt = None
|
||||
agent._cached_system_prompt_static = None
|
||||
if agent._memory_store:
|
||||
agent._memory_store.load_from_disk()
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _summarize_user_message(user_message: str) -> str:
|
||||
"""Collapse a slash-skill-expanded turn back to what the user typed.
|
||||
|
||||
A ``/skill`` invocation expands into a message that embeds the whole skill
|
||||
body, so feeding it to the titler verbatim titles the session after the
|
||||
*skill's* prose — "Kick off a task in a fresh isolated git worktree" — not
|
||||
after the user's request. Reuse the canonical scaffolding parser so the
|
||||
model sees ``/work — fix the title leak`` instead.
|
||||
"""
|
||||
if not user_message:
|
||||
return ""
|
||||
try:
|
||||
from agent.skill_commands import describe_skill_invocation
|
||||
|
||||
described = describe_skill_invocation(user_message)
|
||||
except Exception:
|
||||
logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True)
|
||||
return user_message
|
||||
return described if described is not None else user_message
|
||||
|
||||
|
||||
def generate_title(
|
||||
user_message: str,
|
||||
assistant_response: str,
|
||||
|
|
@ -110,7 +131,7 @@ def generate_title(
|
|||
logger.debug("Title runtime validator raised; proceeding", exc_info=True)
|
||||
|
||||
# Truncate long messages to keep the request small
|
||||
user_snippet = user_message[:500] if user_message else ""
|
||||
user_snippet = _summarize_user_message(user_message)[:500]
|
||||
assistant_snippet = assistant_response[:500] if assistant_response else ""
|
||||
|
||||
language = _title_language()
|
||||
|
|
@ -143,6 +164,11 @@ def generate_title(
|
|||
title = title.strip('"\'')
|
||||
if title.lower().startswith("title:"):
|
||||
title = title[6:].strip()
|
||||
# A title is one line. A model that ignores "return ONLY the title" and
|
||||
# answers the prompt instead (a shell transcript, a bulleted plan) would
|
||||
# otherwise be stored verbatim and truncated mid-command. Keep the first
|
||||
# non-empty line — the closest thing to a title in that response.
|
||||
title = next((line.strip() for line in title.splitlines() if line.strip()), "")
|
||||
# Enforce reasonable length
|
||||
if len(title) > 80:
|
||||
title = title[:77] + "..."
|
||||
|
|
|
|||
|
|
@ -1204,7 +1204,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}")
|
||||
|
||||
agent._current_tool = None
|
||||
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)")
|
||||
_status_suffix = " (error)" if is_error else ""
|
||||
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}")
|
||||
|
||||
if not blocked and agent.tool_complete_callback:
|
||||
try:
|
||||
|
|
@ -1826,7 +1827,8 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
agent._current_tool = None
|
||||
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)")
|
||||
_status_suffix = " (error)" if _is_error_result else ""
|
||||
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}")
|
||||
|
||||
if agent.verbose_logging:
|
||||
logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s")
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ def build_trace_jsonl(
|
|||
if cwd:
|
||||
r = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
capture_output=True, text=True, timeout=3, cwd=cwd,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3, cwd=cwd,
|
||||
)
|
||||
if r.returncode == 0:
|
||||
git_branch = r.stdout.strip()
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ streaming, or the _run_codex_stream() call path.
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.transports.base import ProviderTransport
|
||||
|
|
@ -27,6 +28,49 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
|
|||
return f"pck_{digest}"
|
||||
|
||||
|
||||
_EXTENDED_PROMPT_CACHE_MODELS = (
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5.5",
|
||||
"gpt-5.4",
|
||||
"gpt-5.2",
|
||||
"gpt-5.1-codex-max",
|
||||
"gpt-5.1-codex-mini",
|
||||
"gpt-5.1-chat-latest",
|
||||
"gpt-5.1-codex",
|
||||
"gpt-5.1",
|
||||
"gpt-5-codex",
|
||||
"gpt-5",
|
||||
"gpt-4.1",
|
||||
)
|
||||
_EXTENDED_PROMPT_CACHE_MODEL_RE = re.compile(
|
||||
rf"(?:^|[./:])(?:{'|'.join(re.escape(name) for name in _EXTENDED_PROMPT_CACHE_MODELS)})"
|
||||
r"(?:-\d{4}-\d{2}-\d{2})?$"
|
||||
)
|
||||
|
||||
|
||||
def _default_prompt_cache_retention_for_request(
|
||||
model: str,
|
||||
base_url: Any,
|
||||
) -> Optional[str]:
|
||||
"""Return ``24h`` for supported models on Amazon Bedrock Mantle."""
|
||||
from utils import base_url_hostname
|
||||
|
||||
hostname_parts = base_url_hostname(str(base_url or "")).split(".")
|
||||
is_bedrock_mantle = (
|
||||
len(hostname_parts) == 4
|
||||
and hostname_parts[0] == "bedrock-mantle"
|
||||
and bool(hostname_parts[1])
|
||||
and hostname_parts[2:] == ["api", "aws"]
|
||||
)
|
||||
if not is_bedrock_mantle:
|
||||
return None
|
||||
|
||||
normalized = str(model or "").strip().lower().replace("_", "-")
|
||||
if _EXTENDED_PROMPT_CACHE_MODEL_RE.search(normalized):
|
||||
return "24h"
|
||||
return None
|
||||
|
||||
|
||||
def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]:
|
||||
"""Content-address the prompt cache key from the static request prefix.
|
||||
|
||||
|
|
@ -284,6 +328,13 @@ class ResponsesApiTransport(ProviderTransport):
|
|||
if not is_github_responses and not is_xai_responses and cache_key:
|
||||
kwargs["prompt_cache_key"] = cache_key
|
||||
|
||||
cache_retention = _default_prompt_cache_retention_for_request(
|
||||
model,
|
||||
params.get("base_url"),
|
||||
)
|
||||
if cache_retention:
|
||||
kwargs.setdefault("prompt_cache_retention", cache_retention)
|
||||
|
||||
if reasoning_enabled and is_xai_responses:
|
||||
from agent.model_metadata import grok_supports_reasoning_effort
|
||||
|
||||
|
|
|
|||
|
|
@ -394,7 +394,7 @@ def check_codex_binary(
|
|||
proc = subprocess.run(
|
||||
[codex_bin, "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=10,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,9 @@ from typing import Any, Dict, List, Mapping, Optional
|
|||
from agent.conversation_compression import (
|
||||
IDLE_COMPACTION_STATUS_TEMPLATE,
|
||||
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
|
||||
compression_skipped_due_to_lock,
|
||||
conversation_history_after_compression,
|
||||
recover_rotated_compression_session,
|
||||
)
|
||||
from agent.context_engine import automatic_compaction_status_message
|
||||
from agent.iteration_budget import IterationBudget
|
||||
|
|
@ -352,6 +354,13 @@ def build_turn_context(
|
|||
# Guard stdio against OSError from broken pipes (systemd/headless/daemon).
|
||||
install_safe_stdio()
|
||||
|
||||
# Recover a session rotated by another path before binding log/turn ids or
|
||||
# copying client-supplied history. Everything in this turn must consistently
|
||||
# belong to the canonical child, including observability metadata.
|
||||
recovered_history = recover_rotated_compression_session(agent)
|
||||
if recovered_history is not None:
|
||||
conversation_history = recovered_history
|
||||
|
||||
# NOTE: the DB session row is created later, AFTER the system prompt is
|
||||
# restored/built (see _ensure_db_session() below the system-prompt block).
|
||||
# Creating it here — before _cached_system_prompt is populated — inserts a
|
||||
|
|
@ -710,6 +719,8 @@ def build_turn_context(
|
|||
# issue #27405 (a few very large messages slipping past the count gate).
|
||||
_preflight_compressed = False
|
||||
_preflight_compression_blocked = False
|
||||
agent._turn_received_provider_response = False
|
||||
agent._turn_preflight_display_snapshot = None
|
||||
if agent.compression_enabled and _should_run_preflight_estimate(
|
||||
messages,
|
||||
agent.context_compressor.protect_first_n,
|
||||
|
|
@ -722,6 +733,21 @@ def build_turn_context(
|
|||
tools=agent.tools or None,
|
||||
)
|
||||
_compressor = agent.context_compressor
|
||||
# getattr guard: minimal compressor doubles (SimpleNamespace in the
|
||||
# engine-preflight tests) and plugin context engines lack this
|
||||
# ContextCompressor-only method — absence means no snapshot, and the
|
||||
# finalizer's rollback stays disarmed for the turn (display-only).
|
||||
_snapshot_fn = getattr(
|
||||
_compressor, "snapshot_preflight_display_tokens", None
|
||||
)
|
||||
if callable(_snapshot_fn):
|
||||
_snapshot_val = _snapshot_fn()
|
||||
# Type pin: MagicMock compressors return truthy Mock objects —
|
||||
# only a real int snapshot may arm the interrupted-turn rollback.
|
||||
if isinstance(_snapshot_val, int) and not isinstance(
|
||||
_snapshot_val, bool
|
||||
):
|
||||
agent._turn_preflight_display_snapshot = _snapshot_val
|
||||
_defer_preflight = getattr(
|
||||
_compressor,
|
||||
"should_defer_preflight_to_real_usage",
|
||||
|
|
@ -838,10 +864,29 @@ def build_turn_context(
|
|||
for _pass in range(_max_preflight_passes):
|
||||
_orig_len = len(messages)
|
||||
_orig_tokens = _preflight_tokens
|
||||
_preflight_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message, approx_tokens=_preflight_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
if (
|
||||
messages is _preflight_input
|
||||
and compression_skipped_due_to_lock(agent)
|
||||
):
|
||||
# #69870 lock-skip: another path holds this session's
|
||||
# compression lock, so the pass no-oped. That is a
|
||||
# temporary DEFER, not proof the transcript cannot
|
||||
# compress — do NOT arm the insufficient-progress
|
||||
# blocker (the loop's error handlers must keep their
|
||||
# provider-proven retry budget) and stop preflight
|
||||
# passes for this turn; the lock winner is shrinking
|
||||
# the same session concurrently.
|
||||
logger.info(
|
||||
"Preflight compression deferred: compression lock "
|
||||
"held by another path (session %s)",
|
||||
agent.session_id or "none",
|
||||
)
|
||||
break
|
||||
# Re-estimate now so size-only compression (same row count,
|
||||
# lower token count — e.g. summarising tool outputs) is
|
||||
# recognised as progress instead of being misread as
|
||||
|
|
|
|||
|
|
@ -201,6 +201,36 @@ def finalize_turn(
|
|||
)
|
||||
)
|
||||
|
||||
# Preflight can seed the display count before the provider receives the
|
||||
# request. Roll that estimate back only when an interrupt wins the race
|
||||
# before any successful provider response. Compaction state remains owned
|
||||
# by the real-usage/post-compaction path, including its ``-1`` sentinel.
|
||||
# Guard rules (test-double density on this path is high):
|
||||
# - snapshot is type-pinned to a real int — MagicMock agents auto-create
|
||||
# truthy Mock attributes that must never arm the rollback;
|
||||
# - the received-response flag is pinned to ``is not True`` — its real
|
||||
# domain is True/False, and only a literal True means a provider
|
||||
# response completed;
|
||||
# - the compressor method gets a getattr+callable guard — SimpleNamespace
|
||||
# compressor doubles and plugin context engines lack it.
|
||||
_preflight_snapshot = getattr(
|
||||
agent, "_turn_preflight_display_snapshot", None
|
||||
)
|
||||
if (
|
||||
interrupted is True
|
||||
and isinstance(_preflight_snapshot, int)
|
||||
and not isinstance(_preflight_snapshot, bool)
|
||||
and getattr(agent, "_turn_received_provider_response", False) is not True
|
||||
and getattr(agent, "context_compressor", None) is not None
|
||||
):
|
||||
_rollback_fn = getattr(
|
||||
agent.context_compressor,
|
||||
"rollback_interrupted_preflight_display_tokens",
|
||||
None,
|
||||
)
|
||||
if callable(_rollback_fn):
|
||||
_rollback_fn(_preflight_snapshot)
|
||||
|
||||
# Post-loop cleanup must never lose the response. Trajectory save,
|
||||
# resource teardown, and session persistence all touch fallible
|
||||
# surfaces — file I/O / JSON serialization (_save_trajectory), remote
|
||||
|
|
@ -495,6 +525,34 @@ def finalize_turn(
|
|||
except Exception as exc:
|
||||
logger.warning("post_llm_call hook failed: %s", exc)
|
||||
|
||||
# Context engine observation hook: notify the active engine that this
|
||||
# turn has finished, with the finalized transcript. Complements the
|
||||
# per-request select_context() hook (selection before the request;
|
||||
# observation after the turn). No-op default, fail-open.
|
||||
try:
|
||||
from agent.conversation_loop import _notify_context_engine_turn_complete
|
||||
# Forward the turn's canonical usage when the host has it. The loop
|
||||
# stashes the most recent API response's usage dict (the same
|
||||
# canonical buckets fed to ``update_from_response``) on the agent as
|
||||
# ``_last_turn_usage``. It is ``None`` on turns that never reached a
|
||||
# provider response (early failure / interrupt), which is exactly the
|
||||
# contract: real usage when available, ``None`` otherwise.
|
||||
_turn_usage = getattr(agent, "_last_turn_usage", None)
|
||||
_notify_context_engine_turn_complete(
|
||||
agent,
|
||||
messages,
|
||||
usage=_turn_usage,
|
||||
logger=logger,
|
||||
turn_id=turn_id,
|
||||
task_id=effective_task_id,
|
||||
api_call_count=api_call_count,
|
||||
interrupted=interrupted,
|
||||
failed=failed,
|
||||
turn_exit_reason=_turn_exit_reason,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("on_turn_complete notification failed: %s", exc)
|
||||
|
||||
# Extract reasoning from the CURRENT turn only. Walk backwards
|
||||
# but stop at the user message that started this turn — anything
|
||||
# earlier is from a prior turn and must not leak into the reasoning
|
||||
|
|
@ -627,4 +685,7 @@ def finalize_turn(
|
|||
except Exception as exc:
|
||||
logger.warning("on_session_end hook failed: %s", exc)
|
||||
|
||||
agent._turn_preflight_display_snapshot = None
|
||||
agent._turn_received_provider_response = False
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ import shlex
|
|||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Iterator, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
|
|
@ -65,13 +66,38 @@ def _connect() -> sqlite3.Connection:
|
|||
path = _db_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.row_factory = sqlite3.Row
|
||||
_ensure_schema(conn)
|
||||
try:
|
||||
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
_ensure_schema(conn)
|
||||
except Exception:
|
||||
# A PRAGMA/DDL failure after a successful connect() must not leak the
|
||||
# just-opened connection back to the caller.
|
||||
conn.close()
|
||||
raise
|
||||
return conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction() -> Iterator[sqlite3.Connection]:
|
||||
"""Open a connection, commit/rollback on exit, and ALWAYS close it.
|
||||
|
||||
``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the
|
||||
transaction; they do not close the connection. Using ``with _connect()``
|
||||
alone therefore leaks a connection — and its WAL/SHM file descriptors — on
|
||||
every call, deferring the close to the garbage collector, which over a
|
||||
long-running process can exhaust ``RLIMIT_NOFILE`` (the cron-ledger sibling
|
||||
of this bug was #69567 / PR #69594).
|
||||
"""
|
||||
conn = _connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
|
|
@ -454,7 +480,7 @@ def record_terminal_result(
|
|||
|
||||
created_at = _utc_now()
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO verification_events(
|
||||
|
|
@ -520,7 +546,7 @@ def mark_workspace_edited(
|
|||
edited_at = _utc_now()
|
||||
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT changed_paths_json FROM verification_state
|
||||
|
|
@ -570,7 +596,7 @@ def verification_status(
|
|||
sid = str(session_id or "default")
|
||||
root = str(facts.get("root") or Path(cwd or ".").resolve())
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
state = conn.execute(
|
||||
"""
|
||||
SELECT last_event_id, last_edit_at, changed_paths_json
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ Already have the Hermes CLI? Just run:
|
|||
hermes desktop
|
||||
```
|
||||
|
||||
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. On first launch Hermes walks you through picking a provider and model; nothing else to configure.
|
||||
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. If Desktop cannot find a usable runtime or saved remote connection, first launch lets you connect to an existing Hermes gateway or install Hermes locally. Local onboarding then walks you through choosing a provider and model.
|
||||
|
||||
### Prebuilt installers
|
||||
|
||||
|
|
@ -134,6 +134,19 @@ Desktop supports a managed local backend, explicit remote gateways, and Hermes
|
|||
Cloud connections. Remote and cloud modes use the same remote-capability path;
|
||||
authentication and discovery differ, not the renderer feature model.
|
||||
|
||||
When no usable local runtime or saved remote connection exists, the first-run
|
||||
screen offers **Connect to existing Hermes** before starting the local installer.
|
||||
Desktop probes the gateway to discover token or OAuth authentication, requires a
|
||||
successful HTTP and WebSocket connection test, and saves the connection using
|
||||
the same encrypted Desktop configuration used by Settings. A saved remote
|
||||
connection bypasses this choice on later launches. The regular Desktop build
|
||||
still includes the local-install option; this is a remote operating mode, not a
|
||||
separate client-only application.
|
||||
|
||||
In remote mode the gateway host is the execution boundary: agent tools,
|
||||
terminal commands, and file operations run against the remote Hermes host, not
|
||||
the computer displaying the Desktop UI.
|
||||
|
||||
Projects are the workspace abstraction. A project may own multiple folders,
|
||||
repositories, worktrees, and sessions; a bare new chat remains detached unless
|
||||
the user enters a project or configures a default project directory. Use the
|
||||
|
|
|
|||
|
|
@ -21,8 +21,16 @@ const INFERENCE_SWITCH_TRIGGER = 'E2E_INFERENCE_SWITCH_TRIGGER'
|
|||
const INFERENCE_PROMPT = `${INFERENCE_SWITCH_TRIGGER}: original inference prompt must remain singular.`
|
||||
const INFERENCE_CORRECTION = `${INFERENCE_SWITCH_TRIGGER}: correction sent while inference is live.`
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
|
||||
function activeSurface(page: Page) {
|
||||
return page.locator(SURFACE).last()
|
||||
}
|
||||
|
||||
async function send(page: Page, text: string): Promise<void> {
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
const composer = activeSurface(page).locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await composer.click()
|
||||
await composer.type(text, { delay: 5 })
|
||||
|
|
@ -30,8 +38,9 @@ async function send(page: Page, text: string): Promise<void> {
|
|||
}
|
||||
|
||||
async function steer(page: Page, text: string): Promise<void> {
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
const primary = page.locator('[data-slot="composer-root"] button[type="submit"]')
|
||||
const surface = activeSurface(page)
|
||||
const composer = surface.locator('[contenteditable="true"]').first()
|
||||
const primary = surface.locator('[data-slot="composer-root"] button[type="submit"]')
|
||||
|
||||
await composer.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await composer.click()
|
||||
|
|
@ -42,55 +51,80 @@ async function steer(page: Page, text: string): Promise<void> {
|
|||
|
||||
async function waitForTranscriptText(page: Page, text: string): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
(expected: string) => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
text,
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function textNodeOccurrences(page: Page, text: string): Promise<number> {
|
||||
return page.evaluate((expected: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return 0
|
||||
return page.evaluate(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return 0
|
||||
|
||||
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
|
||||
let count = 0
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent?.includes(expected)) {
|
||||
count += 1
|
||||
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
|
||||
let count = 0
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent?.includes(expected)) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return count
|
||||
}, text)
|
||||
return count
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
)
|
||||
}
|
||||
|
||||
async function transcriptTextOrder(page: Page): Promise<string[]> {
|
||||
return page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return []
|
||||
|
||||
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="message"], [data-message-id]'))
|
||||
.map(message => message.textContent?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
async function transcriptMessageOrder(page: Page): Promise<string[]> {
|
||||
return page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return []
|
||||
|
||||
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"]'))
|
||||
return Array.from(
|
||||
viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"], [data-role="system"]'),
|
||||
)
|
||||
.map(message => message.textContent?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidebar "+" opens a NEW TAB beside the current chat rather than
|
||||
* replacing it, so the prior session stays mounted in its own surface. Wait
|
||||
* for the newly-mounted surface to show an empty transcript instead of waiting
|
||||
* for the old text to disappear from the page (it never will).
|
||||
*/
|
||||
async function openFreshDraft(page: Page, priorSessionText: string): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await page.waitForFunction(
|
||||
(priorText: string) => !(document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(priorText),
|
||||
priorSessionText,
|
||||
([priorText, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
const transcript = active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return surfaces.length > 0 && !transcript.includes(priorText)
|
||||
},
|
||||
[priorSessionText, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
|
@ -116,7 +150,12 @@ async function reopenInferenceSession(page: Page): Promise<void> {
|
|||
}
|
||||
|
||||
function relevantOrder(messages: string[]): string[] {
|
||||
return messages.filter(message => message.includes(ORIGINAL_PROMPT) || message.includes(CORRECTION))
|
||||
return messages.flatMap(message => {
|
||||
if (message.includes(ORIGINAL_PROMPT)) return [ORIGINAL_PROMPT]
|
||||
if (message.includes(CORRECTION)) return [CORRECTION]
|
||||
|
||||
return []
|
||||
})
|
||||
}
|
||||
|
||||
function steerTurnOrder(messages: string[]): string[] {
|
||||
|
|
|
|||
197
apps/desktop/e2e/image-attachment-resume.spec.ts
Normal file
197
apps/desktop/e2e/image-attachment-resume.spec.ts
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
/**
|
||||
* Regression coverage for an attached image in a durable session. The gateway
|
||||
* persists the turn, the builder exits, and desktop renders it from SessionDB
|
||||
* for the first time — the "quit and relaunch" case, where the transcript used
|
||||
* to come back as vision-enrichment prose instead of a thumbnail.
|
||||
*
|
||||
* The fixture pins `image_input_mode: native` because that is the majority
|
||||
* routing path (any vision-capable model) and the one where a text-only
|
||||
* persist override is silently dropped. The image also sits behind directory
|
||||
* and file names containing spaces, mirroring the macOS composer's
|
||||
* `~/Library/Application Support/...` staging path.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import {
|
||||
buildAppEnv,
|
||||
createSandbox,
|
||||
launchDesktop,
|
||||
type Sandbox,
|
||||
waitForAppReady,
|
||||
writeEnvFile,
|
||||
writeMockProviderConfig,
|
||||
} from './fixtures'
|
||||
import { type MockServer, startMockServer } from './mock-server'
|
||||
import { RealSessionBuilder } from './real-session-builder'
|
||||
import { type ElectronApplication, expect, type Page, test } from './test'
|
||||
|
||||
// A seeded session has no generated title, so every label falls back to the
|
||||
// session preview — the first 60 characters of the first user message.
|
||||
const SESSION_TITLE = 'E2E attached image session'
|
||||
const CAPTION = 'E2E attached image must survive a relaunch'
|
||||
const IMAGE_DIR = 'Application Support/e2e shots'
|
||||
const IMAGE_NAME = 'e2e capture.png'
|
||||
const NATIVE_IMAGE_CONFIG = 'agent:\n image_input_mode: native'
|
||||
|
||||
/** A 160x100 framed magenta block — small, but visible in the screenshots. */
|
||||
const PNG_BASE64 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAKAAAABkCAIAAACO1KzYAAAA30lEQVR42u3dwQ2AIBAAQTAWAx1iBXYI7diCuWhEMvP2dZsj+CL30hLr2oxAYARGYARGYARGYIERGIH53n7nozpOk5rQqIcNdkQjMAIjMNPeomP3N54V+5exwY5oBEZgBEZgBEZggREYgREYgREYgQVGYARGYARGYARGYIERGIERGIERGIEFRmAERmAERmAEFhiBERiBERiBERiBBUZgBEZgBEZgBBYYgREYgREYgRFYYCMQGIERmPSj94Njb9ligxEYgRFYYJaQe2mmYIMRGIERGIERGIEFRmAERmDedAFtjAtAGWDnoAAAAABJRU5ErkJggg=='
|
||||
|
||||
interface SeededFixture {
|
||||
app: ElectronApplication
|
||||
mock: MockServer
|
||||
page: Page
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
function writeImage(sandbox: Sandbox): string {
|
||||
const dir = path.join(sandbox.root, IMAGE_DIR)
|
||||
fs.mkdirSync(dir, { recursive: true })
|
||||
|
||||
const imagePath = path.join(dir, IMAGE_NAME)
|
||||
fs.writeFileSync(imagePath, Buffer.from(PNG_BASE64, 'base64'))
|
||||
|
||||
return imagePath
|
||||
}
|
||||
|
||||
async function setupSeededDesktop(): Promise<SeededFixture> {
|
||||
const mock = await startMockServer()
|
||||
const sandbox = createSandbox('image-attachment')
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url, undefined, NATIVE_IMAGE_CONFIG)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
|
||||
|
||||
try {
|
||||
await builder.createSession({
|
||||
title: SESSION_TITLE,
|
||||
turns: [{ images: [writeImage(sandbox)], text: CAPTION }],
|
||||
})
|
||||
} finally {
|
||||
await builder.close()
|
||||
}
|
||||
|
||||
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
|
||||
|
||||
return {
|
||||
app,
|
||||
mock,
|
||||
page,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function sessionRow(page: Page) {
|
||||
return page.locator('[data-slot="sidebar"] button').filter({ hasText: CAPTION }).first()
|
||||
}
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
|
||||
function activeViewportText(surfaceSelector: string): string {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
|
||||
return surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
}
|
||||
|
||||
async function openSeededSession(page: Page): Promise<void> {
|
||||
const row = sessionRow(page)
|
||||
await row.waitFor({ state: 'visible', timeout: 60_000 })
|
||||
await row.click()
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return text.includes(expected)
|
||||
},
|
||||
[CAPTION, SURFACE] as [string, string],
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidebar "+" opens a NEW TAB beside the current chat instead of replacing
|
||||
* it, so the seeded session stays mounted in its own surface. Assert the new
|
||||
* surface is empty rather than waiting for the old caption to leave the page.
|
||||
*/
|
||||
async function openNewSession(page: Page): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return surfaces.length > 0 && !text.includes(expected)
|
||||
},
|
||||
[CAPTION, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function transcriptText(page: Page): Promise<string> {
|
||||
return page.evaluate(activeViewportText, SURFACE)
|
||||
}
|
||||
|
||||
async function assertRendersThumbnail(page: Page, label: string): Promise<void> {
|
||||
const thumbnail = page.locator('[data-slot="aui_directive-image"] img')
|
||||
await expect(thumbnail, `${label}: the attachment should render as an image`).toHaveCount(1)
|
||||
await expect(thumbnail, `${label}: the thumbnail should resolve off disk`).toHaveAttribute('src', /^data:image\//)
|
||||
|
||||
const text = await transcriptText(page)
|
||||
expect(text, `${label}: the caption should survive alongside the image`).toContain(CAPTION)
|
||||
// A broken ref falls back to a chip whose label leaks the path, and a
|
||||
// flattened multimodal turn leaves the agent's placeholder behind.
|
||||
expect(text, `${label}: the raw image path should not leak into the transcript`).not.toContain(IMAGE_NAME)
|
||||
expect(text, `${label}: the image directive should not render literally`).not.toContain('@image:')
|
||||
expect(text, `${label}: the flattening placeholder should not render`).not.toContain('[screenshot]')
|
||||
}
|
||||
|
||||
test.describe('attached image resume', () => {
|
||||
let fixture: SeededFixture | null = null
|
||||
|
||||
test.afterEach(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test('renders a persisted attachment as a thumbnail on first open and after a cold reload', async ({}, testInfo) => {
|
||||
// Seeding through the real gateway plus two full app boots does not fit the
|
||||
// default per-test budget on a cold runner.
|
||||
test.slow()
|
||||
|
||||
fixture = await setupSeededDesktop()
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
|
||||
// The sidebar labels a session by its preview, so the caption has to lead
|
||||
// the persisted turn — a leading directive reads as a truncated file path.
|
||||
const row = sessionRow(fixture.page)
|
||||
await row.waitFor({ state: 'visible', timeout: 60_000 })
|
||||
|
||||
const label = (await row.textContent())?.trim() ?? ''
|
||||
expect(label.startsWith(CAPTION), `sidebar label should open with the caption: ${label}`).toBe(true)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
await assertRendersThumbnail(fixture.page, 'first open')
|
||||
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-first-open.png') })
|
||||
|
||||
// A reload drops every cached attachment ref, so the transcript has to come
|
||||
// back from the persisted turn alone.
|
||||
await fixture.page.reload()
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
await openNewSession(fixture.page)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
await assertRendersThumbnail(fixture.page, 'cold reload')
|
||||
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-cold-reload.png') })
|
||||
})
|
||||
})
|
||||
|
|
@ -14,8 +14,11 @@
|
|||
* prove the full boot → gateway → inference → renderer chain works.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs'
|
||||
import http from 'node:http'
|
||||
import type { ServerResponse } from 'node:http'
|
||||
import os from 'node:os'
|
||||
import nodePath from 'node:path'
|
||||
|
||||
/** A canned assistant reply used for every chat completion request. */
|
||||
export const MOCK_REPLY = 'Hello from the mock inference server! The full boot chain is working.'
|
||||
|
|
@ -27,6 +30,14 @@ export interface MockServerOptions {
|
|||
holdFirstCompletionContaining?: string
|
||||
/** Absolute sandbox path written by the verify-on-stop scripted tool call. */
|
||||
verificationWritePath?: string
|
||||
/**
|
||||
* Sentinel path that ends the E2E_SIDEBAR_CROSS background process.
|
||||
*
|
||||
* Without it that process is a bare `sleep 5`, which races the agent turn and
|
||||
* the 4s auto-dismiss linger — see `createBackgroundReleaseHandle`. Pass a
|
||||
* handle's `path` to let the test decide when the process exits.
|
||||
*/
|
||||
backgroundReleasePath?: string
|
||||
}
|
||||
|
||||
export interface MockServer {
|
||||
|
|
@ -167,37 +178,68 @@ const SIDEBAR_SCRIPT: ScriptedTurn[] = [
|
|||
|
||||
// ─── Sidebar cross-session script ──────────────────────────────────────
|
||||
//
|
||||
// E2E_SIDEBAR_CROSS trigger uses a longer background process (sleep 5) so
|
||||
// the "background running" dot is visible long enough for the test to:
|
||||
// E2E_SIDEBAR_CROSS starts a long background process plus a subagent so the
|
||||
// tests can:
|
||||
// 1. See the background dot while the subagent runs.
|
||||
// 2. Open a different session and see session A's dot transition to
|
||||
// "finished unread" when the background process completes.
|
||||
//
|
||||
// The background process must outlive the agent turn — the whole point is a
|
||||
// dot that is still "running" after the final answer lands. A fixed `sleep`
|
||||
// cannot guarantee that: on a loaded CI runner the turn (two model round
|
||||
// trips + a real subagent delegation) can take longer than the sleep, the
|
||||
// process exits early, the 4s success linger elapses, and the dot is gone
|
||||
// before the test looks. That is a wall-clock race between three independent
|
||||
// timers, and it made this the flakiest spec in the suite.
|
||||
//
|
||||
// When `backgroundReleasePath` is set the process instead blocks until the
|
||||
// test creates that sentinel file, so the test — not the clock — decides when
|
||||
// the dot clears. `sleep 5` remains the fallback for callers that don't pass
|
||||
// a handle.
|
||||
function sidebarCrossBgCommand(releasePath?: string): string {
|
||||
if (!releasePath) {
|
||||
return 'echo "long bg output" && sleep 5 && echo "finished"'
|
||||
}
|
||||
// Bounded wait (60s): if a test forgets to release (or crashes mid-way),
|
||||
// the process still exits instead of hanging the worker until the suite
|
||||
// times out.
|
||||
const quoted = JSON.stringify(releasePath)
|
||||
return [
|
||||
'echo "long bg output"',
|
||||
`for _ in $(seq 1 600); do [ -e ${quoted} ] && break; sleep 0.1; done`,
|
||||
'echo "finished"',
|
||||
].join(' && ')
|
||||
}
|
||||
|
||||
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = [
|
||||
{
|
||||
text: 'Starting a long background task and delegating work.',
|
||||
toolCalls: [
|
||||
{
|
||||
name: 'terminal',
|
||||
args: {
|
||||
command: 'echo "long bg output" && sleep 5 && echo "finished"',
|
||||
background: true,
|
||||
notify_on_complete: true,
|
||||
function sidebarCrossScript(releasePath?: string): ScriptedTurn[] {
|
||||
return [
|
||||
{
|
||||
text: 'Starting a long background task and delegating work.',
|
||||
toolCalls: [
|
||||
{
|
||||
name: 'terminal',
|
||||
args: {
|
||||
command: sidebarCrossBgCommand(releasePath),
|
||||
background: true,
|
||||
notify_on_complete: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'delegate_task',
|
||||
args: {
|
||||
goal: 'Analyze cross-session state',
|
||||
context: 'Testing that the background dot updates across sessions.',
|
||||
{
|
||||
name: 'delegate_task',
|
||||
args: {
|
||||
goal: 'Analyze cross-session state',
|
||||
context: 'Testing that the background dot updates across sessions.',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
text: 'Both tasks are running in the background now.',
|
||||
},
|
||||
]
|
||||
],
|
||||
},
|
||||
{
|
||||
text: 'Both tasks are running in the background now.',
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = sidebarCrossScript()
|
||||
|
||||
const QUEUE_STOP_SCRIPT: ScriptedTurn[] = [
|
||||
{
|
||||
|
|
@ -423,7 +465,8 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
}
|
||||
|
||||
if (isSidebarCrossTrigger) {
|
||||
const turn = SIDEBAR_CROSS_SCRIPT[_sidebarCrossIndex] ?? SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1]
|
||||
const script = sidebarCrossScript(options.backgroundReleasePath)
|
||||
const turn = script[_sidebarCrossIndex] ?? script[script.length - 1]
|
||||
_sidebarCrossIndex++
|
||||
|
||||
if (stream) {
|
||||
|
|
@ -722,6 +765,65 @@ export function restartMockServer(): void {
|
|||
resetScriptIndex()
|
||||
}
|
||||
|
||||
/** Test-controlled lifetime for the E2E_SIDEBAR_CROSS background process. */
|
||||
export interface BackgroundReleaseHandle {
|
||||
/** Sentinel path — pass as `backgroundReleasePath` to `startMockServer`. */
|
||||
path: string
|
||||
/** End the background process now (creates the sentinel). */
|
||||
release: () => void
|
||||
/** Remove the sentinel if it still exists. Safe to call twice. */
|
||||
cleanup: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a sentinel that keeps the E2E_SIDEBAR_CROSS background process alive
|
||||
* until the test explicitly releases it.
|
||||
*
|
||||
* The cross-session sidebar tests need a background process that is still
|
||||
* RUNNING after the agent turn finishes — that is the state under test (a
|
||||
* session whose turn is done but whose background work is not). With a fixed
|
||||
* `sleep`, three independent clocks race: the sleep, the agent turn (two model
|
||||
* round trips plus a real subagent delegation), and the 4s success linger
|
||||
* before a finished task auto-dismisses. When a loaded CI runner makes the
|
||||
* turn slower than the sleep, the process is already gone and the assertion
|
||||
* samples an empty sidebar. Observed on CI 2026-07-26 across two unrelated
|
||||
* PRs: the "should appear" poll needed 7.5s to see the dot, by which point
|
||||
* `sleep 5` had exited.
|
||||
*
|
||||
* With a sentinel there is one clock and the test owns it:
|
||||
*
|
||||
* ```ts
|
||||
* const release = createBackgroundReleaseHandle()
|
||||
* const mock = await startMockServer({ backgroundReleasePath: release.path })
|
||||
* // ... assert the dot is visible; it cannot vanish on its own ...
|
||||
* release.release() // now, and only now, the process exits
|
||||
* ```
|
||||
*/
|
||||
export function createBackgroundReleaseHandle(): BackgroundReleaseHandle {
|
||||
const path = nodePath.join(
|
||||
os.tmpdir(),
|
||||
`hermes-e2e-bg-release-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
)
|
||||
return {
|
||||
path,
|
||||
release: () => {
|
||||
try {
|
||||
fs.writeFileSync(path, 'release')
|
||||
} catch {
|
||||
// The process also has a bounded fallback wait; a failed write must
|
||||
// not crash the test before its real assertions run.
|
||||
}
|
||||
},
|
||||
cleanup: () => {
|
||||
try {
|
||||
fs.rmSync(path, { force: true })
|
||||
} catch {
|
||||
// Best-effort — the sentinel lives in the OS temp dir.
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The interim script's text constants, exported for test assertions.
|
||||
* Each entry is the visible text of one turn. Turns with empty text
|
||||
|
|
@ -756,8 +858,12 @@ export const SIDEBAR_CROSS_TEXTS = {
|
|||
interimText: SIDEBAR_CROSS_SCRIPT[0].text,
|
||||
/** The final answer text. */
|
||||
finalText: SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1].text,
|
||||
/** The longer background process command (sleep 5). */
|
||||
bgCommand: 'echo "long bg output" && sleep 5 && echo "finished"',
|
||||
/**
|
||||
* The default (unheld) background process command. Tests that pass a
|
||||
* `backgroundReleasePath` get a sentinel-waiting command instead — see
|
||||
* `createBackgroundReleaseHandle`.
|
||||
*/
|
||||
bgCommand: sidebarCrossBgCommand(),
|
||||
/** The subagent's goal. */
|
||||
subagentGoal: 'Analyze cross-session state',
|
||||
} as const
|
||||
|
|
|
|||
|
|
@ -28,11 +28,18 @@ interface CreatedSession {
|
|||
stored_session_id: string
|
||||
}
|
||||
|
||||
export interface RealSessionTurn {
|
||||
/** Local image paths attached before the prompt, as the composer would. */
|
||||
images?: readonly string[]
|
||||
text: string
|
||||
}
|
||||
|
||||
export interface RealSessionSpec {
|
||||
/** Human-visible sidebar title, persisted by the first completed turn. */
|
||||
/** Session label. The durable row stores no title, so clients fall back to
|
||||
* the preview (the first 60 characters of the first user message). */
|
||||
title: string
|
||||
/** Each item becomes one real user prompt followed by the mock provider's reply. */
|
||||
turns: readonly string[]
|
||||
turns: readonly (RealSessionTurn | string)[]
|
||||
}
|
||||
|
||||
export interface RealSession {
|
||||
|
|
@ -107,7 +114,13 @@ export class RealSessionBuilder {
|
|||
const runtimeId = requireString(created, 'session_id')
|
||||
const sessionId = requireString(created, 'stored_session_id')
|
||||
|
||||
for (const text of spec.turns) {
|
||||
for (const turn of spec.turns) {
|
||||
const { images = [], text } = typeof turn === 'string' ? { text: turn } : turn
|
||||
|
||||
for (const image of images) {
|
||||
await this.request('image.attach', { session_id: runtimeId, path: image })
|
||||
}
|
||||
|
||||
const completion = this.waitForEvent(
|
||||
frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -122,11 +122,16 @@ auxiliary:
|
|||
// A normal message crosses the tiny configured context budget. The mock
|
||||
// blocks only the resulting summary request, so these assertions run
|
||||
// during automatic compaction rather than a slash-command path.
|
||||
// The payload must cross threshold_tokens (22k) on its OWN weight
|
||||
// (~12k tokens) on top of the system prompt. Do not shrink it: at
|
||||
// repeat(500) the trigger only worked because the ambient system prompt
|
||||
// (skills index + tool schemas) happened to carry it over the line, and
|
||||
// a 160-token skills-index cleanup on main broke the test for a day.
|
||||
await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_ONE '.repeat(5))
|
||||
await waitForTranscript(page, MOCK_REPLY)
|
||||
await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_TWO '.repeat(5))
|
||||
await waitForTranscript(page, MOCK_REPLY)
|
||||
await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(500))
|
||||
await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(1500))
|
||||
await fixture.mock.waitForHeldCompletion()
|
||||
await expect(page.getByRole('status', { name: 'Summarizing thread' }).last()).toBeVisible()
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,12 @@ import {
|
|||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { SIDEBAR_CROSS_TEXTS, SIDEBAR_TEXTS, restartMockServer } from './mock-server'
|
||||
import {
|
||||
createBackgroundReleaseHandle,
|
||||
restartMockServer,
|
||||
SIDEBAR_CROSS_TEXTS,
|
||||
SIDEBAR_TEXTS,
|
||||
} from './mock-server'
|
||||
|
||||
/** Background-running dot aria-label (from i18n en.ts). */
|
||||
const BG_DOT_LABEL = 'Background task running'
|
||||
|
|
@ -176,21 +181,30 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
// Keeps the background process alive until this test releases it, so the
|
||||
// "still running after the turn finished" state can't expire on its own.
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
// Release first so the process exits even if the test failed early,
|
||||
// then drop the sentinel file.
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('background dot transitions to finished when viewing another session', async () => {
|
||||
const page = fixture.page
|
||||
|
||||
// Start a turn with a long background process (sleep 5).
|
||||
// Start a turn whose background process runs until we release it.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 10_000 })
|
||||
await composer.click()
|
||||
|
|
@ -212,8 +226,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
{ timeout: 90_000 },
|
||||
)
|
||||
|
||||
// The background dot should still be visible (sleep 5 hasn't finished yet,
|
||||
// or auto-dismiss hasn't fired).
|
||||
// The background dot must still be visible: the turn is done but the
|
||||
// process is held open by the sentinel, so this is a stable state rather
|
||||
// than a window we have to catch in time.
|
||||
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
|
||||
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
|
||||
|
||||
|
|
@ -225,8 +240,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
await page.locator('button:has-text("New session")').first().click()
|
||||
await page.waitForTimeout(2000)
|
||||
|
||||
// Now wait for the background process to finish (sleep 5 + auto-dismiss).
|
||||
// The session A dot should transition away from "background running".
|
||||
// Now let the background process finish. The session A dot should
|
||||
// transition away from "background running".
|
||||
bgRelease.release()
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),
|
||||
|
|
|
|||
|
|
@ -22,7 +22,12 @@ import {
|
|||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { SIDEBAR_CROSS_TEXTS, restartMockServer } from './mock-server'
|
||||
import {
|
||||
type BackgroundReleaseHandle,
|
||||
createBackgroundReleaseHandle,
|
||||
restartMockServer,
|
||||
SIDEBAR_CROSS_TEXTS,
|
||||
} from './mock-server'
|
||||
|
||||
/** Finished-unread dot aria-label. */
|
||||
const UNREAD_DOT_LABEL = 'Finished — unread'
|
||||
|
|
@ -34,7 +39,7 @@ function sessionRow(page: import('@playwright/test').Page, text: string) {
|
|||
return page.locator('[data-slot="sidebar"] button').filter({ hasText: text }).first()
|
||||
}
|
||||
|
||||
/** Common setup: start a turn with a sleep 5 bg process + subagent, wait for
|
||||
/** Common setup: start a turn with a held bg process + subagent, wait for
|
||||
* the turn to complete, then switch to a new session so the first session is
|
||||
* no longer $selectedStoredSessionId (required before opening a tile). */
|
||||
async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
||||
|
|
@ -67,7 +72,9 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
|||
{ timeout: 90_000 },
|
||||
)
|
||||
|
||||
// The background dot should still be visible (sleep 5 hasn't finished).
|
||||
// The background dot must still be visible: the turn is done but the
|
||||
// process is held open by the sentinel, so this is a stable state rather
|
||||
// than a window we have to catch in time.
|
||||
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
|
||||
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
|
||||
|
||||
|
|
@ -77,8 +84,12 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
|||
await page.waitForTimeout(2000)
|
||||
}
|
||||
|
||||
/** Wait for the background process to finish (sleep 5 + auto-dismiss). */
|
||||
async function waitForBgProcessToFinish(page: import('@playwright/test').Page) {
|
||||
/** Release the held background process, then wait for its dot to clear. */
|
||||
async function waitForBgProcessToFinish(
|
||||
page: import('@playwright/test').Page,
|
||||
release?: BackgroundReleaseHandle,
|
||||
) {
|
||||
release?.release()
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),
|
||||
|
|
@ -95,15 +106,20 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('session opened as a tab (not visible) correctly gets unread dot', async () => {
|
||||
|
|
@ -123,12 +139,21 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
|
|||
// Evidence: the tab is open but the session is not visible on screen.
|
||||
await page.screenshot({ path: 'test-results/tile-bug-tab-opened.png' })
|
||||
|
||||
await waitForBgProcessToFinish(page)
|
||||
await waitForBgProcessToFinish(page, bgRelease)
|
||||
|
||||
// A tab that's not the active tab IS hidden — the unread dot is correct.
|
||||
// The user is NOT looking at it, so marking it "unread" is right.
|
||||
const unreadCount = await page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count()
|
||||
expect(unreadCount, 'hidden tab should be marked unread').toBeGreaterThan(0)
|
||||
//
|
||||
// Poll rather than sampling once: "finished-unread" is an event-driven
|
||||
// transition that lands slightly after the running dot clears, and with a
|
||||
// released (rather than slowly-expiring) process there is no incidental
|
||||
// slack between the two. Same reasoning as the cross-session spec.
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count(),
|
||||
{ timeout: 30_000, message: 'hidden tab should be marked unread' },
|
||||
)
|
||||
.toBeGreaterThan(0)
|
||||
|
||||
await page.screenshot({ path: 'test-results/tile-bug-tab-unread-correct.png' })
|
||||
})
|
||||
|
|
@ -142,15 +167,20 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('session visible in a split tile does NOT get unread dot when it finishes', async () => {
|
||||
|
|
@ -196,7 +226,7 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
|
|||
// Evidence: the split tile is now open side-by-side — both sessions visible.
|
||||
await page.screenshot({ path: 'test-results/tile-bug-split-opened.png' })
|
||||
|
||||
await waitForBgProcessToFinish(page)
|
||||
await waitForBgProcessToFinish(page, bgRelease)
|
||||
|
||||
// THE BUG: the session visible in the split tile should NOT have the green
|
||||
// "finished unread" dot — the user is looking right at it. This assertion
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@
|
|||
* MutationObserver burst), but `$messages` was still set twice.
|
||||
*
|
||||
* The test passes when bursts === 1 AND reconciles === 0.
|
||||
* The sidebar "+" keeps the session warm in another tab. Its reactivation
|
||||
* follows the same contract: one additive paint and zero reconciles.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
|
@ -43,6 +45,11 @@ import { startMockServer } from './mock-server'
|
|||
import { RealSessionBuilder } from './real-session-builder'
|
||||
|
||||
const SESSION_TITLE = 'E2E Warm Resume Jitter Test'
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
const ALL_SURFACES = '[data-composer-target]'
|
||||
/** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */
|
||||
const MESSAGE_COUNT = 32
|
||||
/** Seeded PRNG so the generated content is deterministic across runs. */
|
||||
|
|
@ -154,15 +161,29 @@ test.afterAll(async () => {
|
|||
* after the initial paint, catching key-based reconciles that don't
|
||||
* add/remove nodes.
|
||||
*/
|
||||
async function installRenderCounter(page: import('@playwright/test').Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
async function installRenderCounter(
|
||||
page: import('@playwright/test').Page,
|
||||
transcriptText?: string,
|
||||
): Promise<void> {
|
||||
await page.evaluate(([visibleSelector, allSelector, expected]: [string, string, string | undefined]) => {
|
||||
const surfaces = [...document.querySelectorAll(expected ? allSelector : visibleSelector)]
|
||||
const surface = expected
|
||||
? surfaces.find(candidate =>
|
||||
(candidate.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
)
|
||||
: surfaces.at(-1)
|
||||
const viewport = surface?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) {
|
||||
throw new Error('Thread viewport not found before warm resume')
|
||||
}
|
||||
|
||||
const state = { bursts: 0, mutations: 0, timeline: [] as number[], stopped: false, reconciles: 0 }
|
||||
;(window as unknown as { __RENDER_COUNT__: typeof state }).__RENDER_COUNT__ = state
|
||||
const debugWindow = window as unknown as {
|
||||
__RENDER_COUNT__: typeof state
|
||||
__RENDER_VIEWPORT__: Element
|
||||
}
|
||||
debugWindow.__RENDER_COUNT__ = state
|
||||
debugWindow.__RENDER_VIEWPORT__ = viewport
|
||||
|
||||
let currentBatch = 0
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
|
@ -224,7 +245,53 @@ async function installRenderCounter(page: import('@playwright/test').Page): Prom
|
|||
hasMessages = true
|
||||
}
|
||||
}, 2)
|
||||
})
|
||||
}, [SURFACE, ALL_SURFACES, transcriptText] as [string, string, string | undefined])
|
||||
}
|
||||
|
||||
/** Wait until the ACTIVE chat surface's transcript contains `text`. */
|
||||
async function waitForActiveTranscriptText(
|
||||
page: import('@playwright/test').Page,
|
||||
text: string,
|
||||
timeout = 30_000,
|
||||
): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout },
|
||||
)
|
||||
}
|
||||
|
||||
async function waitForActiveTranscriptWithoutText(
|
||||
page: import('@playwright/test').Page,
|
||||
text: string,
|
||||
): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return !(active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
||||
/** Replace the primary surface with a draft while retaining its warm cache. */
|
||||
async function openFreshDraft(page: import('@playwright/test').Page, priorText: string): Promise<void> {
|
||||
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+N' : 'Control+N')
|
||||
await waitForActiveTranscriptWithoutText(page, priorText)
|
||||
}
|
||||
|
||||
/** Stack an empty tab while leaving the current transcript mounted and warm. */
|
||||
async function openNewSessionTab(page: import('@playwright/test').Page, priorText: string): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await waitForActiveTranscriptWithoutText(page, priorText)
|
||||
}
|
||||
|
||||
/** Stop the render counter and return the recorded burst/reconcile counts. */
|
||||
|
|
@ -245,6 +312,30 @@ async function readRenderCount(page: import('@playwright/test').Page): Promise<{
|
|||
})
|
||||
}
|
||||
|
||||
async function observedViewportIsActive(page: import('@playwright/test').Page): Promise<boolean> {
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const activeViewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
const observedViewport = (window as unknown as { __RENDER_VIEWPORT__?: Element }).__RENDER_VIEWPORT__
|
||||
|
||||
return activeViewport === observedViewport
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
/** A kept-alive tab must become visible without rebuilding its transcript. */
|
||||
function assertNoRepaint(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
|
||||
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
|
||||
expect(
|
||||
result!.bursts,
|
||||
`Expected no additive render bursts for a kept-alive tab, but got ${result!.bursts}. ` +
|
||||
`Mutation timeline: ${JSON.stringify(result!.timeline)}.`,
|
||||
).toBe(0)
|
||||
expect(
|
||||
result!.reconciles,
|
||||
`Expected no transcript reconciles for a kept-alive tab, but got ${result!.reconciles}.`,
|
||||
).toBe(0)
|
||||
}
|
||||
|
||||
/** Assert the render counter shows exactly one paint with no re-renders. */
|
||||
function assertNoJitter(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
|
||||
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
|
||||
|
|
@ -261,7 +352,7 @@ function assertNoJitter(result: { bursts: number; mutations: number; timeline: n
|
|||
).toBe(0)
|
||||
}
|
||||
|
||||
test('warm-route resume paints transcript exactly once (no jitter)', async ({}, testInfo) => {
|
||||
test('tab reactivation preserves the mounted transcript without repainting', async ({}, testInfo) => {
|
||||
const page = fixture!.page
|
||||
|
||||
// Wait for the sidebar to populate with our seeded session.
|
||||
|
|
@ -277,63 +368,29 @@ test('warm-route resume paints transcript exactly once (no jitter)', async ({},
|
|||
|
||||
// Wait for the transcript to appear — the first user message text confirms
|
||||
// the cold-path prefetch painted.
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
|
||||
// Wait for the session to fully settle (cold-path RPC + reconciliation).
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Step 2: Navigate away to a new chat — this does NOT evict the warm cache.
|
||||
const newSessionButton = page
|
||||
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
|
||||
.first()
|
||||
await newSessionButton.click()
|
||||
|
||||
// Wait for the new-chat empty state.
|
||||
await page.waitForFunction(
|
||||
(firstMsg: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return false
|
||||
const text = viewport.textContent ?? ''
|
||||
return !text.includes(firstMsg)
|
||||
},
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
// Stack a new tab, then observe the seeded transcript while it is hidden.
|
||||
// Installing after the switch isolates reactivation from mutations caused
|
||||
// while the new tab was being created.
|
||||
await openNewSessionTab(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(500)
|
||||
await installRenderCounter(page, FIRST_USER_MSG)
|
||||
|
||||
// Step 3: Install render counter, click back (warm resume), wait, assert.
|
||||
await installRenderCounter(page)
|
||||
// Step 3: Click back and verify the same kept-alive viewport becomes active
|
||||
// without rebuilding or reconciling its transcript.
|
||||
await sessionRow.click()
|
||||
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
|
||||
// Wait for at least 1 burst, then settle.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const w = window as unknown as { __RENDER_COUNT__?: { bursts: number } }
|
||||
return Boolean(w.__RENDER_COUNT__ && w.__RENDER_COUNT__.bursts > 0)
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 10_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(2_000)
|
||||
expect(await observedViewportIsActive(page), 'Reactivation should reveal the observed kept-alive viewport').toBe(true)
|
||||
|
||||
const result = await readRenderCount(page)
|
||||
await page.screenshot({ path: testInfo.outputPath('warm-resume-idle.png') })
|
||||
assertNoJitter(result)
|
||||
assertNoRepaint(result)
|
||||
})
|
||||
|
||||
test('warm-route resume after background inference completes (no jitter)', async ({}, testInfo) => {
|
||||
|
|
@ -354,13 +411,7 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
|
||||
// Step 1: Cold resume — populate the warm cache.
|
||||
await sessionRow.click()
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Step 2: Send a message — triggers inference via the mock server.
|
||||
|
|
@ -373,34 +424,15 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
// Wait for the mock response to appear in the transcript, confirming
|
||||
// the turn completed and message.complete fired (which updates the warm
|
||||
// cache via updateSessionState).
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return viewport?.textContent?.includes('mock inference server') ?? false
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, 'mock inference server', 60_000)
|
||||
// Extra settle for message.complete → updateSessionState → cache write.
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Verify the prompt was received by the mock server.
|
||||
expect(mock.receivedPrompts).toContain(PROMPT)
|
||||
|
||||
// Step 3: Navigate away — the warm cache retains the updated messages.
|
||||
const newSessionButton = page
|
||||
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
|
||||
.first()
|
||||
await newSessionButton.click()
|
||||
await page.waitForFunction(
|
||||
(prompt: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return false
|
||||
return !(viewport.textContent ?? '').includes(prompt)
|
||||
},
|
||||
PROMPT,
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
// Step 3: Replace the primary chat; the warm cache retains the updated messages.
|
||||
await openFreshDraft(page, PROMPT)
|
||||
await page.waitForTimeout(500)
|
||||
|
||||
// Step 4: Install render counter, click back (warm resume), wait, assert.
|
||||
|
|
@ -409,13 +441,7 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
|
||||
// Wait for the transcript to reappear — the warm cache should already
|
||||
// have the completed turn (updated by message.complete events).
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
|
||||
// Wait for at least 1 burst, then settle.
|
||||
await page.waitForFunction(
|
||||
|
|
|
|||
|
|
@ -68,6 +68,26 @@ test('buildDesktopBackendEnv extends PYTHONPATH and backend PATH together', () =
|
|||
assert.ok(env.PATH.includes('/opt/homebrew/bin'))
|
||||
})
|
||||
|
||||
test('buildDesktopBackendEnv forces PYTHONUTF8 unless the user set it explicitly', () => {
|
||||
const defaulted = buildDesktopBackendEnv({
|
||||
hermesHome: '/Users/test/.hermes',
|
||||
currentEnv: { PATH: '/usr/bin' },
|
||||
platform: 'darwin',
|
||||
pathModule: path.posix
|
||||
})
|
||||
|
||||
assert.equal(defaulted.PYTHONUTF8, '1')
|
||||
|
||||
const optedOut = buildDesktopBackendEnv({
|
||||
hermesHome: '/Users/test/.hermes',
|
||||
currentEnv: { PATH: '/usr/bin', PYTHONUTF8: '0' },
|
||||
platform: 'darwin',
|
||||
pathModule: path.posix
|
||||
})
|
||||
|
||||
assert.equal(optedOut.PYTHONUTF8, '0')
|
||||
})
|
||||
|
||||
test('normalizeHermesHomeRoot maps profile homes back to the global Hermes root', () => {
|
||||
assert.equal(
|
||||
normalizeHermesHomeRoot('/Users/test/.hermes/profiles/oracle', { pathModule: path.posix }),
|
||||
|
|
|
|||
|
|
@ -104,6 +104,13 @@ function buildDesktopBackendEnv({
|
|||
|
||||
return {
|
||||
PYTHONPATH: appendUniquePathEntries([...pythonPathEntries, currentPythonPath], { delimiter }),
|
||||
// Force PEP 540 UTF-8 mode in the spawned Python backend so its stdio and
|
||||
// subprocess defaults are UTF-8 even on non-UTF-8 Windows locales (GBK,
|
||||
// cp1252, ...). hermes_bootstrap sets this inside the child too, but only
|
||||
// after import — anything emitted earlier (interpreter startup errors,
|
||||
// pre-bootstrap tracebacks) still decodes with the locale default without
|
||||
// this. User's explicit setting wins. Re-port of PR #56499 (echoriver89).
|
||||
PYTHONUTF8: currentEnv?.PYTHONUTF8 ?? '1',
|
||||
[key]: buildDesktopBackendPath({
|
||||
hermesHome,
|
||||
venvRoot,
|
||||
|
|
|
|||
340
apps/desktop/electron/backend-health.test.ts
Normal file
340
apps/desktop/electron/backend-health.test.ts
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import {
|
||||
DEFAULT_HEALTH_PROBE_TIMEOUT_MS,
|
||||
isAuthRejectionError,
|
||||
isGatedMissingHealthError,
|
||||
isMissingHealthEndpointError,
|
||||
isReauthRequiredError,
|
||||
waitForHermesReady
|
||||
} from './backend-health'
|
||||
|
||||
const GATE_401 = '401: {"error":"unauthenticated","detail":"Unauthorized","reason":"no_cookie","login_url":"/login"}'
|
||||
|
||||
test('uses lightweight /api/health for current backends', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000/', {
|
||||
token: 'secret-token',
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
|
||||
return { ok: true }
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
throw new Error('status should not be called')
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [['public', 'http://127.0.0.1:9000/api/health']])
|
||||
})
|
||||
|
||||
test('falls back to /api/status only for old backends without /api/health', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000', {
|
||||
token: 'secret-token',
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
fetchJson: async (url, token) => {
|
||||
calls.push(['token', url, token ?? ''])
|
||||
|
||||
return { version: 'old' }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['public', 'http://127.0.0.1:9000/api/health'],
|
||||
['token', 'http://127.0.0.1:9000/api/status', 'secret-token']
|
||||
])
|
||||
})
|
||||
|
||||
test('does not fall back to heavyweight /api/status for transient health failures', async () => {
|
||||
const calls: string[][] = []
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error('Timed out connecting to Hermes backend after 15000ms')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
},
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 50,
|
||||
pollMs: 1
|
||||
}),
|
||||
/Timed out connecting/
|
||||
)
|
||||
|
||||
assert.ok(calls.length > 0)
|
||||
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
||||
})
|
||||
|
||||
test('probes health on a short timeout but leaves the legacy fallback its own', async () => {
|
||||
const timeouts: (number | undefined)[] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async (_url, options) => {
|
||||
timeouts.push(options?.timeoutMs)
|
||||
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
fetchJson: async (_url, _token, options) => {
|
||||
timeouts.push(options?.timeoutMs)
|
||||
|
||||
return { version: 'old' }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(timeouts, [DEFAULT_HEALTH_PROBE_TIMEOUT_MS, undefined])
|
||||
})
|
||||
|
||||
test('aborts as superseded when the bootstrap signal fires', async () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
signal: controller.signal,
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('should not probe after abort')
|
||||
},
|
||||
fetchJson: async () => {
|
||||
throw new Error('should not probe after abort')
|
||||
},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => error.kind === 'superseded'
|
||||
)
|
||||
})
|
||||
|
||||
test('recognizes missing-route shapes only', () => {
|
||||
assert.equal(isMissingHealthEndpointError(new Error('404: {"detail":"Not Found"}')), true)
|
||||
assert.equal(
|
||||
isMissingHealthEndpointError(
|
||||
new Error('Expected JSON from /api/health but got HTML. The endpoint is likely missing on the Hermes backend.')
|
||||
),
|
||||
true
|
||||
)
|
||||
assert.equal(isMissingHealthEndpointError(new Error('Timed out connecting to Hermes backend after 15000ms')), false)
|
||||
assert.equal(isMissingHealthEndpointError(new Error('500: boom')), false)
|
||||
})
|
||||
|
||||
// --- Gated backends that predate /api/health (release 0.19.0 and earlier) ---
|
||||
//
|
||||
// The dashboard auth gate runs ahead of the SPA catch-all, so on a backend
|
||||
// without the route an ANONYMOUS probe is rejected as unauthenticated rather
|
||||
// than 404 — verified against a simulated 0.19.0 backend:
|
||||
// credential-free: /api/health -> 401 no_cookie, /api/status -> 200
|
||||
// credentialed: /api/health -> 404, /api/sessions -> 200
|
||||
|
||||
test('anonymous gate-shaped 401 falls back to /api/status (backend predates /api/health)', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://192.168.1.132:9119', {
|
||||
token: null,
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error(GATE_401)
|
||||
},
|
||||
fetchJson: async (url, token) => {
|
||||
calls.push(['token', url, token == null ? 'null' : token])
|
||||
|
||||
return { version: '0.19.0', auth_required: true }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['public', 'http://192.168.1.132:9119/api/health'],
|
||||
['token', 'http://192.168.1.132:9119/api/status', 'null']
|
||||
])
|
||||
})
|
||||
|
||||
test('a credentialed 401 fails fast for reauth instead of reporting a dead session ready', async () => {
|
||||
// The regression a blanket 401->fallback introduces: /api/status is public,
|
||||
// so an expired session would answer 200 and boot would report "ready",
|
||||
// deferring the no_cookie to the first real API call.
|
||||
const calls: string[][] = []
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
token: 'session-token',
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('public probe must not be used when credentialed')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['status', url])
|
||||
|
||||
return { version: '0.19.0' }
|
||||
},
|
||||
probeHealth: async url => {
|
||||
calls.push(['probe', url])
|
||||
throw new Error(GATE_401)
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => {
|
||||
assert.equal(isReauthRequiredError(error), true)
|
||||
assert.equal(error.needsOauthLogin, true)
|
||||
assert.match(error.message, /remote gateway session has expired/i)
|
||||
|
||||
return true
|
||||
}
|
||||
)
|
||||
|
||||
// Fail fast: never reached the public /api/status leg.
|
||||
assert.deepEqual(calls, [['probe', 'https://gateway.example/api/health']])
|
||||
})
|
||||
|
||||
test('a credentialed 403 is also a terminal reauth failure', async () => {
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
fetchPublicJson: async () => ({}),
|
||||
fetchJson: async () => ({}),
|
||||
probeHealth: async () => {
|
||||
throw new Error('403: {"detail":"Forbidden"}')
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => isReauthRequiredError(error)
|
||||
)
|
||||
})
|
||||
|
||||
test('a credentialed probe still uses the 404 fallback for a genuinely missing route', async () => {
|
||||
// With credentials the gate lets the request through to the SPA catch-all,
|
||||
// so an old backend answers a real 404 — that must still fall back, not be
|
||||
// mistaken for a rejected session.
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('https://gateway.example', {
|
||||
token: 'session-token',
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('public probe must not be used when credentialed')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['status', url])
|
||||
|
||||
return { version: '0.19.0' }
|
||||
},
|
||||
probeHealth: async url => {
|
||||
calls.push(['probe', url])
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['probe', 'https://gateway.example/api/health'],
|
||||
['status', 'https://gateway.example/api/status']
|
||||
])
|
||||
})
|
||||
|
||||
test('a non-gate 401 keeps polling rather than skipping a misconfigured health route', async () => {
|
||||
const calls: string[][] = []
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error('401: {"detail":"Unauthorized"}')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
},
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 50,
|
||||
pollMs: 1
|
||||
}),
|
||||
/401: \{"detail":"Unauthorized"\}/
|
||||
)
|
||||
|
||||
assert.ok(calls.length > 0)
|
||||
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
||||
})
|
||||
|
||||
test('credentialed 5xx and 429 keep polling — only 401/403 are terminal', async () => {
|
||||
for (const transient of ['500: boom', '429: {"detail":"Too Many Requests"}']) {
|
||||
let attempts = 0
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
fetchPublicJson: async () => ({}),
|
||||
fetchJson: async () => ({}),
|
||||
probeHealth: async () => {
|
||||
attempts += 1
|
||||
throw new Error(transient)
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => isReauthRequiredError(error) === false
|
||||
)
|
||||
|
||||
assert.ok(attempts > 1, `${transient} should have retried, got ${attempts} attempt(s)`)
|
||||
}
|
||||
})
|
||||
|
||||
test('error-shape predicates', () => {
|
||||
assert.equal(isGatedMissingHealthError(new Error(GATE_401)), true)
|
||||
assert.equal(isGatedMissingHealthError(new Error('401: {"detail":"Unauthorized"}')), false)
|
||||
assert.equal(isGatedMissingHealthError(new Error('404: {"detail":"Not Found"}')), false)
|
||||
|
||||
assert.equal(isAuthRejectionError(new Error(GATE_401)), true)
|
||||
assert.equal(isAuthRejectionError(new Error('403: {"detail":"Forbidden"}')), true)
|
||||
assert.equal(isAuthRejectionError(new Error('404: {"detail":"Not Found"}')), false)
|
||||
assert.equal(isAuthRejectionError(new Error('429: slow down')), false)
|
||||
assert.equal(isAuthRejectionError(new Error('500: boom')), false)
|
||||
|
||||
// A gated 401 must NOT be conflated with a missing route by the 404 predicate.
|
||||
assert.equal(isMissingHealthEndpointError(new Error(GATE_401)), false)
|
||||
})
|
||||
169
apps/desktop/electron/backend-health.ts
Normal file
169
apps/desktop/electron/backend-health.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
export const DEFAULT_BACKEND_READY_TIMEOUT_MS = 45_000
|
||||
export const DEFAULT_BACKEND_READY_POLL_MS = 500
|
||||
// A cold backend can stall its event loop for tens of seconds while Windows
|
||||
// scans and byte-compiles the gateway import tree. At the default 15s socket
|
||||
// timeout only three probes fit in the budget; a short one keeps retrying
|
||||
// across the stall. Health only — the legacy /api/status fallback is genuinely
|
||||
// slow to answer and keeps the caller's default timeout.
|
||||
export const DEFAULT_HEALTH_PROBE_TIMEOUT_MS = 5_000
|
||||
|
||||
type FetchPublicJson = (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
type FetchJson = (url: string, token?: string | null, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
|
||||
export interface HermesReadyOptions {
|
||||
fetchPublicJson: FetchPublicJson
|
||||
fetchJson: FetchJson
|
||||
token?: string | null
|
||||
signal?: AbortSignal
|
||||
timeoutMs?: number
|
||||
pollMs?: number
|
||||
healthProbeTimeoutMs?: number
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
now?: () => number
|
||||
/**
|
||||
* Credentialed health probe. When supplied, readiness is probed with the
|
||||
* connection's own credentials instead of anonymously — which is what lets
|
||||
* a gated backend answer 404 for a genuinely missing /api/health, and what
|
||||
* makes a 401 from this probe mean "session rejected" rather than "route
|
||||
* behind a gate". Defaults to the credential-free `fetchPublicJson`.
|
||||
*/
|
||||
probeHealth?: (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
/**
|
||||
* Whether `probeHealth` actually presents credentials. Distinguishes the
|
||||
* two very different meanings of a 401 (see `waitForHermesReady`).
|
||||
*/
|
||||
probeIsCredentialed?: boolean
|
||||
}
|
||||
|
||||
export const REMOTE_SESSION_EXPIRED_MESSAGE =
|
||||
'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.'
|
||||
|
||||
export function isMissingHealthEndpointError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return /^404:/.test(message) || message.includes('endpoint is likely missing')
|
||||
}
|
||||
|
||||
/**
|
||||
* True for a hard auth rejection (401/403) as opposed to a transient failure.
|
||||
* Deliberately shape-based: 429 is a throttle and 5xx is a server fault, and
|
||||
* both must keep polling.
|
||||
*/
|
||||
export function isAuthRejectionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return /^40[13]:/.test(message)
|
||||
}
|
||||
|
||||
/**
|
||||
* True for an auth rejection carrying the dashboard gate's "no session at all"
|
||||
* shape. On a backend that predates `/api/health`, the gate runs ahead of the
|
||||
* SPA catch-all, so an unknown `/api/*` path is rejected as unauthenticated
|
||||
* instead of 404 — this is the signal that an ANONYMOUS probe cannot reach the
|
||||
* route, and the reason a credential-free 401 must fall back to `/api/status`
|
||||
* rather than be reported as a boot failure.
|
||||
*/
|
||||
export function isGatedMissingHealthError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return isAuthRejectionError(error) && message.includes('no_cookie')
|
||||
}
|
||||
|
||||
/** Tag a terminal reauth failure the main process latches and the overlay keys on. */
|
||||
export function makeReauthRequiredError(detail?: string): Error {
|
||||
const error = new Error(REMOTE_SESSION_EXPIRED_MESSAGE) as any
|
||||
error.needsOauthLogin = true
|
||||
error.isReauthRequired = true
|
||||
|
||||
if (detail) {
|
||||
error.detail = detail
|
||||
}
|
||||
|
||||
return error
|
||||
}
|
||||
|
||||
export function isReauthRequiredError(error: unknown): boolean {
|
||||
return Boolean((error as any)?.isReauthRequired)
|
||||
}
|
||||
|
||||
function supersededError() {
|
||||
const error: any = new Error('SSH bootstrap was superseded by newer connection settings.')
|
||||
error.kind = 'superseded'
|
||||
|
||||
return error
|
||||
}
|
||||
|
||||
export async function waitForHermesReady(baseUrl: string, options: HermesReadyOptions): Promise<void> {
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_BACKEND_READY_TIMEOUT_MS
|
||||
const pollMs = options.pollMs ?? DEFAULT_BACKEND_READY_POLL_MS
|
||||
const healthProbeTimeoutMs = options.healthProbeTimeoutMs ?? DEFAULT_HEALTH_PROBE_TIMEOUT_MS
|
||||
const now = options.now ?? Date.now
|
||||
const signal = options.signal
|
||||
|
||||
const sleep =
|
||||
options.sleep ??
|
||||
(ms =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms)
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timer)
|
||||
reject(supersededError())
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
}))
|
||||
|
||||
const base = baseUrl.replace(/\/+$/, '')
|
||||
const deadline = now() + timeoutMs
|
||||
const probeHealth = options.probeHealth ?? options.fetchPublicJson
|
||||
const probeIsCredentialed = Boolean(options.probeIsCredentialed)
|
||||
let lastError: unknown = null
|
||||
let useStatusFallback = false
|
||||
|
||||
while (now() < deadline) {
|
||||
if (signal?.aborted) {
|
||||
throw supersededError()
|
||||
}
|
||||
|
||||
try {
|
||||
if (useStatusFallback) {
|
||||
await options.fetchJson(`${base}/api/status`, options.token)
|
||||
} else {
|
||||
await probeHealth(`${base}/api/health`, { timeoutMs: healthProbeTimeoutMs })
|
||||
}
|
||||
|
||||
return
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
|
||||
// A confirmed 401/403 from a CREDENTIALED probe means the session was
|
||||
// rejected, not that the route is missing. Fail fast into a reauth
|
||||
// state: falling back to the public /api/status would answer 200 and
|
||||
// report a dead session as "ready", deferring the failure to the first
|
||||
// real API call. Applies to the /api/status leg too — it is routed
|
||||
// through the same credentials.
|
||||
if (probeIsCredentialed && isAuthRejectionError(error)) {
|
||||
throw makeReauthRequiredError(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
|
||||
// An explicitly missing route means the backend predates /api/health.
|
||||
// So does a gate-shaped 401 on an ANONYMOUS probe: the dashboard auth
|
||||
// gate runs ahead of the SPA catch-all, so a pre-/api/health backend
|
||||
// rejects the unknown path as unauthenticated instead of 404 and a
|
||||
// credential-free probe can never observe the 404. Timeouts, 5xx, 429,
|
||||
// and non-gate 401s keep polling health.
|
||||
if (!useStatusFallback && (isMissingHealthEndpointError(error) || isGatedMissingHealthError(error))) {
|
||||
useStatusFallback = true
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
await sleep(pollMs)
|
||||
}
|
||||
}
|
||||
|
||||
const detail = lastError instanceof Error ? lastError.message : 'timeout'
|
||||
throw new Error(`Hermes backend did not become ready: ${detail}`)
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
|
|||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { shouldLatchBackendStartFailure } from './backend-start-failure'
|
||||
import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure'
|
||||
|
||||
test('latches a LOCAL backend failure so the install-retry loop is broken', () => {
|
||||
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false }), true)
|
||||
|
|
@ -21,3 +21,32 @@ test('the two branches are mutually exclusive (a failure either latches or stays
|
|||
assert.equal(latched, !attemptedRemote)
|
||||
}
|
||||
})
|
||||
|
||||
test('latches a CONFIRMED remote reauth failure so the overlay stays clickable', () => {
|
||||
// Without this the non-latching remote path re-runs startHermes on every
|
||||
// getConnection/api call, re-emits running:true, and the overlay hides
|
||||
// itself — the "Sign in" button flickers away before it can be clicked.
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: true }), true)
|
||||
})
|
||||
|
||||
test('does not latch a transient remote failure as reauth', () => {
|
||||
// A mint timeout or a host unreachable across sleep must still self-heal.
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: false }), false)
|
||||
})
|
||||
|
||||
test('never latches a LOCAL failure as reauth (that is backendStartFailure job)', () => {
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: true }), false)
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: false }), false)
|
||||
})
|
||||
|
||||
test('the two latches never fire for the same failure', () => {
|
||||
// They are complementary, not overlapping: local failures latch via
|
||||
// backendStartFailure, confirmed remote reauth latches via its own flag.
|
||||
for (const attemptedRemote of [true, false]) {
|
||||
for (const isReauth of [true, false]) {
|
||||
const start = shouldLatchBackendStartFailure({ attemptedRemote })
|
||||
const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth })
|
||||
assert.ok(!(start && reauth), `both latched for remote=${attemptedRemote} reauth=${isReauth}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -39,3 +39,34 @@ export interface BackendStartFailureContext {
|
|||
export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean {
|
||||
return !context.attemptedRemote
|
||||
}
|
||||
|
||||
export interface RemoteReauthFailureContext {
|
||||
/** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */
|
||||
attemptedRemote: boolean
|
||||
/**
|
||||
* True when the failure was a CONFIRMED auth rejection (a credentialed
|
||||
* probe got 401/403), not a transient connectivity fault.
|
||||
*/
|
||||
isReauth: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a failed remote boot should latch as a reauth failure.
|
||||
*
|
||||
* This is the deliberate counterpart to `shouldLatchBackendStartFailure`,
|
||||
* which never latches a remote failure because remote faults are usually
|
||||
* transient and must stay retryable. A *confirmed* reauth rejection is the
|
||||
* exception: it cannot self-heal, because nothing will change until the user
|
||||
* signs in again.
|
||||
*
|
||||
* Without a latch, the non-latching remote path actively prevents recovery.
|
||||
* Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits
|
||||
* `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error)
|
||||
* && !boot.running`) hides itself — so the "Sign in" button flickers out from
|
||||
* under the user before they can click it. Latching holds the overlay still
|
||||
* and clickable. Cleared on every recovery path (reset, repair, apply-config,
|
||||
* and a confirmed sign-in) so a fresh session boots normally.
|
||||
*/
|
||||
export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean {
|
||||
return context.attemptedRemote && context.isReauth
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
async function applyConnectionChange({
|
||||
cancelAndWait,
|
||||
isPrimary,
|
||||
rehomePrimary = null,
|
||||
scope,
|
||||
sendApplied,
|
||||
stopPool,
|
||||
|
|
@ -16,6 +17,12 @@ async function applyConnectionChange({
|
|||
return
|
||||
}
|
||||
|
||||
if (rehomePrimary) {
|
||||
await rehomePrimary()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await teardownPrimary()
|
||||
sendApplied()
|
||||
}
|
||||
|
|
|
|||
54
apps/desktop/electron/find-git-bash.test.ts
Normal file
54
apps/desktop/electron/find-git-bash.test.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { findGitBash } from './find-git-bash'
|
||||
|
||||
const yes = () => true
|
||||
const no = () => false
|
||||
|
||||
test('HERMES_GIT_BASH_PATH override takes precedence', () => {
|
||||
const result = findGitBash({
|
||||
isWindows: true,
|
||||
env: { HERMES_GIT_BASH_PATH: 'D:\\CustomGit\\bin\\bash.exe' },
|
||||
fileExists: yes,
|
||||
findOnPath: () => null
|
||||
})
|
||||
|
||||
assert.equal(result, 'D:\\CustomGit\\bin\\bash.exe')
|
||||
})
|
||||
|
||||
test('HERMES_GIT_BASH_PATH invalid path falls through to candidates', () => {
|
||||
const env = {
|
||||
HERMES_GIT_BASH_PATH: 'X:\\Missing\\bash.exe',
|
||||
LOCALAPPDATA: 'C:\\Users\\test\\AppData\\Local',
|
||||
ProgramFiles: 'C:\\Program Files',
|
||||
'ProgramFiles(x86)': 'C:\\Program Files (x86)'
|
||||
}
|
||||
|
||||
const fileExists = (p: string) => p !== 'X:\\Missing\\bash.exe' && p.includes('Program Files\\Git\\bin\\bash.exe')
|
||||
const result = findGitBash({ isWindows: true, env, fileExists, findOnPath: () => null })
|
||||
assert.equal(result, 'C:\\Program Files\\Git\\bin\\bash.exe')
|
||||
})
|
||||
|
||||
test('HERMES_GIT_BASH_PATH empty string is ignored', () => {
|
||||
const result = findGitBash({
|
||||
isWindows: true,
|
||||
env: { HERMES_GIT_BASH_PATH: '', LOCALAPPDATA: '' },
|
||||
fileExists: no,
|
||||
findOnPath: () => 'C:\\msys64\\usr\\bin\\bash.exe'
|
||||
})
|
||||
|
||||
assert.equal(result, 'C:\\msys64\\usr\\bin\\bash.exe')
|
||||
})
|
||||
|
||||
test('non-Windows uses findOnPath', () => {
|
||||
const result = findGitBash({
|
||||
isWindows: false,
|
||||
env: {},
|
||||
fileExists: no,
|
||||
findOnPath: () => '/usr/bin/bash'
|
||||
})
|
||||
|
||||
assert.equal(result, '/usr/bin/bash')
|
||||
})
|
||||
67
apps/desktop/electron/find-git-bash.ts
Normal file
67
apps/desktop/electron/find-git-bash.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import path from 'node:path'
|
||||
|
||||
export interface GitBashOptions {
|
||||
isWindows: boolean
|
||||
env: Record<string, string | undefined>
|
||||
fileExists: (filePath: string) => boolean
|
||||
findOnPath?: (command: string) => string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate bash.exe on Windows.
|
||||
* Resolution order (first match wins):
|
||||
* 1. HERMES_GIT_BASH_PATH env var override
|
||||
* 2. PortableGit under %LOCALAPPDATA%\hermes\git\ (install.ps1)
|
||||
* 3. Standard Git for Windows install locations
|
||||
* 4. %LOCALAPPDATA%\Programs\Git\ (user-scoped)
|
||||
* 5. bash on PATH
|
||||
*/
|
||||
export function findGitBash(opts: GitBashOptions): string | null {
|
||||
const { isWindows, env, fileExists, findOnPath } = opts
|
||||
|
||||
if (!isWindows) {
|
||||
return findOnPath ? findOnPath('bash') : null
|
||||
}
|
||||
|
||||
// Respect HERMES_GIT_BASH_PATH if set (mirrors tools/environments/local.py:_find_bash).
|
||||
const gitBashPath = env.HERMES_GIT_BASH_PATH
|
||||
|
||||
if (gitBashPath && fileExists(gitBashPath)) {
|
||||
return gitBashPath
|
||||
}
|
||||
|
||||
const localAppData = env.LOCALAPPDATA || ''
|
||||
const candidates: string[] = []
|
||||
|
||||
// Candidate paths are Windows paths regardless of host platform (tests run
|
||||
// on POSIX CI hosts too), so join with win32 semantics explicitly.
|
||||
const joinWin = path.win32.join
|
||||
|
||||
if (localAppData) {
|
||||
candidates.push(joinWin(localAppData, 'hermes', 'git', 'bin', 'bash.exe'))
|
||||
candidates.push(joinWin(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe'))
|
||||
}
|
||||
|
||||
candidates.push(joinWin(env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe'))
|
||||
candidates.push(joinWin(env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe'))
|
||||
|
||||
if (localAppData) {
|
||||
candidates.push(joinWin(localAppData, 'Programs', 'Git', 'bin', 'bash.exe'))
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (fileExists(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (findOnPath) {
|
||||
const onPath = findOnPath('bash')
|
||||
|
||||
if (onPath) {
|
||||
return onPath
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
133
apps/desktop/electron/first-run-setup-gate.test.ts
Normal file
133
apps/desktop/electron/first-run-setup-gate.test.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
function delay(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function settledState(promise: Promise<unknown>) {
|
||||
return Promise.race([promise.then(() => 'resolved'), delay(10).then(() => 'pending')])
|
||||
}
|
||||
|
||||
test('first-run setup gate skips non-bootstrap backends', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
await gate.wait({ kind: 'remote' })
|
||||
await gate.wait(null)
|
||||
|
||||
assert.deepEqual(prompts, [])
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
})
|
||||
|
||||
test('first-run setup gate prompts once for concurrent waits', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const first = gate.wait(bootstrapBackend)
|
||||
const second = gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.hasWaiter(), true)
|
||||
assert.equal(prompts.length, 1)
|
||||
assert.equal(await settledState(first), 'pending')
|
||||
|
||||
gate.continueLocal()
|
||||
|
||||
assert.deepEqual(await Promise.all([first, second]), ['continue-local', 'continue-local'])
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
|
||||
test('continueLocal keeps the setup choice visible until bootstrap owns the overlay', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.continueLocal()
|
||||
|
||||
assert.equal(await pending, 'continue-local')
|
||||
assert.equal(hidden, 0)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
|
||||
test('retry reset preserves the local install confirmation', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
gate.resetForRetry()
|
||||
await gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
assert.equal(prompts.length, 1)
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
})
|
||||
|
||||
test('retry reset explicitly settles an active waiter without allowing local bootstrap', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.resetForRetry()
|
||||
|
||||
assert.equal(await pending, 'reset')
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
})
|
||||
|
||||
test('repair reset clears the local install confirmation and shows the gate again', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
gate.resetForRepair()
|
||||
const next = gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
assert.equal(prompts.length, 2)
|
||||
assert.equal(gate.hasWaiter(), true)
|
||||
|
||||
gate.continueLocal()
|
||||
await next
|
||||
})
|
||||
|
||||
test('remote apply settles the gated boot for remote re-resolution and hides the choice', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
const resumedWaiter = gate.abandonForRemoteApply()
|
||||
|
||||
assert.equal(resumedWaiter, true)
|
||||
assert.equal(hidden, 1)
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
assert.equal(await pending, 'remote-applied')
|
||||
})
|
||||
|
||||
test('remote apply without a waiter has no first-run side effects', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
assert.equal(gate.abandonForRemoteApply(), false)
|
||||
assert.equal(hidden, 0)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
146
apps/desktop/electron/first-run-setup-gate.ts
Normal file
146
apps/desktop/electron/first-run-setup-gate.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
interface FirstRunSetupBackend {
|
||||
activeRoot?: string
|
||||
kind?: string
|
||||
platform?: string
|
||||
}
|
||||
|
||||
interface FirstRunSetupGateOptions {
|
||||
hideChoice?: () => void
|
||||
log?: (message: string) => void
|
||||
onStuck?: (backend: FirstRunSetupBackend, stuckAfterMs: number) => void
|
||||
promptChoice?: (backend: FirstRunSetupBackend) => void
|
||||
stuckAfterMs?: number
|
||||
}
|
||||
|
||||
export type FirstRunSetupDecision = 'continue-local' | 'remote-applied' | 'reset'
|
||||
|
||||
export function createFirstRunSetupGate({
|
||||
hideChoice,
|
||||
log,
|
||||
onStuck,
|
||||
promptChoice,
|
||||
stuckAfterMs = 120000
|
||||
}: FirstRunSetupGateOptions = {}) {
|
||||
let localBootstrapConfirmed = false
|
||||
|
||||
let waiter: {
|
||||
promise: Promise<FirstRunSetupDecision>
|
||||
resolve: (decision: FirstRunSetupDecision) => void
|
||||
} | null = null
|
||||
|
||||
let stuckTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const clearStuckTimer = () => {
|
||||
if (stuckTimer) {
|
||||
clearTimeout(stuckTimer)
|
||||
stuckTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
const armStuckTimer = (backend: FirstRunSetupBackend) => {
|
||||
clearStuckTimer()
|
||||
|
||||
if (!Number.isFinite(stuckAfterMs) || stuckAfterMs <= 0 || typeof log !== 'function') {
|
||||
return
|
||||
}
|
||||
|
||||
stuckTimer = setTimeout(() => {
|
||||
onStuck?.(backend, stuckAfterMs)
|
||||
log(
|
||||
`[bootstrap] still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)}s ` +
|
||||
`(platform=${backend?.platform || 'unknown'})`
|
||||
)
|
||||
}, stuckAfterMs)
|
||||
|
||||
if (typeof stuckTimer.unref === 'function') {
|
||||
stuckTimer.unref()
|
||||
}
|
||||
}
|
||||
|
||||
const shouldGate = (backend?: FirstRunSetupBackend | null) =>
|
||||
Boolean(backend && backend.kind === 'bootstrap-needed' && !localBootstrapConfirmed)
|
||||
|
||||
const wait = async (backend?: FirstRunSetupBackend | null) => {
|
||||
if (!shouldGate(backend)) {
|
||||
return 'continue-local' as const
|
||||
}
|
||||
|
||||
if (waiter) {
|
||||
return waiter.promise
|
||||
}
|
||||
|
||||
promptChoice?.(backend)
|
||||
armStuckTimer(backend)
|
||||
|
||||
let resolveWaiter: (decision: FirstRunSetupDecision) => void = () => {}
|
||||
|
||||
const promise = new Promise<FirstRunSetupDecision>(resolve => {
|
||||
resolveWaiter = resolve
|
||||
})
|
||||
|
||||
waiter = { promise, resolve: resolveWaiter }
|
||||
|
||||
return promise
|
||||
}
|
||||
|
||||
const settleWaiter = (decision: FirstRunSetupDecision) => {
|
||||
clearStuckTimer()
|
||||
|
||||
if (!waiter) {
|
||||
return false
|
||||
}
|
||||
|
||||
const activeWaiter = waiter
|
||||
waiter = null
|
||||
activeWaiter.resolve(decision)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const continueLocal = () => {
|
||||
localBootstrapConfirmed = true
|
||||
settleWaiter('continue-local')
|
||||
}
|
||||
|
||||
const resetForRetry = () => {
|
||||
// Reset paths are followed by a renderer reload / fresh startHermes() call.
|
||||
// Settle the old boot explicitly so it cannot fall through into local
|
||||
// bootstrap and cannot leak a forever-pending connection promise.
|
||||
settleWaiter('reset')
|
||||
}
|
||||
|
||||
const resetForRepair = () => {
|
||||
resetForRetry()
|
||||
localBootstrapConfirmed = false
|
||||
}
|
||||
|
||||
const abandonForRemoteApply = () => {
|
||||
// Resume the gated startHermes() with an explicit remote decision. The
|
||||
// caller re-resolves the newly-persisted remote config instead of falling
|
||||
// through into local bootstrap or leaking the original connection promise.
|
||||
const resumedWaiter = settleWaiter('remote-applied')
|
||||
|
||||
if (!resumedWaiter) {
|
||||
return false
|
||||
}
|
||||
|
||||
localBootstrapConfirmed = false
|
||||
hideChoice?.()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const isLocalBootstrapConfirmed = () => localBootstrapConfirmed
|
||||
const hasWaiter = () => Boolean(waiter)
|
||||
|
||||
return {
|
||||
abandonForRemoteApply,
|
||||
continueLocal,
|
||||
hasWaiter,
|
||||
isLocalBootstrapConfirmed,
|
||||
resetForRepair,
|
||||
resetForRetry,
|
||||
shouldGate,
|
||||
wait
|
||||
}
|
||||
}
|
||||
119
apps/desktop/electron/first-run-setup-main-process.test.ts
Normal file
119
apps/desktop/electron/first-run-setup-main-process.test.ts
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test, vi } from 'vitest'
|
||||
|
||||
import { applyConnectionChange } from './connection-apply'
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
import { runPrimaryBackendStartup } from './primary-backend-startup'
|
||||
import { rehomePrimaryConnection } from './primary-connection-rehome'
|
||||
|
||||
test('a first-run bootstrap-needed remote apply connects without ensuring or bootstrapping locally', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
const candidateRemote = {
|
||||
authMode: 'token',
|
||||
baseUrl: 'https://gateway.example.com/hermes',
|
||||
source: 'settings',
|
||||
token: 'secret',
|
||||
wsUrl: 'wss://gateway.example.com/hermes/api/ws?token=secret'
|
||||
}
|
||||
|
||||
let savedRemote: typeof candidateRemote | null = null
|
||||
|
||||
const resolveRemote = vi.fn(async () => savedRemote)
|
||||
const connectRemote = vi.fn(async remote => ({ ...remote, mode: 'remote' as const }))
|
||||
const runBootstrap = vi.fn()
|
||||
|
||||
const ensureLocalRuntime = vi.fn(async backend => {
|
||||
await runBootstrap()
|
||||
|
||||
return { ...backend, command: 'hermes' }
|
||||
})
|
||||
|
||||
const teardownPrimaryBackend = vi.fn(async () => {})
|
||||
const cancelSshBootstrap = vi.fn(async () => {})
|
||||
const teardownSsh = vi.fn(async () => {})
|
||||
const clearLocalBootstrapFailure = vi.fn()
|
||||
const notifyConnectionApplied = vi.fn()
|
||||
const waitForLocalStart = vi.fn(async () => {})
|
||||
const prepareLocalBackend = vi.fn(async () => bootstrapBackend)
|
||||
|
||||
const pendingConnection = runPrimaryBackendStartup({
|
||||
connectRemote,
|
||||
ensureLocalRuntime,
|
||||
prepareLocalBackend,
|
||||
resolveRemote,
|
||||
waitForDecision: gate.wait,
|
||||
waitForLocalStart
|
||||
})
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
|
||||
// Mirrors the IPC handler's production ordering: persist the tested config,
|
||||
// then re-home. The pending start must re-resolve this saved value.
|
||||
savedRemote = candidateRemote
|
||||
|
||||
await applyConnectionChange({
|
||||
cancelAndWait: cancelSshBootstrap,
|
||||
isPrimary: true,
|
||||
rehomePrimary: () =>
|
||||
rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode: 'remote',
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote: gate.abandonForRemoteApply,
|
||||
teardownPrimaryBackend
|
||||
}),
|
||||
scope: '',
|
||||
sendApplied: notifyConnectionApplied,
|
||||
stopPool: vi.fn(),
|
||||
teardownPrimary: teardownPrimaryBackend,
|
||||
teardownSsh
|
||||
})
|
||||
|
||||
assert.deepEqual(await pendingConnection, {
|
||||
kind: 'remote',
|
||||
connection: { ...candidateRemote, mode: 'remote' }
|
||||
})
|
||||
assert.deepEqual(resolveRemote.mock.calls, [[], []])
|
||||
assert.deepEqual(connectRemote.mock.calls, [[candidateRemote]])
|
||||
assert.deepEqual(waitForLocalStart.mock.calls, [[]])
|
||||
assert.deepEqual(prepareLocalBackend.mock.calls, [[]])
|
||||
assert.equal(ensureLocalRuntime.mock.calls.length, 0)
|
||||
assert.equal(runBootstrap.mock.calls.length, 0)
|
||||
assert.deepEqual(cancelSshBootstrap.mock.calls, [['']])
|
||||
assert.deepEqual(teardownSsh.mock.calls, [['']])
|
||||
assert.equal(teardownPrimaryBackend.mock.calls.length, 0)
|
||||
assert.equal(clearLocalBootstrapFailure.mock.calls.length, 1)
|
||||
assert.equal(notifyConnectionApplied.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('a primary apply without an active first-run gate tears down before reconnect notification', async () => {
|
||||
const order: string[] = []
|
||||
const clearLocalBootstrapFailure = vi.fn(() => order.push('clear-failure'))
|
||||
|
||||
const teardownPrimaryBackend = vi.fn(async () => {
|
||||
order.push('teardown')
|
||||
})
|
||||
|
||||
const notifyConnectionApplied = vi.fn(() => order.push('notify'))
|
||||
|
||||
assert.deepEqual(
|
||||
await rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode: 'remote',
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote: () => false,
|
||||
teardownPrimaryBackend
|
||||
}),
|
||||
{ resumedFirstRunRemote: false }
|
||||
)
|
||||
assert.deepEqual(teardownPrimaryBackend.mock.calls, [[{ soft: true }]])
|
||||
assert.deepEqual(order, ['clear-failure', 'teardown', 'notify'])
|
||||
})
|
||||
|
|
@ -34,9 +34,10 @@ import { stopBackendChild as stopBackendChildImpl } from './backend-child'
|
|||
import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command'
|
||||
import { createBackendConnectionState } from './backend-connection-state'
|
||||
import { buildDesktopBackendEnv, normalizeHermesHomeRoot } from './backend-env'
|
||||
import { isReauthRequiredError, waitForHermesReady } from './backend-health'
|
||||
import { canImportHermesCli, shouldTrustHermesOverride, verifyHermesCli } from './backend-probes'
|
||||
import { waitForDashboardPortAnnouncement } from './backend-ready'
|
||||
import { shouldLatchBackendStartFailure } from './backend-start-failure'
|
||||
import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure'
|
||||
import { detectRemoteDisplay, isWindowsBinaryPathInWsl, isWslEnvironment } from './bootstrap-platform'
|
||||
import { runBootstrap } from './bootstrap-runner'
|
||||
import { applyConnectionChange, resolveTerminalConnection } from './connection-apply'
|
||||
|
|
@ -78,6 +79,8 @@ import {
|
|||
} from './desktop-uninstall'
|
||||
import { installEmbedReferer } from './embed-referer'
|
||||
import { createEventDeduper } from './event-dedupe'
|
||||
import { findGitBash as _findGitBash } from './find-git-bash'
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
import { readDirForIpc } from './fs-read-dir'
|
||||
import { probeGatewayWebSocket } from './gateway-ws-probe'
|
||||
import { scanGitRepos } from './git-repo-scan'
|
||||
|
|
@ -116,7 +119,13 @@ import {
|
|||
} from './hardening'
|
||||
import { createLinkTitleWindow, guardLinkTitleSession, readLinkTitleWindowTitle } from './link-title-window'
|
||||
import { ensureMainWindow } from './main-window-lifecycle'
|
||||
import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions'
|
||||
import {
|
||||
oauthGuardMayHardFail,
|
||||
oauthSessionIsLive,
|
||||
resolveJsonBody,
|
||||
resolveOauthRestAuth,
|
||||
resolveReadinessProbeAuth
|
||||
} from './native-auth-decisions'
|
||||
import {
|
||||
nativeRefreshUrl,
|
||||
type NativeTokenSet,
|
||||
|
|
@ -127,6 +136,8 @@ import {
|
|||
import { runNativeLogin } from './native-oauth-login'
|
||||
import { serializeJsonBody, setJsonRequestHeaders } from './oauth-net-request'
|
||||
import { createKeepAwake } from './power-save'
|
||||
import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup'
|
||||
import { rehomePrimaryConnection } from './primary-connection-rehome'
|
||||
import { decideProfileDeleteAction, profileNameFromDeleteRequest, resolveRouteProfile } from './profile-delete-routing'
|
||||
import * as remoteLifecycle from './remote-lifecycle'
|
||||
import { RemoteLivenessTracker, RemoteRevalidationCoordinator, revalidateRemoteConnection } from './remote-liveness'
|
||||
|
|
@ -1011,6 +1022,13 @@ let bootstrapFailure = null
|
|||
// Latched non-bootstrap backend spawn failure — stops getConnection() from
|
||||
// respawning hermes serve backend children in a tight loop while boot is broken.
|
||||
let backendStartFailure = null
|
||||
// Latched CONFIRMED remote reauth failure. Remote failures deliberately do not
|
||||
// latch via backendStartFailure (they're usually transient and must stay
|
||||
// retryable), but a rejected session cannot self-heal — and the non-latching
|
||||
// path actively breaks recovery: each retry re-emits running:true and hides
|
||||
// the boot-failure overlay, so the "Sign in" button flickers away before it
|
||||
// can be clicked. Cleared on every recovery path and on a confirmed sign-in.
|
||||
let remoteReauthFailure = null
|
||||
// Active first-launch install, so the renderer's Cancel button (and app quit)
|
||||
// can abort the in-flight install.sh/ps1 instead of leaving it running.
|
||||
let bootstrapAbortController = null
|
||||
|
|
@ -1398,13 +1416,17 @@ let bootstrapState = {
|
|||
log: [],
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
setupChoice: null,
|
||||
unsupportedPlatform: null
|
||||
}
|
||||
|
||||
let firstRunSetupGate = null
|
||||
|
||||
function broadcastBootstrapEvent(ev) {
|
||||
if (ev.type === 'manifest') {
|
||||
bootstrapState.manifest = ev
|
||||
bootstrapState.active = true
|
||||
bootstrapState.setupChoice = null
|
||||
bootstrapState.startedAt = bootstrapState.startedAt || Date.now()
|
||||
bootstrapState.stages = {}
|
||||
|
||||
|
|
@ -1432,14 +1454,30 @@ function broadcastBootstrapEvent(ev) {
|
|||
} else if (ev.type === 'failed') {
|
||||
bootstrapState.active = false
|
||||
bootstrapState.error = ev.error || 'unknown error'
|
||||
bootstrapState.setupChoice = null
|
||||
} else if (ev.type === 'unsupported-platform') {
|
||||
bootstrapState.active = false
|
||||
bootstrapState.setupChoice = null
|
||||
bootstrapState.unsupportedPlatform = {
|
||||
platform: ev.platform,
|
||||
activeRoot: ev.activeRoot,
|
||||
installCommand: ev.installCommand,
|
||||
docsUrl: ev.docsUrl
|
||||
}
|
||||
} else if (ev.type === 'setup-choice') {
|
||||
bootstrapState.active = false
|
||||
bootstrapState.error = null
|
||||
bootstrapState.manifest = null
|
||||
bootstrapState.stages = {}
|
||||
bootstrapState.setupChoice = ev.active
|
||||
? {
|
||||
platform: ev.platform,
|
||||
activeRoot: ev.activeRoot
|
||||
}
|
||||
: null
|
||||
bootstrapState.unsupportedPlatform = null
|
||||
} else if (ev.type === 'dismissed') {
|
||||
resetBootstrapSnapshot()
|
||||
}
|
||||
|
||||
if (!mainWindow || mainWindow.isDestroyed()) {
|
||||
|
|
@ -1459,6 +1497,100 @@ function getBootstrapState() {
|
|||
return bootstrapState
|
||||
}
|
||||
|
||||
function resetBootstrapSnapshot() {
|
||||
bootstrapState = {
|
||||
active: false,
|
||||
manifest: null,
|
||||
stages: {},
|
||||
error: null,
|
||||
log: [],
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
setupChoice: null,
|
||||
unsupportedPlatform: null
|
||||
}
|
||||
}
|
||||
|
||||
function promptFirstRunSetupChoice(backend) {
|
||||
broadcastBootstrapEvent({
|
||||
type: 'setup-choice',
|
||||
active: true,
|
||||
platform: backend.platform || process.platform,
|
||||
activeRoot: backend.activeRoot || ACTIVE_HERMES_ROOT
|
||||
})
|
||||
}
|
||||
|
||||
function hideFirstRunSetupChoice() {
|
||||
if (bootstrapState.setupChoice) {
|
||||
broadcastBootstrapEvent({ type: 'setup-choice', active: false })
|
||||
}
|
||||
}
|
||||
|
||||
function getFirstRunSetupGate() {
|
||||
if (!firstRunSetupGate) {
|
||||
firstRunSetupGate = createFirstRunSetupGate({
|
||||
hideChoice: hideFirstRunSetupChoice,
|
||||
log: rememberLog,
|
||||
onStuck: (_backend, stuckAfterMs) => {
|
||||
updateBootProgress(
|
||||
{
|
||||
error: null,
|
||||
message: `Still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)} seconds`,
|
||||
phase: 'bootstrap.choice',
|
||||
progress: 12,
|
||||
running: true
|
||||
},
|
||||
{ allowDecrease: true }
|
||||
)
|
||||
},
|
||||
promptChoice: promptFirstRunSetupChoice
|
||||
})
|
||||
}
|
||||
|
||||
return firstRunSetupGate
|
||||
}
|
||||
|
||||
async function waitForFirstRunSetupChoice(backend) {
|
||||
const gate = getFirstRunSetupGate()
|
||||
|
||||
if (!gate.shouldGate(backend)) {
|
||||
return 'continue-local'
|
||||
}
|
||||
|
||||
updateBootProgress(
|
||||
{
|
||||
error: null,
|
||||
message: 'Waiting for first-run setup choice',
|
||||
phase: 'bootstrap.choice',
|
||||
progress: 12,
|
||||
running: true
|
||||
},
|
||||
{ allowDecrease: true }
|
||||
)
|
||||
|
||||
return gate.wait(backend)
|
||||
}
|
||||
|
||||
function continueFirstRunLocalBootstrap() {
|
||||
getFirstRunSetupGate().continueLocal()
|
||||
}
|
||||
|
||||
function abandonFirstRunSetupChoiceForRemoteApply() {
|
||||
const gate = getFirstRunSetupGate()
|
||||
|
||||
if (!gate.hasWaiter()) {
|
||||
return false
|
||||
}
|
||||
|
||||
const resumedGatedConnection = gate.abandonForRemoteApply()
|
||||
|
||||
if (resumedGatedConnection) {
|
||||
broadcastBootstrapEvent({ type: 'dismissed' })
|
||||
}
|
||||
|
||||
return resumedGatedConnection
|
||||
}
|
||||
|
||||
function updateBootProgress(update, options: { allowDecrease?: boolean } = {}) {
|
||||
const nextProgressRaw =
|
||||
typeof update.progress === 'number' ? clampBootProgress(update.progress) : bootProgressState.progress
|
||||
|
|
@ -1915,48 +2047,16 @@ function findSystemPython() {
|
|||
return null
|
||||
}
|
||||
|
||||
// findGitBash — locate bash.exe on Windows. Hermes' terminal tool requires
|
||||
// bash (POSIX shell), and on Windows that's almost always Git for Windows'
|
||||
// bundled Git Bash. We check the same set of locations tools/environments/
|
||||
// local.py:_find_bash() checks at runtime, so a positive result here means
|
||||
// the agent will be able to start a terminal too.
|
||||
//
|
||||
// On non-Windows hosts bash is part of the OS and this just returns the
|
||||
// first bash on PATH.
|
||||
// findGitBash — locate bash.exe on Windows. Resolves HERMES_GIT_BASH_PATH
|
||||
// first (mirrors tools/environments/local.py:_find_bash), then PortableGit,
|
||||
// standard install locations, and finally PATH.
|
||||
function findGitBash() {
|
||||
if (!IS_WINDOWS) {
|
||||
return findOnPath('bash')
|
||||
}
|
||||
|
||||
// install.ps1 drops PortableGit at %LOCALAPPDATA%\hermes\git\... — checked
|
||||
// first so users who installed via install.ps1 are detected before we
|
||||
// start probing system-wide locations.
|
||||
const localAppData = process.env.LOCALAPPDATA || ''
|
||||
const candidates = []
|
||||
|
||||
if (localAppData) {
|
||||
candidates.push(path.join(localAppData, 'hermes', 'git', 'bin', 'bash.exe'))
|
||||
candidates.push(path.join(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe'))
|
||||
}
|
||||
|
||||
// Standard Git for Windows install locations.
|
||||
candidates.push(path.join(process.env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe'))
|
||||
candidates.push(path.join(process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe'))
|
||||
|
||||
if (localAppData) {
|
||||
candidates.push(path.join(localAppData, 'Programs', 'Git', 'bin', 'bash.exe'))
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (fileExists(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
// Last resort — bash on PATH (covers WSL bash, MSYS2, custom installs).
|
||||
// On WSL hosts findOnPath itself filters out Windows-binary paths via
|
||||
// isWindowsBinaryPathInWsl, so we won't hand back a wsl.exe shim either.
|
||||
return findOnPath('bash')
|
||||
return _findGitBash({
|
||||
isWindows: IS_WINDOWS,
|
||||
env: process.env,
|
||||
fileExists,
|
||||
findOnPath
|
||||
})
|
||||
}
|
||||
|
||||
function getVenvPython(venvRoot) {
|
||||
|
|
@ -2686,6 +2786,11 @@ async function applyUpdates(opts = {}) {
|
|||
|
||||
const venvBin = path.join(updateRoot, 'venv', IS_WINDOWS ? 'Scripts' : 'bin')
|
||||
|
||||
// ── Pre-flight state.db integrity guard (#68474) ─────────────────
|
||||
// Emergency backup and header verification before the update touches
|
||||
// anything. Runs while the backend is still alive.
|
||||
preflightStateDb(HERMES_HOME, rememberLog)
|
||||
|
||||
// Stop our own backend(s) and wait for the venv shim to unlock BEFORE we
|
||||
// spawn the updater. Without this the updater races a still-locked
|
||||
// hermes.exe (held by the backend child / its grandchildren) and the update
|
||||
|
|
@ -2883,6 +2988,92 @@ function runningAppBundle() {
|
|||
return dir.endsWith('.app') ? dir : null
|
||||
}
|
||||
|
||||
// ── Pre-flight state.db integrity guard (#68474) ─────────────────────
|
||||
// Take an emergency snapshot of state.db and verify the live copy is
|
||||
// intact before any update process mutates the install. Runs in the
|
||||
// desktop Electron process itself, before the backend is killed and
|
||||
// before the updater is spawned — a separate safety net from the
|
||||
// Python-level pre-update snapshot inside `hermes update`.
|
||||
function preflightStateDb(hermesHome, rememberLog) {
|
||||
const stateDbPath = path.join(hermesHome, 'state.db')
|
||||
|
||||
if (!fileExists(stateDbPath)) {
|
||||
rememberLog('[updates] state.db pre-flight: not found (fresh install?)')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const stat = fs.statSync(stateDbPath)
|
||||
|
||||
if (stat.size > 100) {
|
||||
const fd = fs.openSync(stateDbPath, 'r')
|
||||
const header = Buffer.alloc(16)
|
||||
|
||||
fs.readSync(fd, header, 0, 16, 0)
|
||||
fs.closeSync(fd)
|
||||
|
||||
const expectedHeader = Buffer.from('SQLite format 3\0')
|
||||
const headerOk = header.equals(expectedHeader)
|
||||
|
||||
rememberLog(
|
||||
`[updates] state.db pre-flight: size=${stat.size}, ` +
|
||||
`headerOk=${headerOk}, headerHex=${header.toString('hex')}`
|
||||
)
|
||||
|
||||
if (!headerOk) {
|
||||
rememberLog(
|
||||
'[updates] state.db header is INVALID before update — ' +
|
||||
'this indicates pre-existing corruption or a concurrent write issue'
|
||||
)
|
||||
}
|
||||
|
||||
// Emergency timestamped backup, separate from the Python-level snapshot.
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-')
|
||||
|
||||
const emergencyPath = path.join(hermesHome, `state.db.pre-update-emergency-${ts}.bak`)
|
||||
|
||||
try {
|
||||
fs.copyFileSync(stateDbPath, emergencyPath)
|
||||
const emergStat = fs.statSync(emergencyPath)
|
||||
|
||||
rememberLog(`[updates] emergency state.db backup: ${emergencyPath} ` + `(${emergStat.size} bytes)`)
|
||||
|
||||
// Prune to the 2 most recent emergency backups.
|
||||
try {
|
||||
const homeDir = fs.readdirSync(hermesHome)
|
||||
|
||||
const backups = homeDir
|
||||
.filter(
|
||||
f =>
|
||||
f.startsWith('state.db.pre-update-emergency-') &&
|
||||
f.endsWith('.bak') &&
|
||||
f !== path.basename(emergencyPath)
|
||||
)
|
||||
.sort()
|
||||
.reverse()
|
||||
|
||||
for (const old of backups.slice(2)) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(hermesHome, old))
|
||||
} catch {
|
||||
void 0
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
void 0
|
||||
}
|
||||
} catch (copyErr) {
|
||||
rememberLog(`[updates] emergency state.db backup failed: ${copyErr.message}`)
|
||||
}
|
||||
} else {
|
||||
rememberLog(`[updates] state.db too small (${stat.size} bytes) for a valid SQLite database`)
|
||||
}
|
||||
} catch (statErr) {
|
||||
rememberLog(`[updates] could not stat state.db before update: ${statErr.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
function shellQuote(value) {
|
||||
return `'${String(value).replace(/'/g, `'\\''`)}'`
|
||||
}
|
||||
|
|
@ -2901,9 +3092,12 @@ async function applyUpdatesPosixInApp(opts: any) {
|
|||
return { ok: true, manual: true, command: 'hermes update', hermesRoot: updateRoot }
|
||||
}
|
||||
|
||||
// ── Pre-flight state.db integrity guard (#68474) ──
|
||||
preflightStateDb(HERMES_HOME, rememberLog)
|
||||
|
||||
// Put the Hermes-managed Node and the venv on PATH so `hermes desktop`'s
|
||||
// npm build can find them on a machine with no system Node. Windows portable
|
||||
// Node lives directly under %LOCALAPPDATA%\hermes\node, not node\bin.
|
||||
// Node lives directly under %LOCALAPPDATA%\\hermes\\node, not node\\bin.
|
||||
// PYTHONUNBUFFERED: `hermes update` writes to a pipe here, so CPython
|
||||
// block-buffers stdout and long quiet steps (the pre-update backup can zip
|
||||
// multi-GB archives for minutes) stream nothing to the progress UI — users
|
||||
|
|
@ -4634,40 +4828,88 @@ function closePreviewWatchers() {
|
|||
}
|
||||
}
|
||||
|
||||
async function waitForHermes(baseUrl, token, signal?) {
|
||||
const deadline = Date.now() + 45_000
|
||||
let lastError = null
|
||||
// Best-effort read of a gateway's advertised auth providers, cached per base
|
||||
// URL for the life of the process. Used by the oauth pre-flight guard to tell
|
||||
// a password-provider gateway (which cannot satisfy the bearer/cookie checks
|
||||
// by design) from a real OAuth one. Any failure returns [] so callers keep the
|
||||
// strict guard — backends predating /api/auth/providers are unaffected.
|
||||
const gatewayAuthProvidersCache = new Map<string, any[]>()
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (signal?.aborted) {
|
||||
const error: any = new Error('SSH bootstrap was superseded by newer connection settings.')
|
||||
error.kind = 'superseded'
|
||||
throw error
|
||||
async function gatewayAuthProviders(baseUrl) {
|
||||
const cached = gatewayAuthProvidersCache.get(baseUrl)
|
||||
|
||||
if (cached) {
|
||||
return cached
|
||||
}
|
||||
|
||||
let providers = []
|
||||
|
||||
try {
|
||||
const body = (await fetchPublicJson(`${baseUrl}/api/auth/providers`, { timeoutMs: 8_000 })) as any
|
||||
|
||||
if (Array.isArray(body?.providers)) {
|
||||
providers = body.providers
|
||||
.filter(p => p && typeof p === 'object')
|
||||
.map(p => ({ name: String(p.name || ''), supportsPassword: Boolean(p.supports_password) }))
|
||||
.filter(p => p.name)
|
||||
}
|
||||
} catch {
|
||||
// Optional metadata — an unreadable list keeps the strict guard.
|
||||
}
|
||||
|
||||
try {
|
||||
await fetchJson(`${baseUrl}/api/status`, token)
|
||||
gatewayAuthProvidersCache.set(baseUrl, providers)
|
||||
|
||||
return
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, 500)
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timer)
|
||||
const aborted: any = new Error('SSH bootstrap was superseded by newer connection settings.')
|
||||
aborted.kind = 'superseded'
|
||||
reject(aborted)
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
return providers
|
||||
}
|
||||
|
||||
// Build the readiness probe for a connection's auth mode. A gated gateway
|
||||
// must be probed with the SAME credentials the rest of the connection uses:
|
||||
// an anonymous probe 401s forever against a live session, and it can never
|
||||
// see the 404 that identifies a backend predating /api/health (the auth gate
|
||||
// answers before the SPA catch-all). `probeIsCredentialed` tells
|
||||
// waitForHermesReady how to read a 401 — rejected session vs gated route.
|
||||
async function buildReadinessHealthProbe(baseUrl, authMode, token) {
|
||||
const nativeAt = authMode === 'oauth' ? await ensureNativeAccessToken(baseUrl).catch(() => null) : null
|
||||
const probeAuth = resolveReadinessProbeAuth(authMode, nativeAt, token)
|
||||
|
||||
if (probeAuth.kind === 'bearer') {
|
||||
return {
|
||||
// fetchJson takes the bearer via `options.bearer` — a raw `headers`
|
||||
// option is ignored, so passing one here would silently probe
|
||||
// uncredentialed and reintroduce the 401 loop.
|
||||
probeHealth: (url, options: any = {}) => fetchJson(url, null, { ...options, bearer: probeAuth.token }),
|
||||
probeIsCredentialed: true
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Hermes backend did not become ready: ${lastError?.message || 'timeout'}`)
|
||||
if (probeAuth.kind === 'cookie') {
|
||||
return {
|
||||
probeHealth: (url, options: any = {}) => fetchJsonViaOauthSession(url, options),
|
||||
probeIsCredentialed: true
|
||||
}
|
||||
}
|
||||
|
||||
if (probeAuth.kind === 'token' && probeAuth.token) {
|
||||
return {
|
||||
probeHealth: (url, options: any = {}) => fetchJson(url, probeAuth.token, options),
|
||||
probeIsCredentialed: true
|
||||
}
|
||||
}
|
||||
|
||||
return { probeHealth: fetchPublicJson, probeIsCredentialed: false }
|
||||
}
|
||||
|
||||
async function waitForHermes(baseUrl, token, signal?, authMode?) {
|
||||
const { probeHealth, probeIsCredentialed } = await buildReadinessHealthProbe(baseUrl, authMode, token)
|
||||
|
||||
return waitForHermesReady(baseUrl, {
|
||||
token,
|
||||
signal,
|
||||
fetchPublicJson,
|
||||
fetchJson: probeIsCredentialed ? (url, _token, options) => probeHealth(url, options) : fetchJson,
|
||||
probeHealth,
|
||||
probeIsCredentialed
|
||||
})
|
||||
}
|
||||
|
||||
function getWindowButtonPosition() {
|
||||
|
|
@ -6668,7 +6910,10 @@ async function buildRemoteConnection(
|
|||
// here would reject a freshly-completed native sign-in and loop the UI back
|
||||
// into "not signed in" even though mintGatewayWsTicket would succeed with
|
||||
// the stored bearer.
|
||||
if (!oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl))) {
|
||||
if (
|
||||
!oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl)) &&
|
||||
oauthGuardMayHardFail(await gatewayAuthProviders(baseUrl))
|
||||
) {
|
||||
const err = new Error(
|
||||
'Remote Hermes gateway uses OAuth, but you are not signed in. ' +
|
||||
'Open Settings → Gateway and click "Sign in", or switch back to Local.'
|
||||
|
|
@ -6909,7 +7154,7 @@ async function bootstrapSshConnectionInner(profile, sshConfig, reuseToken, sourc
|
|||
forward: (localPort, remotePort) => ssh.forward(localPort, remotePort),
|
||||
cancelForward: (localPort, remotePort) => ssh.cancelForward(localPort, remotePort),
|
||||
pickLocalPort,
|
||||
waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal),
|
||||
waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal, 'token'),
|
||||
probeReuseProof: sshProbeReuseProof,
|
||||
adoptServedToken: adoptServedDashboardToken,
|
||||
rememberLog: sshRememberLog,
|
||||
|
|
@ -7382,6 +7627,7 @@ function stopBackendChild(child) {
|
|||
// switch / crash recovery), which still resets boot progress + reloads.
|
||||
function resetHermesConnection({ soft = false } = {}) {
|
||||
backendStartFailure = null
|
||||
remoteReauthFailure = null
|
||||
remoteLiveness.clear()
|
||||
const hermesProcess = backendConnectionState.invalidate()
|
||||
stopBackendChild(hermesProcess)
|
||||
|
|
@ -7582,7 +7828,7 @@ async function spawnPoolBackend(profile, entry) {
|
|||
const remote = await resolveRemoteBackend(profile)
|
||||
|
||||
if (remote) {
|
||||
await waitForHermes(remote.baseUrl, remote.token)
|
||||
await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode)
|
||||
|
||||
return {
|
||||
...remote,
|
||||
|
|
@ -7774,6 +8020,13 @@ async function startHermes() {
|
|||
throw backendStartFailure
|
||||
}
|
||||
|
||||
// A confirmed remote reauth rejection is terminal until the user signs in.
|
||||
// Short-circuiting here keeps the boot-failure overlay latched and its
|
||||
// "Sign in" button clickable, instead of re-driving boot on every retry.
|
||||
if (remoteReauthFailure) {
|
||||
throw remoteReauthFailure
|
||||
}
|
||||
|
||||
// E2E: simulate a boot failure without breaking the real backend. The boot
|
||||
// progresses a few steps, then fails with the given error message.
|
||||
if (BOOT_FAKE_ERROR) {
|
||||
|
|
@ -7798,16 +8051,9 @@ async function startHermes() {
|
|||
let attemptedRemote = primaryBackendIsRemote()
|
||||
|
||||
const connectionPromise = (async () => {
|
||||
await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8)
|
||||
// Resolve for the desktop's primary profile so a per-profile remote
|
||||
// override on the active profile is honored (falls back to env / global).
|
||||
// Re-read once resolved so the classification tracks the value actually used.
|
||||
attemptedRemote = primaryBackendIsRemote()
|
||||
const remote = await resolveRemoteBackend(primaryProfileKey())
|
||||
|
||||
if (remote) {
|
||||
const connectRemote = async remote => {
|
||||
await advanceBootProgress('backend.remote', `Connecting to remote Hermes backend at ${remote.baseUrl}`, 24)
|
||||
await waitForHermes(remote.baseUrl, remote.token)
|
||||
await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode)
|
||||
updateBootProgress({
|
||||
phase: 'backend.ready',
|
||||
message: 'Remote Hermes backend is ready',
|
||||
|
|
@ -7831,14 +8077,9 @@ async function startHermes() {
|
|||
}
|
||||
}
|
||||
|
||||
// Mutual exclusion with an in-app update (#50238). If this instance was
|
||||
// relaunched while the Tauri updater is still applying an update, spawning
|
||||
// a local backend now re-locks the venv shim and gets killed by the
|
||||
// updater's straggler cleanup — looping. Park until the update finishes (or
|
||||
// is detected stale), THEN start the backend. Local backends only; remote
|
||||
// connections returned above and never touch the install tree.
|
||||
await waitForUpdateToFinish()
|
||||
|
||||
await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8)
|
||||
// Resolve for the desktop's primary profile so a per-profile remote
|
||||
// override on the active profile is honored (falls back to env / global).
|
||||
const token = crypto.randomBytes(32).toString('base64url')
|
||||
// --port 0: the OS assigns an ephemeral port; the child announces it on stdout.
|
||||
const backendArgs = ['serve', '--host', '127.0.0.1', '--port', '0']
|
||||
|
|
@ -7853,8 +8094,32 @@ async function startHermes() {
|
|||
backendArgs.unshift('--profile', activeProfile)
|
||||
}
|
||||
|
||||
await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28)
|
||||
const backend = await ensureRuntime(resolveHermesBackend(backendArgs))
|
||||
const setup = await runPrimaryBackendStartup({
|
||||
connectRemote,
|
||||
ensureLocalRuntime: ensureRuntime,
|
||||
prepareLocalBackend: async () => {
|
||||
await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28)
|
||||
|
||||
return resolveHermesBackend(backendArgs)
|
||||
},
|
||||
resolveRemote: () => {
|
||||
// Classify immediately before each throwing resolve. This callback runs
|
||||
// both for an already-saved remote and after first-run remote Apply.
|
||||
attemptedRemote = primaryBackendIsRemote()
|
||||
|
||||
return resolveRemoteBackend(primaryProfileKey())
|
||||
},
|
||||
waitForDecision: waitForFirstRunSetupChoice,
|
||||
// Mutual exclusion with an in-app update (#50238). Remote connections
|
||||
// return before this waiter; local starts park until the updater exits.
|
||||
waitForLocalStart: waitForUpdateToFinish
|
||||
})
|
||||
|
||||
if (setup.kind === 'remote') {
|
||||
return setup.connection
|
||||
}
|
||||
|
||||
const backend = setup.backend
|
||||
// Route old runtimes (no `serve`) through the legacy `dashboard --no-open`.
|
||||
backend.args = getBackendArgsForRuntime(backend)
|
||||
const hermesCwd = resolveHermesCwd()
|
||||
|
|
@ -8010,6 +8275,10 @@ async function startHermes() {
|
|||
throw error
|
||||
}
|
||||
|
||||
if (error instanceof FirstRunSetupResetError) {
|
||||
throw error
|
||||
}
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
|
||||
// Only latch LOCAL boot failures. A remote failure (lapsed session / mint
|
||||
|
|
@ -8021,6 +8290,12 @@ async function startHermes() {
|
|||
backendStartFailure = error instanceof Error ? error : new Error(message)
|
||||
}
|
||||
|
||||
// A confirmed reauth rejection latches separately: it can't self-heal, and
|
||||
// leaving it unlatched hides the overlay's "Sign in" button on every retry.
|
||||
if (shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) })) {
|
||||
remoteReauthFailure = error instanceof Error ? error : new Error(message)
|
||||
}
|
||||
|
||||
updateBootProgress(
|
||||
{
|
||||
error: message,
|
||||
|
|
@ -8824,16 +9099,9 @@ ipcMain.handle('hermes:bootstrap:reset', async () => {
|
|||
await teardownPrimaryBackendAndWait()
|
||||
bootstrapFailure = null
|
||||
backendStartFailure = null
|
||||
bootstrapState = {
|
||||
active: false,
|
||||
manifest: null,
|
||||
stages: {},
|
||||
error: null,
|
||||
log: [],
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
unsupportedPlatform: null
|
||||
}
|
||||
remoteReauthFailure = null
|
||||
getFirstRunSetupGate().resetForRetry()
|
||||
resetBootstrapSnapshot()
|
||||
|
||||
return { ok: true }
|
||||
})
|
||||
|
|
@ -8854,10 +9122,18 @@ ipcMain.handle('hermes:bootstrap:repair', async () => {
|
|||
|
||||
bootstrapFailure = null
|
||||
backendStartFailure = null
|
||||
remoteReauthFailure = null
|
||||
getFirstRunSetupGate().resetForRepair()
|
||||
resetHermesConnection()
|
||||
|
||||
return { ok: true }
|
||||
})
|
||||
ipcMain.handle('hermes:bootstrap:continue-local', async () => {
|
||||
rememberLog('[bootstrap] local install selected by renderer; continuing first-launch bootstrap')
|
||||
continueFirstRunLocalBootstrap()
|
||||
|
||||
return { ok: true }
|
||||
})
|
||||
ipcMain.handle('hermes:bootstrap:cancel', async () => {
|
||||
// Renderer's Cancel button during first-launch install. Abort the running
|
||||
// install script (SIGTERM via the runner's abortSignal). runBootstrap
|
||||
|
|
@ -8956,6 +9232,9 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) =>
|
|||
})
|
||||
|
||||
_storeNativeTokens(baseUrl, tokens)
|
||||
// Confirmed sign-in — release the reauth latch so the next
|
||||
// startHermes() re-dials instead of replaying the stale rejection.
|
||||
remoteReauthFailure = null
|
||||
|
||||
return { ok: true, baseUrl, connected: true }
|
||||
} catch (error) {
|
||||
|
|
@ -8972,7 +9251,16 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) =>
|
|||
// Legacy embedded-webview cookie flow.
|
||||
await openOauthLoginWindow(baseUrl)
|
||||
|
||||
return { ok: true, baseUrl, connected: await hasOauthSessionCookie(baseUrl) }
|
||||
const connected = await hasOauthSessionCookie(baseUrl)
|
||||
|
||||
// Only a CONFIRMED sign-in releases the latch. A cancelled/closed login
|
||||
// window must leave it set, or the overlay's "Sign in" button starts
|
||||
// flickering again on the next retry.
|
||||
if (connected) {
|
||||
remoteReauthFailure = null
|
||||
}
|
||||
|
||||
return { ok: true, baseUrl, connected }
|
||||
})
|
||||
ipcMain.handle('hermes:connection-config:oauth-logout', async (_event, rawUrl) => {
|
||||
const baseUrl = rawUrl ? normalizeRemoteBaseUrl(rawUrl) : ''
|
||||
|
|
@ -9036,6 +9324,18 @@ ipcMain.handle('hermes:connection-config:apply', async (_event, payload) => {
|
|||
await applyConnectionChange({
|
||||
cancelAndWait: value => sshBootstrapCoordinator.cancelAndWait(value),
|
||||
isPrimary: !key || key === primaryProfileKey(),
|
||||
rehomePrimary: () =>
|
||||
rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure: () => {
|
||||
// A remote connection bypasses local runtime/bootstrap failures. Clear
|
||||
// the local-install latch so unsupported/failure escape paths can re-home.
|
||||
bootstrapFailure = null
|
||||
},
|
||||
mode: config.mode,
|
||||
notifyConnectionApplied: sendConnectionApplied,
|
||||
resumeFirstRunRemote: abandonFirstRunSetupChoiceForRemoteApply,
|
||||
teardownPrimaryBackend: teardownPrimaryBackendAndWait
|
||||
}),
|
||||
scope,
|
||||
sendApplied: sendConnectionApplied,
|
||||
stopPool: stopPoolBackend,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Regression tests for electron/native-auth-decisions.ts — the three pure
|
||||
* decision seams behind the RFC 8252 native-app auth flow, each of which was a
|
||||
* real runtime bug that the mocked flow tests could not catch.
|
||||
* Regression tests for electron/native-auth-decisions.ts — the pure decision
|
||||
* seams behind the RFC 8252 native-app auth flow, each of which was a real
|
||||
* runtime bug that the mocked flow tests could not catch.
|
||||
*
|
||||
* Run via the vitest `electron` project (electron/**\/*.test.ts).
|
||||
*/
|
||||
|
|
@ -10,7 +10,13 @@ import assert from 'node:assert/strict'
|
|||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions'
|
||||
import {
|
||||
oauthGuardMayHardFail,
|
||||
oauthSessionIsLive,
|
||||
resolveJsonBody,
|
||||
resolveOauthRestAuth,
|
||||
resolveReadinessProbeAuth
|
||||
} from './native-auth-decisions'
|
||||
|
||||
// --- 1. body encoding (guards the double-JSON.stringify 422) ---
|
||||
|
||||
|
|
@ -66,3 +72,61 @@ test('resolveOauthRestAuth falls back to cookie when there is no native token',
|
|||
// Empty string is not a usable bearer — must fall back, not send "Bearer ".
|
||||
assert.deepEqual(resolveOauthRestAuth(''), { kind: 'cookie' })
|
||||
})
|
||||
|
||||
// --- 4. readiness-probe auth (guards the credential-free 401 boot loop) ---
|
||||
|
||||
test('resolveReadinessProbeAuth reuses the oauth bearer-vs-cookie choice', () => {
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', 'native-at'), { kind: 'bearer', token: 'native-at' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', null), { kind: 'cookie' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', ''), { kind: 'cookie' })
|
||||
})
|
||||
|
||||
test('resolveReadinessProbeAuth sends the session token for a token gateway', () => {
|
||||
assert.deepEqual(resolveReadinessProbeAuth('token', null, 'session-token'), {
|
||||
kind: 'token',
|
||||
token: 'session-token'
|
||||
})
|
||||
assert.deepEqual(resolveReadinessProbeAuth('token', null, null), { kind: 'token', token: null })
|
||||
})
|
||||
|
||||
test('resolveReadinessProbeAuth stays public for local and unknown modes', () => {
|
||||
// A loopback backend has no gate; sending credentials it never issued is
|
||||
// meaningless, and an unknown mode must not invent a credential.
|
||||
assert.deepEqual(resolveReadinessProbeAuth('local', 'native-at', 'session-token'), { kind: 'public' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth(undefined, 'native-at', 'session-token'), { kind: 'public' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('something-new', null, null), { kind: 'public' })
|
||||
})
|
||||
|
||||
// --- 5. oauth guard vs password gateways (guards the false "not signed in") ---
|
||||
|
||||
test('oauthGuardMayHardFail is false only when EVERY provider is password-based', () => {
|
||||
assert.equal(oauthGuardMayHardFail([{ name: 'basic', supportsPassword: true }]), false)
|
||||
assert.equal(
|
||||
oauthGuardMayHardFail([
|
||||
{ name: 'basic', supportsPassword: true },
|
||||
{ name: 'ldap', supportsPassword: true }
|
||||
]),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('oauthGuardMayHardFail keeps the strict guard for oauth and mixed deployments', () => {
|
||||
assert.equal(oauthGuardMayHardFail([{ name: 'nous', supportsPassword: false }]), true)
|
||||
assert.equal(
|
||||
oauthGuardMayHardFail([
|
||||
{ name: 'nous', supportsPassword: false },
|
||||
{ name: 'basic', supportsPassword: true }
|
||||
]),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('oauthGuardMayHardFail keeps the strict guard when the list is unusable', () => {
|
||||
// Backends predating /api/auth/providers, or an unreachable probe, must not
|
||||
// silently weaken the guard.
|
||||
assert.equal(oauthGuardMayHardFail([]), true)
|
||||
assert.equal(oauthGuardMayHardFail(null), true)
|
||||
assert.equal(oauthGuardMayHardFail(undefined), true)
|
||||
assert.equal(oauthGuardMayHardFail('nonsense' as any), true)
|
||||
assert.equal(oauthGuardMayHardFail([{ supportsPassword: true }]), true)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -21,7 +21,17 @@
|
|||
* native bearer when present, else the cookie partition. Cookie-only
|
||||
* routing returns 401 no_cookie for a cookieless native session.
|
||||
*
|
||||
* All three are trivial once named; the value is the test that pins the
|
||||
* 4. resolveReadinessProbeAuth — the boot readiness probe must authenticate
|
||||
* the same way the rest of the connection does. A credential-free probe
|
||||
* against a gated gateway 401s forever; worse, it cannot tell a missing
|
||||
* route from a rejected session (see backend-health.ts).
|
||||
*
|
||||
* 5. oauthGuardMayHardFail — `auth_required: true` means "this gateway is
|
||||
* gated", NOT "this gateway speaks OAuth". A password-provider gateway
|
||||
* can satisfy neither the native-bearer nor the OAuth-partition-cookie
|
||||
* check by design, so the pre-flight guard must not hard-fail it.
|
||||
*
|
||||
* All five are trivial once named; the value is the test that pins the
|
||||
* contract so the god-file call sites can't drift back to the buggy shape.
|
||||
*/
|
||||
|
||||
|
|
@ -60,3 +70,75 @@ export function resolveOauthRestAuth(nativeAccessToken: string | null | undefine
|
|||
|
||||
return { kind: 'cookie' }
|
||||
}
|
||||
|
||||
export type ReadinessProbeAuth = OauthRestAuth | { kind: 'token'; token: string | null } | { kind: 'public' }
|
||||
|
||||
/**
|
||||
* Decide how the boot readiness probe authenticates.
|
||||
*
|
||||
* The probe must present the SAME credentials the rest of the connection
|
||||
* will use. A credential-free probe against a gated gateway 401s until the
|
||||
* boot deadline even though the session is perfectly valid — and because the
|
||||
* dashboard auth gate runs ahead of the SPA catch-all, an unknown `/api/*`
|
||||
* path answers 401 rather than 404, so the probe also cannot detect a backend
|
||||
* that predates `/api/health`. Sending credentials is what lets a missing
|
||||
* route surface as a real 404 (see `isMissingHealthEndpointError`).
|
||||
*
|
||||
* `oauth` reuses `resolveOauthRestAuth` so the probe and every other oauth
|
||||
* REST call make the identical bearer-vs-cookie choice. `token` presents the
|
||||
* connection's session token. `local` (and anything unrecognized) stays
|
||||
* public: a loopback backend has no gate, and sending credentials it never
|
||||
* issued would be meaningless.
|
||||
*/
|
||||
export function resolveReadinessProbeAuth(
|
||||
authMode: string | null | undefined,
|
||||
nativeAccessToken?: string | null,
|
||||
connectionToken?: string | null
|
||||
): ReadinessProbeAuth {
|
||||
if (authMode === 'oauth') {
|
||||
return resolveOauthRestAuth(nativeAccessToken)
|
||||
}
|
||||
|
||||
if (authMode === 'token') {
|
||||
return { kind: 'token', token: connectionToken ?? null }
|
||||
}
|
||||
|
||||
return { kind: 'public' }
|
||||
}
|
||||
|
||||
export interface AdvertisedAuthProvider {
|
||||
name?: string
|
||||
supportsPassword?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the oauth pre-flight guard may hard-fail a connection for "not
|
||||
* signed in".
|
||||
*
|
||||
* `authModeFromStatus` maps the gateway's `auth_required: true` onto
|
||||
* `'oauth'`, but that flag only means the dashboard is GATED — it says
|
||||
* nothing about how you authenticate. A gateway whose providers are all
|
||||
* username/password cannot satisfy the guard's checks by construction:
|
||||
* `start_login` raises NotImplementedError, `/auth/native/authorize` rejects
|
||||
* password providers, and its cookies are set by a plain password-login POST
|
||||
* rather than the `/auth/callback` redirect the OAuth partition is primed
|
||||
* for. Hard-failing there rejects a live session one line before the
|
||||
* ws-ticket mint that would have succeeded against that very partition.
|
||||
*
|
||||
* Returns false only when EVERY advertised provider is password-based. An
|
||||
* unknown or empty list keeps the strict guard, so backends that predate
|
||||
* `/api/auth/providers` are unaffected.
|
||||
*/
|
||||
export function oauthGuardMayHardFail(providers: AdvertisedAuthProvider[] | null | undefined): boolean {
|
||||
if (!Array.isArray(providers) || providers.length === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
const named = providers.filter(provider => provider && typeof provider === 'object' && provider.name)
|
||||
|
||||
if (named.length === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
return !named.every(provider => provider.supportsPassword)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -237,6 +237,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
|
|||
// current snapshot via getBootstrapState() to recover after a devtools
|
||||
// reload mid-bootstrap.
|
||||
getBootstrapState: () => ipcRenderer.invoke('hermes:bootstrap:get'),
|
||||
continueBootstrapLocal: () => ipcRenderer.invoke('hermes:bootstrap:continue-local'),
|
||||
resetBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:reset'),
|
||||
repairBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:repair'),
|
||||
cancelBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:cancel'),
|
||||
|
|
|
|||
110
apps/desktop/electron/primary-backend-startup.test.ts
Normal file
110
apps/desktop/electron/primary-backend-startup.test.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test, vi } from 'vitest'
|
||||
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup'
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
function startupOptions(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
connectRemote: vi.fn(async remote => ({ baseUrl: remote.baseUrl, mode: 'remote' as const })),
|
||||
ensureLocalRuntime: vi.fn(async backend => ({ ...backend, command: 'hermes' })),
|
||||
prepareLocalBackend: vi.fn(async () => bootstrapBackend),
|
||||
resolveRemote: vi.fn(async () => null),
|
||||
waitForDecision: vi.fn(async () => 'continue-local' as const),
|
||||
waitForLocalStart: vi.fn(async () => {}),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('remote apply re-resolves the saved connection without ensuring a local runtime', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
|
||||
let configuredRemote: typeof savedRemote | null = null
|
||||
|
||||
const options = startupOptions({
|
||||
resolveRemote: vi.fn(async () => configuredRemote),
|
||||
waitForDecision: gate.wait
|
||||
})
|
||||
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
configuredRemote = savedRemote
|
||||
assert.equal(gate.abandonForRemoteApply(), true)
|
||||
|
||||
assert.deepEqual(await pending, {
|
||||
kind: 'remote',
|
||||
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
|
||||
})
|
||||
assert.deepEqual(options.resolveRemote.mock.calls, [[], []])
|
||||
assert.deepEqual(options.connectRemote.mock.calls, [[savedRemote]])
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('an already-saved remote bypasses every local startup step', async () => {
|
||||
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
|
||||
const options = startupOptions({ resolveRemote: vi.fn(async () => savedRemote) })
|
||||
|
||||
assert.deepEqual(await runPrimaryBackendStartup(options), {
|
||||
kind: 'remote',
|
||||
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
|
||||
})
|
||||
assert.equal(options.waitForLocalStart.mock.calls.length, 0)
|
||||
assert.equal(options.prepareLocalBackend.mock.calls.length, 0)
|
||||
assert.equal(options.waitForDecision.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('remote apply fails clearly when no saved remote can be resolved', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const options = startupOptions({ waitForDecision: gate.wait })
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.abandonForRemoteApply()
|
||||
|
||||
await assert.rejects(pending, /without a saved remote backend/)
|
||||
assert.equal(options.connectRemote.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('continue local waits for update exclusion and ensures the prepared runtime exactly once', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const runtimeBackend = { ...bootstrapBackend, command: 'hermes' }
|
||||
|
||||
const options = startupOptions({
|
||||
ensureLocalRuntime: vi.fn(async () => runtimeBackend),
|
||||
waitForDecision: gate.wait
|
||||
})
|
||||
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.continueLocal()
|
||||
|
||||
assert.deepEqual(await pending, { kind: 'local', backend: runtimeBackend })
|
||||
assert.deepEqual(options.waitForLocalStart.mock.calls, [[]])
|
||||
assert.deepEqual(options.prepareLocalBackend.mock.calls, [[]])
|
||||
assert.deepEqual(options.ensureLocalRuntime.mock.calls, [[bootstrapBackend]])
|
||||
assert.deepEqual(options.resolveRemote.mock.calls, [[]])
|
||||
})
|
||||
|
||||
test('reset rejects with a typed error and never enters either backend', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const options = startupOptions({ waitForDecision: gate.wait })
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.resetForRetry()
|
||||
|
||||
await assert.rejects(pending, error => error instanceof FirstRunSetupResetError && error.firstRunSetupReset)
|
||||
assert.equal(options.connectRemote.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
66
apps/desktop/electron/primary-backend-startup.ts
Normal file
66
apps/desktop/electron/primary-backend-startup.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import type { FirstRunSetupDecision } from './first-run-setup-gate'
|
||||
|
||||
export interface PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection> {
|
||||
connectRemote: (remote: Remote) => Promise<Connection>
|
||||
ensureLocalRuntime: (backend: Backend) => Promise<RuntimeBackend>
|
||||
prepareLocalBackend: () => Backend | Promise<Backend>
|
||||
resolveRemote: () => Promise<Remote | null>
|
||||
waitForDecision: (backend: Backend) => Promise<FirstRunSetupDecision>
|
||||
waitForLocalStart: () => Promise<unknown>
|
||||
}
|
||||
|
||||
export type PrimaryBackendStartupResult<RuntimeBackend, Connection> =
|
||||
| { kind: 'local'; backend: RuntimeBackend }
|
||||
| { kind: 'remote'; connection: Connection }
|
||||
|
||||
export class FirstRunSetupResetError extends Error {
|
||||
readonly firstRunSetupReset = true
|
||||
|
||||
constructor() {
|
||||
super('First-run setup was reset before a choice completed.')
|
||||
this.name = 'FirstRunSetupResetError'
|
||||
}
|
||||
}
|
||||
|
||||
// Owns the production startHermes path up to the local process spawn. Keeping
|
||||
// the full ordering here makes the first-run remote boundary executable in a
|
||||
// test: an already-saved remote wins immediately; otherwise update exclusion
|
||||
// and local backend resolution happen before the setup gate, and a remote Apply
|
||||
// re-resolves persisted config without ever entering ensureRuntime/bootstrap.
|
||||
export async function runPrimaryBackendStartup<Backend, RuntimeBackend, Remote, Connection>({
|
||||
connectRemote,
|
||||
ensureLocalRuntime,
|
||||
prepareLocalBackend,
|
||||
resolveRemote,
|
||||
waitForDecision,
|
||||
waitForLocalStart
|
||||
}: PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection>): Promise<
|
||||
PrimaryBackendStartupResult<RuntimeBackend, Connection>
|
||||
> {
|
||||
const savedRemote = await resolveRemote()
|
||||
|
||||
if (savedRemote) {
|
||||
return { kind: 'remote', connection: await connectRemote(savedRemote) }
|
||||
}
|
||||
|
||||
await waitForLocalStart()
|
||||
|
||||
const backend = await prepareLocalBackend()
|
||||
const decision = await waitForDecision(backend)
|
||||
|
||||
if (decision === 'remote-applied') {
|
||||
const appliedRemote = await resolveRemote()
|
||||
|
||||
if (!appliedRemote) {
|
||||
throw new Error('First-run remote setup completed without a saved remote backend.')
|
||||
}
|
||||
|
||||
return { kind: 'remote', connection: await connectRemote(appliedRemote) }
|
||||
}
|
||||
|
||||
if (decision === 'reset') {
|
||||
throw new FirstRunSetupResetError()
|
||||
}
|
||||
|
||||
return { kind: 'local', backend: await ensureLocalRuntime(backend) }
|
||||
}
|
||||
35
apps/desktop/electron/primary-connection-rehome.ts
Normal file
35
apps/desktop/electron/primary-connection-rehome.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
export interface PrimaryConnectionRehomeOptions {
|
||||
clearLocalBootstrapFailure: () => void
|
||||
mode: string
|
||||
notifyConnectionApplied: () => void
|
||||
resumeFirstRunRemote: () => boolean
|
||||
teardownPrimaryBackend: (options: { soft: boolean }) => Promise<void>
|
||||
}
|
||||
|
||||
// Production seam shared by the connection-config IPC handler and the
|
||||
// first-run integration test. A remote apply that resumes the active setup
|
||||
// gate must keep that connection attempt alive; ordinary mode changes tear the
|
||||
// current backend down before the renderer is told to reconnect.
|
||||
export async function rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode,
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote,
|
||||
teardownPrimaryBackend
|
||||
}: PrimaryConnectionRehomeOptions): Promise<{ resumedFirstRunRemote: boolean }> {
|
||||
let resumedFirstRunRemote = false
|
||||
|
||||
if (mode === 'remote') {
|
||||
resumedFirstRunRemote = resumeFirstRunRemote()
|
||||
clearLocalBootstrapFailure()
|
||||
}
|
||||
|
||||
if (resumedFirstRunRemote) {
|
||||
return { resumedFirstRunRemote: true }
|
||||
}
|
||||
|
||||
await teardownPrimaryBackend({ soft: true })
|
||||
notifyConnectionApplied()
|
||||
|
||||
return { resumedFirstRunRemote: false }
|
||||
}
|
||||
|
|
@ -8,7 +8,8 @@ import {
|
|||
needsExecBit,
|
||||
spawnHelperCandidates,
|
||||
type SpawnHelperFs,
|
||||
withExecBits
|
||||
withExecBits,
|
||||
writableNodePtyRoot
|
||||
} from './spawn-helper-perms'
|
||||
|
||||
interface FakeFile {
|
||||
|
|
@ -56,6 +57,30 @@ function fakeFs(
|
|||
}
|
||||
}
|
||||
|
||||
test('rewrites an archived node-pty root to the matching unpacked tree exactly once', () => {
|
||||
assert.equal(
|
||||
writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty'),
|
||||
'/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'
|
||||
)
|
||||
assert.equal(
|
||||
writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'),
|
||||
'/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'
|
||||
)
|
||||
})
|
||||
|
||||
test('uses the unpacked helper when resolution reports an app.asar node-pty root', () => {
|
||||
const archivedRoot = '/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty'
|
||||
const unpackedRoot = writableNodePtyRoot(archivedRoot)
|
||||
const helper = join(unpackedRoot, 'prebuilds', 'darwin-arm64', 'spawn-helper')
|
||||
const fs = fakeFs({ [helper]: { mode: 0o644 } }, { [join(unpackedRoot, 'prebuilds')]: ['darwin-arm64'] })
|
||||
|
||||
const result = ensureSpawnHelperExecutable(archivedRoot, fs)
|
||||
|
||||
assert.deepEqual(result.fixed, [helper])
|
||||
assert.deepEqual(result.errors, [])
|
||||
assert.deepEqual(fs.chmods, [{ path: helper, mode: 0o755 }])
|
||||
})
|
||||
|
||||
test('needsExecBit / withExecBits treat any missing exec bit as non-executable', () => {
|
||||
assert.equal(needsExecBit(0o644), true)
|
||||
assert.equal(needsExecBit(0o755), false)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,14 @@ import { join } from 'node:path'
|
|||
|
||||
const EXEC_BITS = 0o111
|
||||
|
||||
// Electron exposes module paths inside app.asar even when electron-builder has
|
||||
// unpacked the native payload beside it. `stat` can read an archived path, but
|
||||
// chmod cannot mutate it (ENOTDIR). Native node-pty helpers belong in the
|
||||
// writable app.asar.unpacked tree; leave an already-unpacked path unchanged.
|
||||
export function writableNodePtyRoot(nodePtyRoot: string): string {
|
||||
return nodePtyRoot.replace(/app\.asar(?!\.unpacked)/, 'app.asar.unpacked')
|
||||
}
|
||||
|
||||
export interface SpawnHelperFs {
|
||||
existsSync(path: string): boolean
|
||||
readdirSync(path: string): string[]
|
||||
|
|
@ -81,8 +89,9 @@ export function ensureSpawnHelperExecutable(
|
|||
fs: SpawnHelperFs = defaultFs
|
||||
): EnsureSpawnHelperResult {
|
||||
const result: EnsureSpawnHelperResult = { fixed: [], errors: [] }
|
||||
const writableRoot = writableNodePtyRoot(nodePtyRoot)
|
||||
|
||||
for (const path of spawnHelperCandidates(nodePtyRoot, fs)) {
|
||||
for (const path of spawnHelperCandidates(writableRoot, fs)) {
|
||||
if (!fs.existsSync(path)) {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,5 +39,43 @@ export default [
|
|||
rules: {
|
||||
'no-restricted-globals': ['warn', 'document']
|
||||
}
|
||||
},
|
||||
{
|
||||
// Ban mirroring reactive values into refs via useEffect — the "atom-mirrored
|
||||
// ref" antipattern. A ref synced from a nanostores atom via useEffect lags the
|
||||
// atom by one render, which creates stale-read bugs in callbacks that read the
|
||||
// ref (cancelRun sent session.interrupt to the wrong session; steerPrompt,
|
||||
// restoreToMessage, editMessage all had closure-priority stale reads). The fix
|
||||
// is to read $atom.get() directly in callbacks instead. This rule catches the
|
||||
// mirroring effect at lint time so the pattern can't reappear. Legitimate
|
||||
// non-atom ref writes inside useEffect (DOM instance refs, mount flags, request
|
||||
// tokens, prop mirrors) get an eslint-disable-next-line with a comment.
|
||||
files: ['src/**/*.{ts,tsx}'],
|
||||
rules: {
|
||||
'no-restricted-syntax': [
|
||||
'error',
|
||||
{
|
||||
// useEffect(() => { someRef.current = value }, [value])
|
||||
selector:
|
||||
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="AssignmentExpression"][body.left.type="MemberExpression"][body.left.property.name="current"]',
|
||||
message:
|
||||
'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
|
||||
},
|
||||
{
|
||||
// useEffect(() => { someRef.current = value; ... }, [value])
|
||||
selector:
|
||||
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(AssignmentExpression[left.type="MemberExpression"][left.property.name="current"])',
|
||||
message:
|
||||
'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
|
||||
},
|
||||
{
|
||||
// useEffect(() => { setMutableRef(ref, value) }, [value])
|
||||
selector:
|
||||
'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(CallExpression[callee.name="setMutableRef"])',
|
||||
message:
|
||||
'Do not mirror reactive values into refs via useEffect (setMutableRef included). Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@
|
|||
"motion": "^12.38.0",
|
||||
"nanostores": "^1.3.0",
|
||||
"node-pty": "1.1.0",
|
||||
"radix-ui": "^1.4.3",
|
||||
"radix-ui": "^1.6.5",
|
||||
"react": "^19.2.5",
|
||||
"react-arborist": "^3.5.0",
|
||||
"react-dnd-html5-backend": "^14.0.3",
|
||||
|
|
|
|||
|
|
@ -57,7 +57,8 @@
|
|||
* - electronPlatformName: 'win32' | 'darwin' | 'linux'
|
||||
* - arch: Arch enum (0=ia32, 1=x64, 2=armv7l, 3=arm64, 4=universal)
|
||||
*/
|
||||
import { existsSync, rmSync } from 'node:fs'
|
||||
import { existsSync, rmSync, renameSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { Arch } from 'electron-builder'
|
||||
import { stageNodePty } from './stage-native-deps.mjs'
|
||||
|
||||
|
|
@ -75,10 +76,52 @@ export function cleanStaleAppOutDir(appOutDir) {
|
|||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Windows rollback material (#69179): before wiping the previous unpacked
|
||||
* tree, preserve it as `<appOutDir>.bak` — but ONLY when it holds the product
|
||||
* exe (i.e. it is a previously-working build, not the corrupted partial state
|
||||
* cleanStaleAppOutDir exists to remove). If the fresh pack then produces a
|
||||
* Hermes.exe that Windows can't load (truncated PE from a corrupt cached
|
||||
* Electron zip, wrong arch), the updater's integrity gate in
|
||||
* `hermes desktop --build-only` (hermes_cli/main.py
|
||||
* `_ensure_desktop_exe_launchable`) restores this .bak instead of leaving the
|
||||
* user with "This app can't run on your computer".
|
||||
*
|
||||
* Returns true when the tree was preserved (appOutDir no longer exists), false
|
||||
* when there was nothing worth preserving (caller falls through to the wipe).
|
||||
* A rename failure (AV holding a handle) also returns false — the wipe is the
|
||||
* safe fallback and matches pre-#69179 behavior exactly.
|
||||
*/
|
||||
export function preserveRollbackBackup(appOutDir, productExeName = 'Hermes.exe') {
|
||||
if (!appOutDir || typeof appOutDir !== 'string' || !existsSync(appOutDir)) {
|
||||
return false
|
||||
}
|
||||
if (!existsSync(path.join(appOutDir, productExeName))) {
|
||||
// Partial/corrupt tree (interrupted prior pack) — not rollback material.
|
||||
return false
|
||||
}
|
||||
const backupDir = `${appOutDir}.bak`
|
||||
try {
|
||||
rmSync(backupDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 })
|
||||
renameSync(appOutDir, backupDir)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export default async function beforePack(context) {
|
||||
const appOutDir = context && context.appOutDir
|
||||
const platformName = context && context.electronPlatformName
|
||||
try {
|
||||
if (cleanStaleAppOutDir(appOutDir)) {
|
||||
// Windows: keep the previous working build as rollback material for the
|
||||
// post-build integrity gate (#69179) instead of destroying it. Falls
|
||||
// through to the plain wipe when the old tree is partial/corrupt or the
|
||||
// rename fails.
|
||||
const productExe = `${(context && context.packager?.appInfo?.productFilename) || 'Hermes'}.exe`
|
||||
if (platformName === 'win32' && preserveRollbackBackup(appOutDir, productExe)) {
|
||||
console.log(`[before-pack] preserved previous unpacked dir for rollback: ${appOutDir}.bak`)
|
||||
} else if (cleanStaleAppOutDir(appOutDir)) {
|
||||
console.log(`[before-pack] removed stale unpacked dir before staging: ${appOutDir}`)
|
||||
}
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import os from 'node:os'
|
|||
import path from 'node:path'
|
||||
import { test } from 'vitest'
|
||||
|
||||
import beforePack, { cleanStaleAppOutDir } from '../scripts/before-pack.mjs'
|
||||
import beforePack, { cleanStaleAppOutDir, preserveRollbackBackup } from '../scripts/before-pack.mjs'
|
||||
|
||||
test('cleanStaleAppOutDir removes a populated unpacked directory', () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
|
|
@ -50,3 +50,106 @@ test('beforePack default export resolves even when cleanup throws', async () =>
|
|||
// remove; the contract under test is that the hook never rejects.
|
||||
await assert.doesNotReject(beforePack({ appOutDir: '', electronPlatformName: 'linux' }))
|
||||
})
|
||||
|
||||
// ─── Windows rollback preservation (#69179) ────────────────────────────────
|
||||
|
||||
test('preserveRollbackBackup moves a working build to .bak', () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
try {
|
||||
const appOutDir = path.join(tempRoot, 'win-unpacked')
|
||||
fs.mkdirSync(appOutDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-old-build', 'utf8')
|
||||
fs.writeFileSync(path.join(appOutDir, 'resources.pak'), 'x', 'utf8')
|
||||
|
||||
const preserved = preserveRollbackBackup(appOutDir, 'Hermes.exe')
|
||||
|
||||
assert.equal(preserved, true)
|
||||
// Original slot vacated so electron-builder stages into a clean tree...
|
||||
assert.equal(fs.existsSync(appOutDir), false)
|
||||
// ...and the previous working build is intact under .bak for rollback.
|
||||
assert.equal(
|
||||
fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'),
|
||||
'MZ-old-build'
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('preserveRollbackBackup replaces a stale .bak from an older update', () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
try {
|
||||
const appOutDir = path.join(tempRoot, 'win-unpacked')
|
||||
fs.mkdirSync(appOutDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'current', 'utf8')
|
||||
fs.mkdirSync(`${appOutDir}.bak`, { recursive: true })
|
||||
fs.writeFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'two-updates-ago', 'utf8')
|
||||
|
||||
assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), true)
|
||||
assert.equal(fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'), 'current')
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('preserveRollbackBackup refuses a partial tree missing the product exe', () => {
|
||||
// The corrupted partial state (interrupted prior pack) must NOT become
|
||||
// rollback material — it is exactly what cleanStaleAppOutDir exists to wipe.
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
try {
|
||||
const appOutDir = path.join(tempRoot, 'win-unpacked')
|
||||
fs.mkdirSync(appOutDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(appOutDir, 'LICENSE.electron.txt'), 'x', 'utf8')
|
||||
|
||||
assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), false)
|
||||
// Tree untouched; the caller's wipe path handles it.
|
||||
assert.equal(fs.existsSync(appOutDir), true)
|
||||
assert.equal(fs.existsSync(`${appOutDir}.bak`), false)
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('preserveRollbackBackup ignores missing or invalid input', () => {
|
||||
assert.equal(preserveRollbackBackup(''), false)
|
||||
assert.equal(preserveRollbackBackup(undefined), false)
|
||||
assert.equal(preserveRollbackBackup(null), false)
|
||||
assert.equal(preserveRollbackBackup(path.join(os.tmpdir(), 'does-not-exist-xyz')), false)
|
||||
})
|
||||
|
||||
test('beforePack on win32 preserves the previous build instead of wiping it', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
try {
|
||||
const appOutDir = path.join(tempRoot, 'win-unpacked')
|
||||
fs.mkdirSync(appOutDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-working', 'utf8')
|
||||
|
||||
// No packager info in the context → default 'Hermes.exe' product name.
|
||||
// node-pty staging is skipped because arch is not a number here.
|
||||
await beforePack({ appOutDir, electronPlatformName: 'win32' })
|
||||
|
||||
assert.equal(fs.existsSync(appOutDir), false)
|
||||
assert.equal(
|
||||
fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'),
|
||||
'MZ-working'
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('beforePack on linux keeps the plain wipe (no .bak)', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-'))
|
||||
try {
|
||||
const appOutDir = path.join(tempRoot, 'linux-unpacked')
|
||||
fs.mkdirSync(appOutDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'x', 'utf8')
|
||||
|
||||
await beforePack({ appOutDir, electronPlatformName: 'linux' })
|
||||
|
||||
assert.equal(fs.existsSync(appOutDir), false)
|
||||
assert.equal(fs.existsSync(`${appOutDir}.bak`), false)
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -4,16 +4,20 @@
|
|||
import coldStart from './cold-start.mjs'
|
||||
import firstToken from './first-token.mjs'
|
||||
import keystroke from './keystroke.mjs'
|
||||
import multitab from './multitab.mjs'
|
||||
import profileSwitch from './profile-switch.mjs'
|
||||
import sessionSwitch from './session-switch.mjs'
|
||||
import stream from './stream.mjs'
|
||||
import streamHistory from './stream-history.mjs'
|
||||
import submit from './submit.mjs'
|
||||
import transcript from './transcript.mjs'
|
||||
|
||||
export const SCENARIOS = {
|
||||
[stream.name]: stream,
|
||||
[streamHistory.name]: streamHistory,
|
||||
[keystroke.name]: keystroke,
|
||||
[transcript.name]: transcript,
|
||||
[multitab.name]: multitab,
|
||||
[coldStart.name]: coldStart,
|
||||
[firstToken.name]: firstToken,
|
||||
[submit.name]: submit,
|
||||
|
|
|
|||
246
apps/desktop/scripts/perf/scenarios/multitab.mjs
Normal file
246
apps/desktop/scripts/perf/scenarios/multitab.mjs
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
// Multi-tab working sessions: N session tiles stacked as tabs in the main
|
||||
// zone, EVERY tab mounted (keep-alive), all streaming concurrently — the
|
||||
// "5 tabs doing PR review" workload. Measures frame pacing + longtasks while
|
||||
// the whole stack streams, which is where multitab renderers crawl.
|
||||
//
|
||||
// Drives the real pipeline synthetically (no backend, no credits):
|
||||
// publishSessionState per session per flush — exactly what the gateway's
|
||||
// delta flush does — via the __HERMES_SESSION_TILES__ hook.
|
||||
//
|
||||
// node scripts/perf/run.mjs multitab --spawn [--tiles 5] [--tokens 240]
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
import { frameHistogram, percentile } from '../lib/stats.mjs'
|
||||
|
||||
// Same recorder pattern as stream.mjs (generation-guarded rAF + longtasks).
|
||||
const RECORDERS = `
|
||||
(() => {
|
||||
window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1
|
||||
const ftGen = window.__FT_GEN__
|
||||
window.__FT__ = { times: [], stop: false }
|
||||
let last = performance.now()
|
||||
const tick = () => {
|
||||
if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return
|
||||
const now = performance.now()
|
||||
window.__FT__.times.push(now - last)
|
||||
last = now
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
|
||||
window.__LT__ = { entries: [], stop: false }
|
||||
try {
|
||||
const po = new PerformanceObserver((list) => {
|
||||
if (window.__LT__.stop) return
|
||||
for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime })
|
||||
})
|
||||
po.observe({ entryTypes: ['longtask'] })
|
||||
window.__LT__.po = po
|
||||
} catch {}
|
||||
return 'armed'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT = `
|
||||
(() => {
|
||||
window.__FT__.stop = true
|
||||
window.__LT__.stop = true
|
||||
try { window.__LT__.po && window.__LT__.po.disconnect() } catch {}
|
||||
return JSON.stringify({ frames: window.__FT__.times, longtasks: window.__LT__.entries })
|
||||
})()
|
||||
`
|
||||
|
||||
/** Page-side setup: open `tiles` session tiles stacked into the main zone,
|
||||
* bind fake runtime ids, and seed each with a realistic transcript. */
|
||||
const setup = (tiles, seedTurns, streamSeed) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (!hook) return 'no-hook'
|
||||
|
||||
const turn = (sid, i) => ([
|
||||
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
|
||||
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff in module ' + i + ' handle the error path?' }] },
|
||||
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: [
|
||||
'## Finding ' + i, '',
|
||||
'The handler swallows the rejection. Key points for hunk \\\`' + i + '\\\`:', '',
|
||||
'- The catch block drops the original error.',
|
||||
'- Retries are unbounded — see [the loop](https://example.com/loop).', '',
|
||||
'\\\`\\\`\\\`ts',
|
||||
'async function retry' + i + '(fn: () => Promise<void>) {',
|
||||
' for (;;) { try { return await fn() } catch {} }',
|
||||
'}',
|
||||
'\\\`\\\`\\\`', '',
|
||||
'| path | covered |', '|---|---|', '| happy | yes |', '| error | no |', ''
|
||||
].join('\\n') }] }
|
||||
])
|
||||
|
||||
const state = (sid, rid) => {
|
||||
const messages = []
|
||||
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
|
||||
// Streaming tail the driver grows (--code seeds an open fence).
|
||||
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
|
||||
parts: [{ type: 'text', text: ${JSON.stringify(streamSeed)} }] })
|
||||
return {
|
||||
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
|
||||
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
|
||||
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
|
||||
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
|
||||
needsInput: false, turnStartedAt: Date.now(), usage: null
|
||||
}
|
||||
}
|
||||
|
||||
window.__MT__ = { ids: [], timer: null }
|
||||
for (let n = 1; n <= ${tiles}; n++) {
|
||||
const sid = 'perf-tile-' + n
|
||||
const rid = 'perf-rt-' + n
|
||||
window.__MT__.ids.push({ sid, rid })
|
||||
hook.open(sid, 'center')
|
||||
hook.patch(sid, { runtimeId: rid })
|
||||
hook.publish(rid, state(sid, rid))
|
||||
}
|
||||
return 'ok'
|
||||
})()
|
||||
`
|
||||
|
||||
// Activate every tab once so keep-alive mounts the full stack (lazy mount:
|
||||
// a never-activated tab stays unmounted, which would understate the cost).
|
||||
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
|
||||
|
||||
/** Page-side driver: grow every tile's streaming tail by `chunk` each
|
||||
* `intervalMs`, through the same publish path the gateway flush uses. */
|
||||
const drive = (chunk, intervalMs, totalTokens) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
let pushed = 0
|
||||
const tick = () => {
|
||||
const states = hook.states()
|
||||
for (const { rid } of window.__MT__.ids) {
|
||||
const prev = states[rid]
|
||||
if (!prev) continue
|
||||
const messages = prev.messages.map(m => {
|
||||
if (m.id !== prev.streamId) return m
|
||||
const head = m.parts.slice(0, -1)
|
||||
const last = m.parts[m.parts.length - 1]
|
||||
return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] }
|
||||
})
|
||||
hook.publish(rid, { ...prev, messages })
|
||||
}
|
||||
pushed += 1
|
||||
if (pushed < ${totalTokens}) window.__MT__.timer = setTimeout(tick, ${intervalMs})
|
||||
else window.__MT__.done = true
|
||||
}
|
||||
window.__MT__.timer = setTimeout(tick, ${intervalMs})
|
||||
return 'driving'
|
||||
})()
|
||||
`
|
||||
|
||||
const CLEANUP = `
|
||||
(() => {
|
||||
if (window.__MT__) {
|
||||
clearTimeout(window.__MT__.timer)
|
||||
for (const { sid, rid } of window.__MT__.ids) {
|
||||
window.__HERMES_SESSION_TILES__.publish(rid, {
|
||||
...window.__HERMES_SESSION_TILES__.states()[rid], busy: false, streamId: null
|
||||
})
|
||||
window.__HERMES_SESSION_TILES__.close(sid)
|
||||
}
|
||||
window.__MT__ = null
|
||||
}
|
||||
return 'cleaned'
|
||||
})()
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'multitab',
|
||||
tier: 'ci',
|
||||
description: 'N mounted session-tile tabs all streaming: frame pacing + longtasks.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tiles = Number(opts.tiles ?? 5)
|
||||
const seedTurns = Number(opts.turns ?? 20)
|
||||
const tokens = Number(opts.tokens ?? 240)
|
||||
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
|
||||
const intervalMs = Number(opts.intervalMs ?? 33)
|
||||
// --code: every tile grows ONE giant fenced code block with no settle
|
||||
// boundaries — what a coding agent streams. The block re-parses and
|
||||
// re-renders fully every flush (block memoization can't settle it), the
|
||||
// documented worst case and the "5 tabs all coding" crawl.
|
||||
const chunk = opts.code
|
||||
? ' const value = await resolve(ctx, { retry: true }) // step\n'
|
||||
: (opts.chunk ?? 'A streamed review sentence with **bold**, `code`, and ordinary prose.\n\n')
|
||||
const streamSeed = opts.code ? '```ts\n' : ''
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ok = await cdp.eval(setup(tiles, seedTurns, streamSeed))
|
||||
|
||||
if (ok !== 'ok') {
|
||||
throw new Error(`multitab setup failed (${ok}) — dev hooks missing? (needs a dev/probe renderer)`)
|
||||
}
|
||||
|
||||
// Mount every tab (keep-alive mounts on first activation), then settle.
|
||||
for (let n = 1; n <= tiles; n++) {
|
||||
await cdp.eval(reveal(`perf-tile-${n}`))
|
||||
await sleep(350)
|
||||
}
|
||||
|
||||
await sleep(1000)
|
||||
await cdp.eval(RECORDERS)
|
||||
await cdp.eval(drive(chunk, intervalMs, tokens))
|
||||
await sleep(tokens * intervalMs + 1500)
|
||||
|
||||
const data = JSON.parse(await cdp.eval(COLLECT))
|
||||
await cdp.eval(CLEANUP)
|
||||
|
||||
// Drop the first 500ms (recorder install + settle).
|
||||
const frames = []
|
||||
let acc = 0
|
||||
|
||||
for (const f of data.frames) {
|
||||
acc += f
|
||||
|
||||
if (acc >= 500) {
|
||||
frames.push(f)
|
||||
}
|
||||
}
|
||||
|
||||
const ltDurations = data.longtasks.map(e => e.duration)
|
||||
const windowS = frames.reduce((a, b) => a + b, 0) / 1000
|
||||
// The felt numbers: sustained fps over the window, and the fps of the
|
||||
// worst 1-second slice (a 333ms frame IS "3fps" even if the average looks
|
||||
// fine). Worst slice = max summed frame time in any sliding 1s window.
|
||||
const avgFps = windowS ? frames.length / windowS : 0
|
||||
let worstFps = avgFps
|
||||
|
||||
for (let i = 0, j = 0, sum = 0; j < frames.length; j++) {
|
||||
sum += frames[j]
|
||||
|
||||
while (sum > 1000) {
|
||||
sum -= frames[i++]
|
||||
}
|
||||
|
||||
// Only a window that actually spans ~1s counts; short prefixes don't.
|
||||
if (sum >= 900) {
|
||||
worstFps = Math.min(worstFps, ((j - i + 1) / sum) * 1000)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
longtasks_n: data.longtasks.length,
|
||||
longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10,
|
||||
frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10,
|
||||
frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10,
|
||||
slow_frames_33: frames.filter(f => f > 33).length
|
||||
},
|
||||
detail: {
|
||||
tiles,
|
||||
code: Boolean(opts.code),
|
||||
windowS: Math.round(windowS * 10) / 10,
|
||||
avgFps: Math.round(avgFps * 10) / 10,
|
||||
worstSecondFps: Math.round(worstFps * 10) / 10,
|
||||
frameHistogram: frameHistogram(frames)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
18
apps/desktop/scripts/perf/scenarios/stream-history.mjs
Normal file
18
apps/desktop/scripts/perf/scenarios/stream-history.mjs
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but
|
||||
// the history is mounted and allowed to settle before the recorders start, so
|
||||
// what it captures is the per-delta cost that scales with transcript length —
|
||||
// the regression reported in #69120.
|
||||
//
|
||||
// Report-only (tier: manual): the number depends on how much history the host
|
||||
// can mount, so it is not gated against the committed baseline.
|
||||
|
||||
import stream from './stream.mjs'
|
||||
|
||||
export default {
|
||||
name: 'stream-history',
|
||||
tier: 'manual',
|
||||
description: 'Streaming cost with a long settled transcript already mounted.',
|
||||
run(cdp, opts = {}) {
|
||||
return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) })
|
||||
}
|
||||
}
|
||||
|
|
@ -70,7 +70,7 @@ const COLLECT = `
|
|||
})()
|
||||
`
|
||||
|
||||
function analyze(data, warmupMs) {
|
||||
function analyze(data, warmupMs, extra = {}) {
|
||||
// Drop warm-up frames (recorder installs before the stream starts).
|
||||
const frames = []
|
||||
let acc = 0
|
||||
|
|
@ -102,6 +102,7 @@ function analyze(data, warmupMs) {
|
|||
intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10
|
||||
},
|
||||
detail: {
|
||||
...extra,
|
||||
windowS: Math.round(windowS * 10) / 10,
|
||||
avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0,
|
||||
frameHistogram: frameHistogram(frames),
|
||||
|
|
@ -128,13 +129,36 @@ export default {
|
|||
// noise unrelated to render cost).
|
||||
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
|
||||
const real = Boolean(opts.real)
|
||||
const historyTurns = Number(opts.historyTurns ?? 0)
|
||||
const historySettleMs = Number(opts.historySettleMs ?? 1500)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
// Mount the settled history BEFORE the recorders start, so the measurement
|
||||
// window contains only streaming work — not the one-off mount cost.
|
||||
if (historyTurns > 0) {
|
||||
if (real) {
|
||||
throw new Error('--historyTurns is only supported by the synthetic stream path')
|
||||
}
|
||||
|
||||
await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`)
|
||||
await sleep(historySettleMs)
|
||||
|
||||
const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()'))
|
||||
const expected = historyTurns * 2
|
||||
|
||||
if (mounted !== expected) {
|
||||
throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`)
|
||||
}
|
||||
}
|
||||
|
||||
await cdp.eval(RECORDERS)
|
||||
|
||||
if (real) {
|
||||
// Backend path: fire a real prompt and wait for the stream to appear.
|
||||
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
|
||||
const baseCount = await cdp.eval(
|
||||
`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`
|
||||
)
|
||||
await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 })
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
|
|
@ -186,6 +210,6 @@ export default {
|
|||
await cdp.eval('window.__PERF_DRIVE__.reset()')
|
||||
}
|
||||
|
||||
return analyze(data, real ? 0 : 500)
|
||||
return analyze(data, real ? 0 : 500, { historyTurns })
|
||||
}
|
||||
}
|
||||
|
|
|
|||
80
apps/desktop/scripts/probe-model-picker.mjs
Normal file
80
apps/desktop/scripts/probe-model-picker.mjs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
// Model picker open latency probe: click the composer model pill, time until
|
||||
// the dropdown/dialog content paints, repeat. Run with --cpuprofile via the
|
||||
// harness runner once promoted; standalone for iteration.
|
||||
// node scripts/perf/probe-model-picker.mjs [--port 9222] [--rounds 5]
|
||||
import { CDP } from './perf/lib/cdp.mjs'
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const flag = name => {
|
||||
const i = args.indexOf(`--${name}`)
|
||||
return i >= 0 ? args[i + 1] : undefined
|
||||
}
|
||||
const port = Number(flag('port') ?? 9222)
|
||||
const rounds = Number(flag('rounds') ?? 5)
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms))
|
||||
|
||||
const cdp = await CDP.connect({ port })
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
// Find the pill (dropdown path) — the composer model selector button.
|
||||
const PILL = `(() => {
|
||||
const btns = [...document.querySelectorAll('button[aria-label]')]
|
||||
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || '') && b.closest('[data-slot]'))
|
||||
return pill ? (pill.getAttribute('aria-label') || 'found') : null
|
||||
})()`
|
||||
|
||||
console.log('pill:', await cdp.eval(PILL))
|
||||
|
||||
const MEASURE = `
|
||||
(async () => {
|
||||
const btns = [...document.querySelectorAll('button[aria-label]')]
|
||||
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || ''))
|
||||
if (!pill) return JSON.stringify({ error: 'no pill' })
|
||||
|
||||
const t0 = performance.now()
|
||||
pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true }))
|
||||
pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true }))
|
||||
pill.click()
|
||||
|
||||
// Wait for menu/dialog content to exist AND paint (double rAF after found).
|
||||
const found = await new Promise(resolve => {
|
||||
const deadline = performance.now() + 5000
|
||||
const check = () => {
|
||||
const menu = document.querySelector('[role="menu"], [role="dialog"] [cmdk-list]')
|
||||
if (menu && menu.childElementCount > 0) {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now())))
|
||||
return
|
||||
}
|
||||
if (performance.now() > deadline) { resolve(null); return }
|
||||
requestAnimationFrame(check)
|
||||
}
|
||||
check()
|
||||
})
|
||||
|
||||
const openMs = found ? found - t0 : null
|
||||
const rows = document.querySelectorAll('[role="menu"] [role="menuitem"], [cmdk-item]').length
|
||||
|
||||
// Close: Escape.
|
||||
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
|
||||
const menu = document.querySelector('[role="menu"], [role="dialog"]')
|
||||
menu?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
|
||||
await new Promise(r => setTimeout(r, 300))
|
||||
|
||||
return JSON.stringify({ openMs, rows })
|
||||
})()
|
||||
`
|
||||
|
||||
const samples = []
|
||||
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
const raw = await cdp.eval(MEASURE, { awaitPromise: true })
|
||||
const r = JSON.parse(raw)
|
||||
console.log(`round ${i}:`, r)
|
||||
if (typeof r.openMs === 'number') samples.push(r.openMs)
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
samples.sort((a, b) => a - b)
|
||||
console.log('\nopen latency ms — min/median/max:',
|
||||
Math.round(samples[0]), '/', Math.round(samples[Math.floor(samples.length / 2)]), '/', Math.round(samples.at(-1)))
|
||||
cdp.close()
|
||||
60
apps/desktop/scripts/profile-model-picker.mjs
Normal file
60
apps/desktop/scripts/profile-model-picker.mjs
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
// CPU-profile one model-picker open.
|
||||
// node scripts/profile-model-picker.mjs [--port 9222]
|
||||
import { writeFileSync } from 'node:fs'
|
||||
|
||||
import { CDP } from './perf/lib/cdp.mjs'
|
||||
import { cpuProfileTopSelf } from './perf/lib/stats.mjs'
|
||||
|
||||
const port = Number(process.argv.includes('--port') ? process.argv[process.argv.indexOf('--port') + 1] : 9222)
|
||||
const cdp = await CDP.connect({ port })
|
||||
await cdp.send('Runtime.enable')
|
||||
await cdp.send('Profiler.enable')
|
||||
await cdp.send('Profiler.setSamplingInterval', { interval: 100 })
|
||||
|
||||
const OPEN = `
|
||||
(async () => {
|
||||
// Reset: close any open menu first.
|
||||
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
|
||||
await new Promise(r => setTimeout(r, 250))
|
||||
|
||||
const btns = [...document.querySelectorAll('button[aria-label]')]
|
||||
const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || ''))
|
||||
if (!pill) return -1
|
||||
const t0 = performance.now()
|
||||
pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true }))
|
||||
pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true }))
|
||||
pill.click()
|
||||
const found = await new Promise(resolve => {
|
||||
const deadline = performance.now() + 8000
|
||||
const check = () => {
|
||||
const menu = document.querySelector('[role="menu"]')
|
||||
if (menu && menu.childElementCount > 0) {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now())))
|
||||
return
|
||||
}
|
||||
if (performance.now() > deadline) { resolve(-1); return }
|
||||
requestAnimationFrame(check)
|
||||
}
|
||||
check()
|
||||
})
|
||||
return found < 0 ? -1 : found - t0
|
||||
})()
|
||||
`
|
||||
|
||||
await cdp.send('Profiler.start')
|
||||
const openMs = await cdp.eval(OPEN)
|
||||
const { profile } = await cdp.send('Profiler.stop')
|
||||
|
||||
console.log('openMs:', Math.round(openMs))
|
||||
const out = `/tmp/model-picker-open.cpuprofile`
|
||||
writeFileSync(out, JSON.stringify(profile))
|
||||
console.log('wrote', out)
|
||||
console.log('top self-time (ms):')
|
||||
|
||||
for (const r of cpuProfileTopSelf(profile, 20)) {
|
||||
console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(44)} ${r.url.split('/').slice(-2).join('/')}:${r.line}`)
|
||||
}
|
||||
|
||||
// Close the menu again.
|
||||
await cdp.eval(`document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))`)
|
||||
cdp.close()
|
||||
|
|
@ -50,6 +50,9 @@ export function slashChipKindForItem(item: Unstable_TriggerItem): SlashChipKind
|
|||
return 'command'
|
||||
}
|
||||
|
||||
/** True for a skill completion — the only kind offered mid-message. */
|
||||
export const isSkillItem = (item: Unstable_TriggerItem) => slashChipKindForItem(item) === 'skill'
|
||||
|
||||
/** A `/` query is at its arg stage once it's past the command name. */
|
||||
export const slashArgStage = (query: string) => query.includes(' ')
|
||||
|
||||
|
|
|
|||
50
apps/desktop/src/app/chat/composer/focus.test.ts
Normal file
50
apps/desktop/src/app/chat/composer/focus.test.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
|
||||
import { blurComposerInput } from './focus'
|
||||
import { RICH_INPUT_SLOT } from './rich-editor'
|
||||
|
||||
/**
|
||||
* Inactive tabs keep their composer mounted, so an unscoped lookup can blur a
|
||||
* background input and leave the one the user is typing in focused.
|
||||
*/
|
||||
|
||||
/** A composer input inside its own pane layer, hidden or not. */
|
||||
function mountInput(hidden = false) {
|
||||
const layer = document.createElement('div')
|
||||
const input = document.createElement('div')
|
||||
input.dataset.slot = RICH_INPUT_SLOT
|
||||
input.tabIndex = 0
|
||||
layer.toggleAttribute('data-pane-hidden', hidden)
|
||||
layer.append(input)
|
||||
document.body.append(layer)
|
||||
|
||||
return input
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = ''
|
||||
})
|
||||
|
||||
describe('blurComposerInput', () => {
|
||||
it('blurs the foreground composer while a hidden tab matches first', () => {
|
||||
const background = mountInput(true)
|
||||
const foreground = mountInput()
|
||||
|
||||
foreground.focus()
|
||||
blurComposerInput()
|
||||
|
||||
expect(document.activeElement).not.toBe(foreground)
|
||||
expect(document.activeElement).not.toBe(background)
|
||||
})
|
||||
|
||||
it('leaves focus alone when the composer does not hold it', () => {
|
||||
const outside = document.createElement('button')
|
||||
document.body.append(outside)
|
||||
mountInput()
|
||||
|
||||
outside.focus()
|
||||
blurComposerInput()
|
||||
|
||||
expect(document.activeElement).toBe(outside)
|
||||
})
|
||||
})
|
||||
|
|
@ -10,6 +10,8 @@
|
|||
* steal focus from the composer effect.
|
||||
*/
|
||||
|
||||
import { queryVisible } from '@/components/pane-shell/pane-visibility'
|
||||
|
||||
import type { InlineRefInput } from './inline-refs'
|
||||
import { RICH_INPUT_SLOT } from './rich-editor'
|
||||
|
||||
|
|
@ -175,9 +177,11 @@ export const focusComposerInput = (el: HTMLElement | null) => {
|
|||
window.setTimeout(focus, 0)
|
||||
}
|
||||
|
||||
/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). */
|
||||
/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.).
|
||||
* Skips inactive tabs — they stay mounted, so an unscoped lookup can land on a
|
||||
* background composer and leave the visible one focused. */
|
||||
export const blurComposerInput = () => {
|
||||
const el = document.querySelector(`[data-slot="${RICH_INPUT_SLOT}"]`) as HTMLElement | null
|
||||
const el = queryVisible(`[data-slot="${RICH_INPUT_SLOT}"]`)
|
||||
|
||||
if (el && document.activeElement === el) {
|
||||
el.blur()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
import { act, cleanup, render } from '@testing-library/react'
|
||||
import { useLayoutEffect } from 'react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { type ComposerAttachment, mainComposerScope, stashSessionDraft } from '@/store/composer'
|
||||
|
||||
import type { QueueEditState } from '../composer-utils'
|
||||
|
||||
import { useComposerDraft } from './use-composer-draft'
|
||||
|
||||
const mockComposerApi = { setText: vi.fn() }
|
||||
|
||||
vi.mock('@assistant-ui/react', () => ({
|
||||
useAui: () => ({ composer: () => mockComposerApi }),
|
||||
useAuiState: (selector: (state: { composer: { text: string } }) => unknown) => selector({ composer: { text: '' } }),
|
||||
useComposerRuntime: () => ({
|
||||
getState: () => ({ text: '' }),
|
||||
subscribe: () => () => undefined
|
||||
})
|
||||
}))
|
||||
|
||||
interface ProbeHarnessProps {
|
||||
activeQueueSessionKey: string | null
|
||||
onLayoutSnapshot: (attachments: ComposerAttachment[]) => void
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
function ProbeHarness({ activeQueueSessionKey, onLayoutSnapshot, sessionId }: ProbeHarnessProps) {
|
||||
useComposerDraft({
|
||||
activeQueueSessionKey,
|
||||
focusKey: null,
|
||||
inputDisabled: false,
|
||||
queueEditRef: { current: null as QueueEditState | null },
|
||||
sessionId
|
||||
})
|
||||
|
||||
// useLayoutEffect fires synchronously right after the DOM commit, BEFORE
|
||||
// the hook's per-thread scope-swap useEffect (a passive effect) has a
|
||||
// chance to swap attachmentScope.$attachments over to the new session. A
|
||||
// synchronous read here — the same read ChatBar's `attachments` prop
|
||||
// performs at render time — observes the OUTGOING session's attachments.
|
||||
useLayoutEffect(() => {
|
||||
onLayoutSnapshot(mainComposerScope.$attachments.get())
|
||||
})
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
describe('useComposerDraft — attachment scope stays coherent with the committed session on switch (#59305)', () => {
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
mainComposerScope.clear()
|
||||
})
|
||||
|
||||
it('clears the outgoing session attachments by the layout phase right after switching sessions', () => {
|
||||
const attachmentA: ComposerAttachment = { id: 'url-A', kind: 'url', label: 'A' }
|
||||
stashSessionDraft('session-A', 'hi from A', [attachmentA])
|
||||
|
||||
const snapshots: ComposerAttachment[][] = []
|
||||
|
||||
const { rerender } = render(
|
||||
<ProbeHarness activeQueueSessionKey="session-A" onLayoutSnapshot={s => snapshots.push(s)} sessionId="session-A" />
|
||||
)
|
||||
|
||||
// Mount loads session A's stashed attachment into the (module-level) main
|
||||
// scope — confirms the fixture actually seeded the leak precondition.
|
||||
expect(mainComposerScope.$attachments.get()).toEqual([attachmentA])
|
||||
|
||||
snapshots.length = 0 // drop the initial-mount snapshot; only the switch matters
|
||||
|
||||
act(() => {
|
||||
rerender(
|
||||
<ProbeHarness
|
||||
activeQueueSessionKey="session-B"
|
||||
onLayoutSnapshot={s => snapshots.push(s)}
|
||||
sessionId="session-B"
|
||||
/>
|
||||
)
|
||||
})
|
||||
|
||||
// By the layout phase the scope must already be B's (empty) — a submit
|
||||
// fired the instant B renders must never ship session A's attachment.
|
||||
expect(snapshots[0]).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('useComposerDraft — rehydrate diagnostic log stays redacted', () => {
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
mainComposerScope.clear()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('logs counts/kinds/scope on restore but never the raw url, refText, or label', () => {
|
||||
const secretUrl = 'https://secret.example.com/private-workspace-path'
|
||||
|
||||
const attachment: ComposerAttachment = {
|
||||
id: 'url-secret',
|
||||
kind: 'url',
|
||||
label: 'do-not-leak-label',
|
||||
refText: `@url:${secretUrl}`
|
||||
}
|
||||
|
||||
stashSessionDraft('session-secret', '', [attachment])
|
||||
|
||||
const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined)
|
||||
|
||||
render(
|
||||
<ProbeHarness
|
||||
activeQueueSessionKey="session-secret"
|
||||
onLayoutSnapshot={() => undefined}
|
||||
sessionId="session-secret"
|
||||
/>
|
||||
)
|
||||
|
||||
const rehydrateCalls = debugSpy.mock.calls.filter(call => call[0] === '[composer-rehydrate]')
|
||||
expect(rehydrateCalls.length).toBeGreaterThan(0)
|
||||
|
||||
const serialized = JSON.stringify(rehydrateCalls)
|
||||
expect(serialized).not.toContain(secretUrl)
|
||||
expect(serialized).not.toContain(attachment.label)
|
||||
expect(serialized).not.toContain(attachment.refText)
|
||||
|
||||
expect(rehydrateCalls[0]?.[1]).toMatchObject({
|
||||
attachmentCount: 1,
|
||||
attachmentKinds: ['url'],
|
||||
scope: 'session-secret'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { useAui, useAuiState, useComposerRuntime } from '@assistant-ui/react'
|
||||
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { type RefObject, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
|
||||
import { SLASH_COMMAND_RE } from '@/lib/chat-runtime'
|
||||
import { type ComposerAttachment, stashSessionDraft, takeSessionDraft } from '@/store/composer'
|
||||
|
|
@ -20,7 +20,7 @@ import {
|
|||
onComposerInsertRequest
|
||||
} from '../focus'
|
||||
import { type InlineRefInput, insertInlineRefsIntoEditor } from '../inline-refs'
|
||||
import { composerPlainText, placeCaretEnd, renderComposerContents } from '../rich-editor'
|
||||
import { composerPlainText, placeCaretEnd, REF_RE, renderComposerContents } from '../rich-editor'
|
||||
import { useComposerScope } from '../scope'
|
||||
import type { ChatBarProps } from '../types'
|
||||
|
||||
|
|
@ -189,6 +189,21 @@ export function useComposerDraft({
|
|||
stashSessionDraft(scope, text, attachments)
|
||||
|
||||
const loadIntoComposer = (text: string, attachments: ComposerAttachment[]) => {
|
||||
// Diagnostic breadcrumb for #59305-class reports: identifies WHAT kind of
|
||||
// state got restored into the composer (session switch, queue-edit
|
||||
// restore, history browse) without logging any raw content. REF_RE has the
|
||||
// global flag — testing against a throwaway clone avoids mutating the
|
||||
// shared instance's lastIndex, which would otherwise corrupt this check on
|
||||
// the next call.
|
||||
if (attachments.length > 0 || new RegExp(REF_RE.source, REF_RE.flags).test(text)) {
|
||||
console.debug('[composer-rehydrate]', {
|
||||
attachmentCount: attachments.length,
|
||||
attachmentKinds: attachments.map(a => a.kind),
|
||||
hasTextRefs: new RegExp(REF_RE.source, REF_RE.flags).test(text),
|
||||
scope: activeQueueSessionKeyRef.current
|
||||
})
|
||||
}
|
||||
|
||||
attachmentScope.$attachments.set(cloneAttachments(attachments))
|
||||
paintDraft(text, false)
|
||||
}
|
||||
|
|
@ -231,6 +246,7 @@ export function useComposerDraft({
|
|||
// source otherwise), and (3) schedule the debounced per-session stash.
|
||||
// Browsing history / editing a queued prompt suppress the stash so recalled
|
||||
// text never clobbers the draft.
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
const sync = () => {
|
||||
const text = composerRuntime.getState().text
|
||||
|
|
@ -318,7 +334,17 @@ export function useComposerDraft({
|
|||
// Per-thread draft swap — the composer's only session coupling. Lifecycle
|
||||
// never clears composer state; this effect alone stashes on leave, restores
|
||||
// on enter. Keyed writes are idempotent, so no skip-sentinel.
|
||||
useEffect(() => {
|
||||
//
|
||||
// MUST be a layout effect, not a passive one: it swaps attachmentScope's
|
||||
// module-level $attachments atom, and a passive effect fires only after the
|
||||
// browser paints the new session's view — leaving a window where the DOM
|
||||
// already shows session B while $attachments (and therefore ChatBar's
|
||||
// `attachments` prop) still holds session A's chips. A submit fired in that
|
||||
// window (e.g. a fast session switch immediately followed by Enter) would
|
||||
// ship A's attachments into B's turn (#59305). useLayoutEffect closes the
|
||||
// window by running before paint.
|
||||
|
||||
useLayoutEffect(() => {
|
||||
// A pending debounce timer from the outgoing session is now stale — its
|
||||
// scope was correct when scheduled, but the authoritative stash below
|
||||
// (and the cleanup on the way out) already covers that text. Letting it
|
||||
|
|
@ -344,6 +370,7 @@ export function useComposerDraft({
|
|||
|
||||
// pagehide is load-bearing: React skips effect cleanups on reload, so Cmd+R
|
||||
// inside the debounce/rAF window would drop trailing keystrokes without this.
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
const flushPendingDraftPersist = () => {
|
||||
const scope = draftScopeRef.current
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import { useAuiState } from '@assistant-ui/react'
|
||||
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
|
||||
|
||||
import {
|
||||
clearSurfaceVar,
|
||||
COMPOSER_HEIGHT_VAR,
|
||||
COMPOSER_SURFACE_HEIGHT_VAR,
|
||||
setSurfaceVar
|
||||
} from '@/app/chat/surface-vars'
|
||||
import { useMediaQuery } from '@/hooks/use-media-query'
|
||||
import { useResizeObserver } from '@/hooks/use-resize-observer'
|
||||
import { $composerPoppedOut } from '@/store/composer-popout'
|
||||
|
|
@ -89,18 +95,16 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
|
|||
// (Read globals here so the callback stays stable; mirror the popoutAllowed
|
||||
// gate since secondary windows are forced docked.)
|
||||
if ($composerPoppedOut.get() && !isSecondaryWindow()) {
|
||||
const root = document.documentElement
|
||||
lastBucketedHeightRef.current = 0
|
||||
lastBucketedSurfaceHeightRef.current = 0
|
||||
root.style.setProperty('--composer-measured-height', '0px')
|
||||
root.style.setProperty('--composer-surface-measured-height', '0px')
|
||||
setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, '0px')
|
||||
setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, '0px')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
const { height, width } = composer.getBoundingClientRect()
|
||||
const surfaceHeight = composerSurfaceRef.current?.getBoundingClientRect().height
|
||||
const root = document.documentElement
|
||||
|
||||
if (width > 0) {
|
||||
const nextTight = width < COMPOSER_STACK_BREAKPOINT_PX
|
||||
|
|
@ -135,7 +139,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
|
|||
|
||||
if (bucket !== lastBucketedHeightRef.current) {
|
||||
lastBucketedHeightRef.current = bucket
|
||||
root.style.setProperty('--composer-measured-height', `${bucket}px`)
|
||||
setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, `${bucket}px`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -144,7 +148,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
|
|||
|
||||
if (bucket !== lastBucketedSurfaceHeightRef.current) {
|
||||
lastBucketedSurfaceHeightRef.current = bucket
|
||||
root.style.setProperty('--composer-surface-measured-height', `${bucket}px`)
|
||||
setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, `${bucket}px`)
|
||||
}
|
||||
}
|
||||
}, [composerRef, composerSurfaceRef, editorRef])
|
||||
|
|
@ -160,12 +164,13 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef,
|
|||
}, [poppedOut, syncComposerMetrics])
|
||||
|
||||
useEffect(() => {
|
||||
const composer = composerRef.current
|
||||
|
||||
return () => {
|
||||
const root = document.documentElement
|
||||
root.style.removeProperty('--composer-measured-height')
|
||||
root.style.removeProperty('--composer-surface-measured-height')
|
||||
clearSurfaceVar(composer, COMPOSER_HEIGHT_VAR)
|
||||
clearSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR)
|
||||
}
|
||||
}, [])
|
||||
}, [composerRef])
|
||||
|
||||
// Pill compacts on real width (tile/pane), OR when stacked for any reason
|
||||
// (viewport-narrow / wrapped) so the controls row never over-runs.
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ export function useComposerPlaceholder({ disabled, reconnecting, sessionId }: Us
|
|||
|
||||
const prevSessionIdRef = useRef(sessionId)
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
const prev = prevSessionIdRef.current
|
||||
prevSessionIdRef.current = sessionId
|
||||
|
|
|
|||
|
|
@ -322,6 +322,7 @@ export function useComposerQueue({
|
|||
// never churns, so a change there is a real session switch and must NOT
|
||||
// migrate; only the runtime-derived key (queueSessionKey falsy → key is
|
||||
// sessionId) churns on a backend bounce/resume of the same conversation.
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
const prev = prevQueueKeyRef.current
|
||||
prevQueueKeyRef.current = activeQueueSessionKey
|
||||
|
|
|
|||
|
|
@ -113,7 +113,9 @@ describe('useComposerSubmit busy-turn routing', () => {
|
|||
hook.result.current.submitDraft()
|
||||
})
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('/compress preserve context'))
|
||||
await waitFor(() =>
|
||||
expect(onSubmit).toHaveBeenCalledWith('/compress preserve context', { composerScope: 'stored-session' })
|
||||
)
|
||||
expect(clearDraft).toHaveBeenCalledTimes(1)
|
||||
expect(onSteer).not.toHaveBeenCalled()
|
||||
expect(queueCurrentDraft).not.toHaveBeenCalled()
|
||||
|
|
@ -159,9 +161,26 @@ describe('useComposerSubmit busy-turn routing', () => {
|
|||
hook.result.current.submitDraft()
|
||||
})
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('ordinary question', { attachments: [] }))
|
||||
await waitFor(() =>
|
||||
expect(onSubmit).toHaveBeenCalledWith('ordinary question', {
|
||||
attachments: [],
|
||||
composerScope: 'stored-session'
|
||||
})
|
||||
)
|
||||
expect(onSteer).not.toHaveBeenCalled()
|
||||
expect(queueCurrentDraft).not.toHaveBeenCalled()
|
||||
expect(onCancel).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('threads the loaded composer scope through onSubmit for the #59305 submit-time guard', async () => {
|
||||
const { hook, onSubmit } = renderSubmitHook({ text: 'hello' })
|
||||
|
||||
act(() => {
|
||||
hook.result.current.submitDraft()
|
||||
})
|
||||
|
||||
await waitFor(() =>
|
||||
expect(onSubmit).toHaveBeenCalledWith('hello', expect.objectContaining({ composerScope: 'stored-session' }))
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -89,7 +89,11 @@ export function useComposerSubmit({
|
|||
stashAt(submittedScope, text, submittedAttachments)
|
||||
}
|
||||
|
||||
void Promise.resolve(attachments ? onSubmit(text, { attachments }) : onSubmit(text))
|
||||
void Promise.resolve(
|
||||
attachments
|
||||
? onSubmit(text, { attachments, composerScope: submittedScope })
|
||||
: onSubmit(text, { composerScope: submittedScope })
|
||||
)
|
||||
.then(accepted => void (accepted === false ? restore() : clearSessionDraft(submittedScope)))
|
||||
.catch(restore)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core'
|
||||
import { act, renderHook } from '@testing-library/react'
|
||||
import { createRef } from 'react'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { composerPlainText, renderComposerContents, RICH_INPUT_SLOT } from '../rich-editor'
|
||||
|
||||
import { useComposerTrigger } from './use-composer-trigger'
|
||||
|
||||
/** A live contentEditable seeded with `text`, caret parked at the end. */
|
||||
function mountEditor(text: string) {
|
||||
const editor = document.createElement('div')
|
||||
editor.dataset.slot = RICH_INPUT_SLOT
|
||||
editor.contentEditable = 'true'
|
||||
document.body.append(editor)
|
||||
renderComposerContents(editor, text)
|
||||
|
||||
const range = document.createRange()
|
||||
range.selectNodeContents(editor)
|
||||
range.collapse(false)
|
||||
const selection = window.getSelection()!
|
||||
selection.removeAllRanges()
|
||||
selection.addRange(range)
|
||||
|
||||
return editor
|
||||
}
|
||||
|
||||
const item = (command: string, group = 'Skills'): Unstable_TriggerItem => ({
|
||||
id: command,
|
||||
type: 'slash',
|
||||
label: command.slice(1),
|
||||
metadata: { command, display: command, meta: '', group, action: '', rawText: command }
|
||||
})
|
||||
|
||||
function mountTrigger(editor: HTMLDivElement, items: Unstable_TriggerItem[]) {
|
||||
const editorRef = createRef<HTMLDivElement>() as { current: HTMLDivElement | null }
|
||||
editorRef.current = editor
|
||||
|
||||
const draftRef = { current: composerPlainText(editor) }
|
||||
|
||||
const adapter: Unstable_TriggerAdapter = {
|
||||
categories: () => [],
|
||||
categoryItems: () => [],
|
||||
search: () => items
|
||||
}
|
||||
|
||||
const setComposerText = vi.fn()
|
||||
|
||||
const hook = renderHook(() =>
|
||||
useComposerTrigger({
|
||||
at: { adapter: null, loading: false },
|
||||
draftRef,
|
||||
editorRef,
|
||||
requestMainFocus: vi.fn(),
|
||||
setComposerText,
|
||||
slash: { adapter, loading: false }
|
||||
})
|
||||
)
|
||||
|
||||
return { draftRef, hook, setComposerText }
|
||||
}
|
||||
|
||||
describe('useComposerTrigger — slash anywhere in the prompt', () => {
|
||||
it('opens the completion list for a slash typed mid-message', () => {
|
||||
const editor = mountEditor('please run /cle')
|
||||
const { hook } = mountTrigger(editor, [item('/clean')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
|
||||
expect(hook.result.current.trigger).toMatchObject({ kind: '/', inline: true, query: 'cle' })
|
||||
expect(hook.result.current.triggerItems).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('inserts the picked skill inline and keeps the surrounding prose intact', () => {
|
||||
const editor = mountEditor('please run /cle')
|
||||
const { hook } = mountTrigger(editor, [item('/clean')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
act(() => hook.result.current.replaceTriggerWithChip(item('/clean')))
|
||||
|
||||
// The `/cle` the user typed is replaced by the full command; "please run"
|
||||
// in front of it survives untouched.
|
||||
expect(composerPlainText(editor)).toBe('please run /clean ')
|
||||
})
|
||||
|
||||
it('offers only skills mid-message, not app commands', () => {
|
||||
// `/model` and `/new` act on the app — meaningless as a reference in prose.
|
||||
const editor = mountEditor('please run /')
|
||||
const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands'), item('/new', 'Commands')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
|
||||
expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean'])
|
||||
})
|
||||
|
||||
it('still offers the full command set at the start of the prompt', () => {
|
||||
const editor = mountEditor('/')
|
||||
const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
|
||||
expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean', 'model'])
|
||||
})
|
||||
|
||||
it('still opens the list for a slash at the start of the prompt', () => {
|
||||
const editor = mountEditor('/cle')
|
||||
const { hook } = mountTrigger(editor, [item('/clean')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
|
||||
expect(hook.result.current.trigger).toMatchObject({ kind: '/', query: 'cle' })
|
||||
expect(hook.result.current.trigger?.inline).toBeUndefined()
|
||||
})
|
||||
|
||||
it('leaves a mid-message file path alone', () => {
|
||||
const editor = mountEditor('open src/foo/bar')
|
||||
const { hook } = mountTrigger(editor, [item('/clean')])
|
||||
|
||||
act(() => hook.result.current.refreshTrigger())
|
||||
|
||||
expect(hook.result.current.trigger).toBeNull()
|
||||
})
|
||||
})
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue