diff --git a/.gitignore b/.gitignore index 29489633104..8c0ce9c23d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .DS_Store /venv/ /venv.old/ +/venv.stale.runtime-*/ +/.hermes-runtime/ /_pycache/ *.pyc* __pycache__/ diff --git a/acp_adapter/entry.py b/acp_adapter/entry.py index 55773536122..fb9ed95450c 100644 --- a/acp_adapter/entry.py +++ b/acp_adapter/entry.py @@ -32,6 +32,7 @@ else: import argparse import asyncio import logging +import os import sys from pathlib import Path from hermes_constants import get_hermes_home @@ -251,11 +252,13 @@ def main(argv: list[str] | None = None) -> None: # MCP servers dynamically via asyncio.to_thread inside the event # loop; that path is unaffected.) Moved from model_tools.py module # scope to avoid freezing the gateway's loop on lazy import (#16856). - try: - from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() - except Exception: - logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) + # Metadata-only hosts can opt out of unrelated global MCP startup. + if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1": + try: + from tools.mcp_tool import discover_mcp_tools + discover_mcp_tools() + except Exception: + logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) agent = HermesACPAgent() try: diff --git a/acp_adapter/server.py b/acp_adapter/server.py index 3e79bdcd38a..7fee2d932f8 100644 --- a/acp_adapter/server.py +++ b/acp_adapter/server.py @@ -85,6 +85,110 @@ from tools.approval import ( logger = logging.getLogger(__name__) + +def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]: + """Return ``(slug, label, [(model_id, description), ...])`` for named endpoints. + + Covers both the v12 ``providers:`` mapping and the legacy + ``custom_providers:`` list. These endpoints never appear in canonical + provider enumeration, so without this the ACP model selector hides every + named endpoint that the TUI ``/model`` picker already renders (#47039 + implemented named-endpoint rows for the TUI surface only). + + Model lists come from the entry's declared models (``default_model`` + + ``models``), refreshed from the endpoint's live ``/models`` listing when a + credential is available and ``discover_models`` is not disabled. Declared + models are kept even when live discovery fails — some OpenAI-compatible + endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at + all yet serve the declared models fine. + + Slugs use the ``custom:`` shape that ``parse_model_input`` and + ``resolve_runtime_provider`` already resolve, so encoded choice ids + (``custom::``) round-trip through ``set_session_model`` + unchanged. + """ + try: + from hermes_cli.config import ( + get_compatible_custom_providers, + is_provider_enabled, + load_config, + ) + from hermes_cli.models import fetch_api_models + except ImportError: + return [] + + try: + cfg = load_config() + entries = get_compatible_custom_providers(cfg) + except Exception: + logger.debug("Could not load named custom providers", exc_info=True) + return [] + + # ``get_compatible_custom_providers`` drops the ``enabled`` flag during + # normalization, so collect explicitly disabled provider keys from the + # raw config and skip their entries below. + disabled_keys: set[str] = set() + raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None + if isinstance(raw_providers, dict): + for raw_key, raw_entry in raw_providers.items(): + if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry): + disabled_keys.add(str(raw_key).strip().lower()) + + catalogs: list[tuple[str, str, list[tuple[str, str]]]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + provider_key = str(entry.get("provider_key", "") or "").strip() + if provider_key.lower() in disabled_keys: + continue + name = str(entry.get("name", "") or "").strip() + base_url = str(entry.get("base_url", "") or "").strip() + if not name or not base_url: + continue + slug_source = provider_key or name + slug = "custom:" + slug_source.strip().lower().replace(" ", "-") + + api_key = str(entry.get("api_key", "") or "").strip() + if not api_key: + key_env = str(entry.get("key_env", "") or "").strip() + api_key = os.environ.get(key_env, "").strip() if key_env else "" + + declared: list[str] = [] + default_model = str(entry.get("model", "") or "").strip() + if default_model: + declared.append(default_model) + models_cfg = entry.get("models") + if isinstance(models_cfg, dict): + for mid in models_cfg: + mid = str(mid or "").strip() + if mid and mid not in declared: + declared.append(mid) + + if not api_key and not declared: + # No credential to discover with and nothing declared: + # not addressable from the selector. + continue + + model_ids = list(declared) + discover = entry.get("discover_models", True) + if isinstance(discover, str): + discover = discover.lower() not in {"false", "no", "0"} + if discover and api_key: + try: + live = fetch_api_models( + api_key, base_url, api_mode=entry.get("api_mode") + ) + except Exception: + live = None + if live: + model_ids = declared + [m for m in live if m not in declared] + + if not model_ids: + continue + catalogs.append((slug, name, [(mid, "") for mid in model_ids])) + + return catalogs + try: from hermes_cli import __version__ as HERMES_VERSION except Exception: @@ -97,6 +201,13 @@ _executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="acp-agent") # does not expose a client-side limit, so this is a fixed cap that clients # paginate against using `cursor` / `next_cursor`. _LIST_SESSIONS_PAGE_SIZE = 50 +# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render +# the whole `availableModels` array in one dropdown, so an unbounded +# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker +# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not +# the total; aggregator providers stay intentionally uncapped inside the shared +# inventory, and the current model is always kept via the fallback insert below. +ACP_MAX_MODELS_PER_PROVIDER = 200 _MAX_ACP_RESOURCE_BYTES = 512 * 1024 _TEXT_RESOURCE_MIME_PREFIXES = ("text/",) _TEXT_RESOURCE_MIME_TYPES = { @@ -585,46 +696,108 @@ class HermesACPAgent(acp.Agent): return f"{raw_provider}:{raw_model}" def _build_model_state(self, state: SessionState) -> SessionModelState | None: - """Return the ACP model selector payload for editors like Zed.""" + """Return authenticated providers and their models for ACP clients. + + The shared Hermes inventory is also used by ``hermes model``, the TUI, + and the dashboard. Keeping ACP on that substrate prevents its selector + from silently collapsing to the current provider's curated list. + """ model = str(state.model or getattr(state.agent, "model", "") or "").strip() provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter" try: - from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label + from hermes_cli.inventory import build_models_payload, load_picker_context + from hermes_cli.models import normalize_provider, provider_label normalized_provider = normalize_provider(provider) - provider_name = provider_label(normalized_provider) + context = load_picker_context().with_overrides( + current_provider=normalized_provider, + current_model=model, + current_base_url=str(getattr(state.agent, "base_url", "") or ""), + ) + payload = build_models_payload( + context, + explicit_only=True, + include_unconfigured=False, + picker_hints=False, + canonical_order=True, + pricing=False, + capabilities=False, + refresh=False, + probe_custom_providers=False, + probe_current_custom_provider=False, + max_models=ACP_MAX_MODELS_PER_PROVIDER, + ) + available_models: list[ModelInfo] = [] seen_ids: set[str] = set() - - for model_id, description in curated_models_for_provider(normalized_provider): - rendered_model = str(model_id or "").strip() - if not rendered_model: + for row in payload.get("providers") or []: + row_provider = normalize_provider(str(row.get("slug") or "").strip()) + if not row_provider: continue - choice_id = self._encode_model_choice(normalized_provider, rendered_model) - if choice_id in seen_ids: - continue - desc_parts = [f"Provider: {provider_name}"] - if description: - desc_parts.append(str(description).strip()) - if rendered_model == model: - desc_parts.append("current") - available_models.append( - ModelInfo( - model_id=choice_id, - name=rendered_model, - description=" • ".join(part for part in desc_parts if part), - ) + provider_name = str(row.get("name") or "").strip() or provider_label( + row_provider ) - seen_ids.add(choice_id) + for model_entry in row.get("models") or []: + if isinstance(model_entry, dict): + rendered_model = str( + model_entry.get("id") + or model_entry.get("model") + or model_entry.get("name") + or "" + ).strip() + else: + rendered_model = str(model_entry or "").strip() + if not rendered_model: + continue + choice_id = self._encode_model_choice(row_provider, rendered_model) + if choice_id in seen_ids: + continue + is_current = ( + row_provider == normalized_provider and rendered_model == model + ) + description = f"Provider: {provider_name}" + if is_current: + description += " • current" + available_models.append( + ModelInfo( + model_id=choice_id, + name=f"{provider_name} · {rendered_model}", + description=description, + ) + ) + seen_ids.add(choice_id) + + # Named user-defined endpoints (providers: / custom_providers:) + # are invisible to canonical provider enumeration — append them + # so editor clients can select them like the TUI /model picker. + for named_slug, named_label, named_catalog in _named_custom_provider_catalogs(): + for named_model, named_desc in named_catalog: + named_choice = self._encode_model_choice(named_slug, named_model) + if not named_choice or named_choice in seen_ids: + continue + named_parts = [f"Provider: {named_label}"] + if named_desc: + named_parts.append(str(named_desc).strip()) + if named_slug == normalized_provider and named_model == model: + named_parts.append("current") + available_models.append( + ModelInfo( + model_id=named_choice, + name=named_model, + description=" • ".join(part for part in named_parts if part), + ) + ) + seen_ids.add(named_choice) current_model_id = self._encode_model_choice(normalized_provider, model) if current_model_id and current_model_id not in seen_ids: + provider_name = provider_label(normalized_provider) available_models.insert( 0, ModelInfo( model_id=current_model_id, - name=model, + name=f"{provider_name} · {model}", description=f"Provider: {provider_name} • current", ), ) @@ -1588,7 +1761,16 @@ class HermesACPAgent(acp.Agent): clear_session_vars, set_session_vars, ) - session_tokens = set_session_vars(session_key=session_id) + # ``cwd`` pins the logical working directory for this context, + # which is what the system prompt's "Current working directory" + # line reports (agent/prompt_builder.py -> resolve_agent_cwd). + # Without it the prompt advertises the global Hermes workspace + # while the tools are rooted at the client's project, so the + # model emits absolute paths under ~/.hermes/workspace and the + # edit silently lands outside the editor's workspace. + session_tokens = set_session_vars( + session_key=session_id, cwd=state.cwd, + ) except Exception: session_tokens = None clear_session_vars = None # type: ignore[assignment] @@ -1875,8 +2057,26 @@ class HermesACPAgent(acp.Agent): if handler is None: return None # not a known command — let the LLM handle it - try: + # Slash handlers run on the event-loop thread, OUTSIDE the per-turn + # contextvars.copy_context() that pins the session cwd for the agent + # call. ``/compress`` and ``/model`` reach code that REBUILDS the + # system prompt (agent._build_system_prompt -> resolve_agent_cwd), so + # an unpinned handler bakes the Hermes install tree into the session's + # cached prompt — persisted, and therefore poisoning every later turn + # even though the turn itself is pinned. Pin inside a fresh context so + # the write can't leak into other concurrent ACP sessions and needs no + # teardown. + def _dispatch() -> str | None: + try: + from agent.runtime_cwd import set_session_cwd + + set_session_cwd(state.cwd) + except Exception: + logger.debug("Could not pin ACP session cwd for slash command", exc_info=True) return handler(args, state) + + try: + return contextvars.copy_context().run(_dispatch) except Exception as e: logger.error("Slash command /%s error: %s", cmd, e, exc_info=True) return f"Error executing /{cmd}: {e}" diff --git a/agent/agent_init.py b/agent/agent_init.py index 241d3689ebd..955ca11cff8 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -69,6 +69,43 @@ def _ra(): return run_agent +def _moa_reference_output_allowed(agent: Any) -> bool: + """Keep MoA display events off only the machine-readable ``-Q`` surface.""" + return not ( + getattr(agent, "platform", None) == "cli" + and getattr(agent, "tool_progress_mode", "all") == "off" + ) + + +def _relay_moa_reference_event(agent: Any, event: str, **kwargs: Any) -> None: + """Relay MoA display events while preserving the ``-Q`` stdout contract.""" + if not _moa_reference_output_allowed(agent): + return + cb = getattr(agent, "tool_progress_callback", None) + if cb is None: + return + try: + if event == "moa.reference": + cb( + "moa.reference", + str(kwargs.get("label") or ""), + str(kwargs.get("text") or ""), + None, + moa_index=kwargs.get("index"), + moa_count=kwargs.get("count"), + ) + elif event == "moa.aggregating": + cb( + "moa.aggregating", + str(kwargs.get("aggregator") or ""), + None, + None, + moa_ref_count=kwargs.get("ref_count"), + ) + except Exception: + pass + + def _normalize_route_base_url(base_url: Any) -> str: """Canonicalize an endpoint URL for model-route identity comparisons.""" return normalize_route_base_url(base_url) @@ -786,9 +823,10 @@ def init_agent( # Anthropic prompt caching: auto-enabled for Claude models on native # Anthropic, OpenRouter, and third-party gateways that speak the # Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces - # input costs by ~75% on multi-turn conversations. Uses system_and_3 - # strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy`` - # for the layout-vs-transport decision. + # input costs by ~75% on multi-turn conversations. Uses four breakpoints: + # the static system prefix, full system prompt, and last two messages + # (falling back to system-and-3 when no static prefix is available). See + # ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision. agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy() ) @@ -1025,49 +1063,20 @@ def init_agent( elif isinstance(effective_key, str) and len(effective_key) > 12: print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}") elif agent.provider == "moa": - from agent.moa_loop import MoAClient + from agent.moa_loop import build_moa_facade agent.api_mode = "chat_completions" - # Route reference-model outputs to the agent's tool_progress_callback so + # build_moa_facade wires the reference relay that routes + # reference-model outputs to the agent's tool_progress_callback so # every surface that already consumes it (CLI spinner/scrollback, TUI, - # desktop, gateway) can show each reference's answer as a labelled block - # before the aggregator acts. The facade emits "moa.reference" and - # "moa.aggregating" events; we forward them through the same callback - # the tool lifecycle uses. Best-effort and cache-safe — these are - # display-only events, they never touch the message history. - def _moa_reference_relay(event: str, **kwargs: Any) -> None: - cb = getattr(agent, "tool_progress_callback", None) - if cb is None: - return - try: - if event == "moa.reference": - label = str(kwargs.get("label") or "") - text = str(kwargs.get("text") or "") - idx = kwargs.get("index") - count = kwargs.get("count") - cb( - "moa.reference", - label, - text, - None, - moa_index=idx, - moa_count=count, - ) - elif event == "moa.aggregating": - cb( - "moa.aggregating", - str(kwargs.get("aggregator") or ""), - None, - None, - moa_ref_count=kwargs.get("ref_count"), - ) - except Exception: - pass - - agent.client = MoAClient( - agent.model or "default", - reference_callback=_moa_reference_relay, - ) + # desktop, gateway) can show each reference's answer as a labelled + # block before the aggregator acts. The facade emits "moa.reference", + # "moa.progress", "moa.phase", and "moa.aggregating" events, forwarded + # through the same callback the tool lifecycle uses. Best-effort and + # cache-safe — display-only events, they never touch the message + # history. The factory is shared with the fallback-restore/recovery + # paths so a restored facade keeps emitting these events (#53802). + agent.client = build_moa_facade(agent, agent.model) agent._client_kwargs = {} agent.api_key = api_key or "moa-virtual-provider" agent.base_url = "moa://local" @@ -1333,6 +1342,13 @@ def init_agent( print("⚠️ Warning: API key appears invalid or missing") except Exception as e: raise RuntimeError(f"Failed to initialize OpenAI client: {e}") + + # Keep a stable identity for the pool entry that supplied this runtime. + # OAuth refreshes can replace the runtime token before a failed request is + # recovered, so the mutable API-key value alone cannot reliably attribute + # the failure to its source entry. + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) # Provider fallback chain — ordered list of backup providers tried # when the primary is exhausted (rate-limit, overload, connection @@ -1479,6 +1495,9 @@ def init_agent( # Cached system prompt -- built once per session, only rebuilt on compression agent._cached_system_prompt: Optional[str] = None + # Cross-session-stable prefix of the cached prompt. It remains separate + # from the persisted string and is used only to place an early cache marker. + agent._cached_system_prompt_static: Optional[str] = None # Filesystem checkpoint manager (transparent — not a tool) from tools.checkpoint_manager import CheckpointManager @@ -1810,6 +1829,28 @@ def init_agent( compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"} compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20)) compression_protect_last = int(_compression_cfg.get("protect_last_n", 20)) + # Minimum REAL (actionable) user messages guaranteed to survive in the + # uncompressed tail (compression.min_tail_user_messages). Default 1 + # preserves current behavior exactly — the existing single-user tail + # anchor. Values > 1 extend the guarantee to the last N actionable + # user turns. Booleans rejected (bool subclasses int), non-int-like + # values fall back to 1, floor at 1. + _raw_min_tail_users = _compression_cfg.get("min_tail_user_messages", 1) + if isinstance(_raw_min_tail_users, bool): + compression_min_tail_users = 1 + elif isinstance(_raw_min_tail_users, int): + compression_min_tail_users = _raw_min_tail_users + elif isinstance(_raw_min_tail_users, float): + compression_min_tail_users = ( + int(_raw_min_tail_users) if _raw_min_tail_users.is_integer() else 1 + ) + else: + try: + compression_min_tail_users = int(str(_raw_min_tail_users).strip()) + except (TypeError, ValueError): + compression_min_tail_users = 1 + if compression_min_tail_users < 1: + compression_min_tail_users = 1 # Cap on compression retry rounds before a turn gives up with "max # compression attempts reached" (compression.max_attempts). Hardcoding 3 # strands sessions that legitimately need more rounds — e.g. a restart @@ -1838,6 +1879,39 @@ def init_agent( if compression_max_attempts < 1: compression_max_attempts = 3 compression_max_attempts = min(compression_max_attempts, 10) + + def _parse_prune_int(raw, default): + # Same parser semantics as compression.max_attempts above: reject + # booleans (bool subclasses int — YAML `true` would coerce to 1), + # reject fractional floats rather than truncating them, accept + # integral floats and numeric strings, fall back to the default on + # anything else. + if isinstance(raw, bool): + return default + if isinstance(raw, int): + return raw + if isinstance(raw, float): + return int(raw) if raw.is_integer() else default + try: + return int(str(raw).strip()) + except (TypeError, ValueError): + return default + + # Opt-in proactive tool-result prune trigger (0 = disabled — the + # default, so an unset key is behavior-neutral). Negative values are + # treated as disabled rather than erroring. + compression_proactive_prune_tokens = max( + 0, _parse_prune_int(_compression_cfg.get("proactive_prune_tokens", 0), 0) + ) + compression_proactive_prune_min_chars = _parse_prune_int( + _compression_cfg.get("proactive_prune_min_result_chars", 8000), 8000 + ) + compression_proactive_prune_min_reclaim = max( + 0, + _parse_prune_int( + _compression_cfg.get("proactive_prune_min_reclaim_tokens", 4096), 4096 + ), + ) # protect_first_n is the number of non-system messages to protect at # the head, in addition to the system prompt (which is always # implicitly protected by the compressor). Floor at 0 — a value of @@ -2312,6 +2386,10 @@ def init_agent( max_tokens=agent.max_tokens, model_thresholds=compression_model_thresholds, threshold_tokens_cap=compression_threshold_tokens, + proactive_prune_tokens=compression_proactive_prune_tokens, + proactive_prune_min_result_chars=compression_proactive_prune_min_chars, + proactive_prune_min_reclaim_tokens=compression_proactive_prune_min_reclaim, + min_tail_user_messages=compression_min_tail_users, ) _bind_session_state = getattr(agent.context_compressor, "bind_session_state", None) if callable(_bind_session_state): diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 9bd59daa0cc..b70afabfc4e 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -850,6 +850,25 @@ def strip_think_blocks(agent, content: str) -> str: +def sync_credential_pool_entry_id(agent) -> None: + """Rebind ``agent._credential_pool_entry_id`` from the current pool + key. + + OAuth refreshes can replace the runtime token before a failed request is + recovered, so the mutable API-key value alone cannot reliably attribute + the failure to its source entry. This resolves the stable pool-entry ID + for the agent's current ``api_key`` and clears it when no pool is bound. + """ + pool = getattr(agent, "_credential_pool", None) + try: + agent._credential_pool_entry_id = ( + pool.entry_id_for_api_key(getattr(agent, "api_key", None)) + if pool is not None + else None + ) + except Exception: + agent._credential_pool_entry_id = None + + def recover_with_credential_pool( agent, *, @@ -934,10 +953,30 @@ def recover_with_credential_pool( # failing entry exactly; fall back to current()'s key only when the agent # carries no key at all. _api_key_hint = getattr(agent, "api_key", None) or None + _raw_credential_id = getattr(agent, "_credential_pool_entry_id", None) + _credential_id = ( + _raw_credential_id + if isinstance(_raw_credential_id, str) and _raw_credential_id + else None + ) if not _api_key_hint: _cur = pool.current() if _cur: _api_key_hint = getattr(_cur, "runtime_api_key", None) + if not _credential_id: + _current_id = getattr(_cur, "id", None) + if isinstance(_current_id, str) and _current_id: + _credential_id = _current_id + + def _rotate_failed_credential(rotate_status: int): + kwargs = { + "status_code": rotate_status, + "error_context": error_context, + "api_key_hint": _api_key_hint, + } + if _credential_id: + kwargs["credential_id"] = _credential_id + return pool.mark_exhausted_and_rotate(**kwargs) effective_reason = classified_reason if effective_reason is None: @@ -972,11 +1011,7 @@ def recover_with_credential_pool( # Runtime credentials can be resolved by a separate pool instance, # leaving this recovery pool without ``current_id``. Match the key # that actually failed instead of quarantining a different account. - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (billing) — rotated to pool entry %s", @@ -995,8 +1030,13 @@ def recover_with_credential_pool( # Prefer the entry matching the failing key over the shared current() # pointer, for the same attribution reason as above. current_entry = None - if _api_key_hint: + if _credential_id: current_entry = next( + (e for e in pool.entries() if e.id == _credential_id), + None, + ) + if _api_key_hint: + current_entry = current_entry or next( (e for e in pool.entries() if e.runtime_api_key == _api_key_hint), None, ) @@ -1009,11 +1049,7 @@ def recover_with_credential_pool( current_last_status, ) rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s", @@ -1037,11 +1073,7 @@ def recover_with_credential_pool( if not has_retried_429 and not usage_limit_reached: return False, True rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit) — rotated to pool entry %s", @@ -1113,7 +1145,10 @@ def recover_with_credential_pool( # the shared pointer can reference a different, healthy entry, and # refreshing it would consume that entry's single-use refresh token # (or mark it exhausted on failure) for a failure it never had. - refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint) + refresh_kwargs = {"api_key_hint": _api_key_hint} + if _credential_id: + refresh_kwargs["credential_id"] = _credential_id + refreshed = pool.try_refresh_matching(**refresh_kwargs) if refreshed is not None: # ``try_refresh_matching()`` re-mints a fresh OAuth token and reports # success even when the upstream keeps rejecting it — a single-entry @@ -1145,11 +1180,7 @@ def recover_with_credential_pool( # Refresh failed — rotate to next credential instead of giving up. # The failed entry is already marked exhausted by the refresh attempt. rotate_status = status_code if status_code is not None else 401 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (auth refresh failed) — rotated to pool entry %s", @@ -1194,11 +1225,17 @@ def try_recover_primary_transport( return False try: - # Close existing client to release stale connections + # Retire the existing client to release stale connections. #70773: + # never hard-close the shared client here — this runs on the + # conversation-loop thread while workers from stale-killed streaming + # attempts may still be unwinding their SSL BIOs on the old pool. + # ``_retire_shared_openai_client`` shuts the sockets down (FD-safe + # from any thread) and defers the FD release to GC, which cannot + # complete until every borrowing thread has unwound. if getattr(agent, "client", None) is not None: try: - agent._close_openai_client( - agent.client, reason="primary_recovery", shared=True, + agent._retire_shared_openai_client( + agent.client, reason="primary_recovery", ) except Exception: pass @@ -1225,6 +1262,14 @@ def try_recover_primary_transport( ) agent._is_anthropic_oauth = rt["is_anthropic_oauth"] agent.client = None + elif (agent.provider or "").strip().lower() == "moa": + # MoA is a virtual provider with empty client_kwargs — rebuilding + # via _create_openai_client would raise "api_key client option + # must be set". Recreate the facade through the shared factory so + # the reference_callback relay survives recovery (#53802). + from agent.moa_loop import build_moa_facade + + agent.client = build_moa_facade(agent, agent.model) else: agent.client = agent._create_openai_client( dict(rt["client_kwargs"]), @@ -1388,7 +1433,18 @@ def restore_primary_runtime(agent) -> bool: ) # ── Rebuild client for the primary provider ── - if agent.api_mode == "anthropic_messages": + if agent.provider == "moa": + # MoA is a virtual chat-completions provider. It never has real + # OpenAI client kwargs; restoring it after a fallback must recreate + # the facade, not call OpenAI() with an empty api_key. Use the + # shared factory so the restored facade keeps the reference_callback + # relay wired at init — a bare MoAClient() would silently stop + # emitting moa.reference/moa.aggregating display events (#53802). + from agent.moa_loop import build_moa_facade + + agent.client = build_moa_facade(agent, agent.model) + agent._anthropic_client = None + elif agent.api_mode == "anthropic_messages": from agent.anthropic_adapter import build_anthropic_client agent._anthropic_api_key = rt["anthropic_api_key"] agent._anthropic_base_url = rt["anthropic_base_url"] @@ -1441,6 +1497,7 @@ def restore_primary_runtime(agent) -> bool: pool_matches_primary = False if pool is not None and pool_provider and not pool_matches_primary: agent._credential_pool = None + agent._credential_pool_entry_id = None try: from agent.credential_pool import load_pool @@ -1460,6 +1517,7 @@ def restore_primary_runtime(agent) -> bool: # the pool for its current best entry and swap the live credential in. # When the pool is absent, empty, or the entry has no usable key, we # keep the snapshot key (the existing behavior). Fixes #25205. + agent._credential_pool_entry_id = None pool = getattr(agent, "_credential_pool", None) if pool is not None and pool.has_available(): entry = pool.select() @@ -2056,6 +2114,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # restore the original pool (issue #52727: pool reload is part of this # switch and must be reversible on rollback). _snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING) + _snapshot["_credential_pool_entry_id"] = getattr( + agent, "_credential_pool_entry_id", _MISSING + ) try: # Clear the per-config context_length override so the new model's @@ -2112,6 +2173,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # A pool bound to the old provider is worse than no pool: the # recovery guard rejects it and every later 401/429 skips rotation. agent._credential_pool = None + agent._credential_pool_entry_id = None try: from agent.credential_pool import load_pool agent._credential_pool = load_pool(new_provider) @@ -2121,10 +2183,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "continuing without pool rotation this turn", new_provider, _pool_exc, ) - # ── Build new client ── if (new_provider or "").strip().lower() == "moa": - from agent.moa_loop import MoAClient + from agent.moa_loop import build_moa_facade # The MoA virtual provider speaks only chat.completions via the # MoAClient facade — the aggregator's real transport @@ -2141,7 +2202,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo agent.api_key = api_key or "moa-virtual-provider" agent.base_url = "moa://local" agent._client_kwargs = {} - agent.client = MoAClient(agent.model or "default") + agent.client = build_moa_facade(agent, agent.model) elif api_mode == "anthropic_messages": from agent.anthropic_adapter import ( build_anthropic_client, @@ -2217,6 +2278,8 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo reason="switch_model", shared=True, ) + + sync_credential_pool_entry_id(agent) except Exception: # Rollback every mutated field to the pre-swap snapshot so the agent # is left consistent (old model + old provider + old client) and the diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 38431a8c1f5..7fa3ba391fd 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -368,7 +368,7 @@ def _detect_claude_code_version() -> str: try: result = _sp.run( [cmd, "--version"], - capture_output=True, text=True, timeout=5, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5, ) if result.returncode == 0 and result.stdout.strip(): # Output is like "2.1.74 (Claude Code)" or just "2.1.74" @@ -914,7 +914,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: "-s", "Claude Code-credentials", "-w"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=5, stdin=subprocess.DEVNULL, ) @@ -1920,10 +1920,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]: return None btype = b.get("type") if btype == "text": - # Coerce empty/whitespace-only text to a non-whitespace placeholder; - # the Messages input schema rejects blank text blocks (#69512), and a - # blank block stored in history replays on every turn → permanent 400. - out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))} + text_val = b.get("text", "") + # Bedrock and strict Anthropic-compatible endpoints reject text + # blocks where "text" is empty or whitespace-only (#69512). Drop the + # blank block (the caller relocates any cache_control it carried and + # falls back to a non-whitespace placeholder when nothing survives) + # rather than coercing in place — a coerced "(empty)" block would be + # model-visible noise next to surviving thinking/tool_use blocks. + # Type-safe: captured blocks can carry text=None from an invalid + # upstream payload, which a bare .strip() would crash on. + if not isinstance(text_val, str) or not text_val.strip(): + return None + out: Dict[str, Any] = {"type": "text", "text": text_val} # citations is input-valid ONLY when it's a non-empty list; the SDK # emits citations=None on responses, which the input schema rejects. cits = b.get("citations") @@ -2011,9 +2019,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: parsed_args = {} redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args replayed: List[Dict[str, Any]] = [] + _relocated_replay_cache_control = None + _dropped_blank_text = False for b in ordered_blocks: clean = _sanitize_replay_block(b) if clean is None: + if isinstance(b, dict) and b.get("type") == "text": + _dropped_blank_text = True + if isinstance(b, dict) and isinstance(b.get("cache_control"), dict): + # A dropped blank text block can still carry the cache + # breakpoint marker -- relocate it rather than losing it. + _relocated_replay_cache_control = b["cache_control"] continue if clean.get("type") == "tool_use": # Override raw (un-redacted) input with the redacted copy when @@ -2023,20 +2039,68 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: if redacted is not None: clean["input"] = redacted replayed.append(clean) + # When every text block was blank and nothing cacheable survived + # (e.g. signed thinking + a blank text block, or a SOLE blank + # cache-marked block), emit the non-whitespace placeholder so the + # replayed message stays schema-valid (#69512) and a relocated cache + # marker still has a carrier instead of being silently lost. + _has_cacheable_replay = any( + isinstance(b, dict) and b.get("type") in {"text", "tool_use"} + for b in replayed + ) + if not _has_cacheable_replay and ( + _dropped_blank_text or _relocated_replay_cache_control is not None + ): + replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}) if replayed: + if _relocated_replay_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + replayed, _relocated_replay_cache_control + ) _apply_assistant_cache_control_to_last_cacheable_block( replayed, m.get("cache_control") ) return {"role": "assistant", "content": replayed} blocks = _extract_preserved_thinking_blocks(m) + # Cache markers dropped along with a blank block are relocated onto the + # last surviving cacheable block below (via + # _apply_assistant_cache_control_to_last_cacheable_block), rather than + # lost -- prompt_caching.py's _apply_cache_marker() sets cache_control + # directly on content[-1] for list content, so if that last part happens + # to be blank text, dropping it silently would lose the breakpoint. + _relocated_cache_control = None if content: if isinstance(content, list): converted_content = _convert_content_to_anthropic(content) if isinstance(converted_content, list): - blocks.extend(converted_content) + # Bedrock and strict Anthropic-compatible endpoints reject + # text blocks where "text" is empty or whitespace-only. The + # ordered-replay path enforces the same invariant via + # _sanitize_replay_block(). Type-safe against ANY invalid + # "text" value from an upstream payload -- None, or a + # truthy non-string like an int -- not just None: checking + # isinstance() first (rather than `blk.get("text") or ""`) + # means a non-string value is treated as blank/invalid + # instead of reaching .strip() and raising AttributeError. + for blk in converted_content: + _blk_text = blk.get("text") if isinstance(blk, dict) else None + if ( + isinstance(blk, dict) + and blk.get("type") == "text" + and (not isinstance(_blk_text, str) or not _blk_text.strip()) + ): + if isinstance(blk.get("cache_control"), dict): + _relocated_cache_control = blk["cache_control"] + continue + blocks.append(blk) else: - blocks.append({"type": "text", "text": str(content)}) + # Scalar (non-list) content: a whitespace-only string is the + # same invalid-payload case as an empty list block -- drop it + # rather than emitting a blank text block. + text_str = str(content) + if text_str.strip(): + blocks.append({"type": "text", "text": text_str}) for tc in m.get("tool_calls", []): if not tc or not isinstance(tc, dict): continue @@ -2052,9 +2116,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: "name": fn.get("name", ""), "input": parsed_args, }) - _apply_assistant_cache_control_to_last_cacheable_block( - blocks, m.get("cache_control") - ) # Kimi's /coding endpoint (Anthropic protocol) requires assistant # tool-call messages to carry reasoning_content when thinking is # enabled server-side. Preserve it as a thinking block so Kimi @@ -2080,19 +2141,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: ) if isinstance(reasoning_content, str) and not _already_has_thinking: blocks.insert(0, {"type": "thinking", "thinking": reasoning_content}) - # Anthropic rejects empty assistant content - effective = blocks or content - if not effective or effective == "": - effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] - elif isinstance(effective, list): - # The all-empty guard above misses a list that still contains a - # whitespace-only text block (e.g. from a content array of blank parts, - # or compression). Those also trip "text content blocks must contain - # non-whitespace text" (#69512). Coerce text blocks in place; other - # block types (thinking/tool_use/image) are left untouched. - for blk in effective: - if isinstance(blk, dict) and blk.get("type") == "text": - blk["text"] = _safe_text(blk.get("text", "")) + # Anthropic rejects empty assistant content. IMPORTANT: fall back only + # to the placeholder, never to the raw `content` variable -- `content` + # is the UNFILTERED original message content, and can itself be exactly + # the blank/whitespace-only payload the filtering above just removed + # (a sole blank text block, or scalar whitespace with no tool_calls). + # `blocks or content` there would silently restore the invalid provider + # payload this function exists to prevent (#69512). + effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] + # Applied here (after the empty-fallback resolution) rather than + # earlier against `blocks` directly, so a cache_control relocated from + # a dropped blank block that was the ONLY block still lands on the + # (empty) placeholder instead of being silently lost when blocks was + # empty at the point the marker would otherwise have been applied. + if _relocated_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + effective, _relocated_cache_control + ) + _apply_assistant_cache_control_to_last_cacheable_block( + effective, m.get("cache_control") + ) return {"role": "assistant", "content": effective} @@ -2825,6 +2893,7 @@ def create_anthropic_message( *, log_prefix: str = "", prefer_stream: bool = True, + on_stream_event=None, ) -> Any: """Create an Anthropic message, aggregating via stream when available. @@ -2834,6 +2903,13 @@ def create_anthropic_message( crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to match the main turn path, falling back to ``create()`` only for providers that explicitly do not support streaming, such as restricted Bedrock roles. + + ``on_stream_event``: optional callable invoked once per streamed event + (best-effort, exceptions swallowed). Lets callers report forward progress + to liveness watchdogs — e.g. the auxiliary compression path ticking its + progress hook so a slow-but-generating summary model isn't treated as + hung. Only fires on the streaming path; the ``create()`` fallback has no + events to report. """ sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix) @@ -2844,6 +2920,18 @@ def create_anthropic_message( stream_kwargs.pop("stream", None) try: with stream_fn(**stream_kwargs) as stream: + if callable(on_stream_event): + # Consume the event stream manually so each event can + # tick the caller's progress callback; get_final_message + # then returns the accumulated snapshot. + for _event in stream: + try: + on_stream_event(_event) + except Exception: + logger.debug( + "%son_stream_event callback failed", + log_prefix, exc_info=True, + ) return stream.get_final_message() except Exception as exc: if not _is_stream_unavailable_error(exc): diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index bb64d16810e..b31d90e7880 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -54,7 +54,7 @@ import time import uuid from pathlib import Path # noqa: F401 — used by test mocks from types import SimpleNamespace -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse, parse_qs, urlunparse # NOTE: `from openai import OpenAI` is deliberately NOT at module top — the @@ -249,6 +249,50 @@ def aux_interrupt_protection(active: bool = True): _aux_interrupt_protection.active = prev +# ── Forward-progress hook for streamed auxiliary calls ─────────────────── +# Long auxiliary calls (context compression is the prime case) are watched by +# wall-clock deadlines in their hosts (gateway session hygiene). A fixed +# deadline punishes SLOW summary models exactly as hard as HUNG ones: a +# reasoning model happily streaming a large summary is killed mid-generation. +# This thread-local hook lets the host observe liveness instead: the wire +# consumers below tick it on every streamed token/SSE event, and the host +# extends its deadline while tokens are moving (see gateway/run.py session +# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the +# call topology — the aux call and its stream consumption run synchronously +# on the thread that installed the hook. +_aux_progress = threading.local() + + +def _notify_aux_progress() -> None: + """Tick the installed forward-progress hook, if any. Never raises.""" + hook = getattr(_aux_progress, "hook", None) + if hook is None: + return + try: + hook() + except Exception: + logger.debug("aux progress hook failed", exc_info=True) + + +def _aux_progress_active() -> bool: + return getattr(_aux_progress, "hook", None) is not None + + +@contextlib.contextmanager +def aux_progress_hook(hook): + """Install *hook* as the current thread's aux forward-progress callback. + + ``hook=None`` is a no-op passthrough so callers can wire it + unconditionally. Re-entrant-safe: restores the previous hook on exit. + """ + prev = getattr(_aux_progress, "hook", None) + _aux_progress.hook = hook if callable(hook) else prev + try: + yield + finally: + _aux_progress.hook = prev + + def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: """Return False instead of raising when a patched symbol is not a type.""" try: @@ -1060,16 +1104,29 @@ class _CodexCompletionsAdapter: # key in extra_body (not top-level) and GitHub/Copilot Responses opts # out of cache-key routing entirely — for those hosts, skip it here. try: - from agent.transports.codex import _content_cache_key + from agent.transports.codex import ( + _content_cache_key, + _default_prompt_cache_retention_for_request, + ) from utils import base_url_host_matches _host_src = str(getattr(self._client, "base_url", "") or "") _is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai") - _is_github = base_url_host_matches(_host_src, "githubcopilot.com") + _is_github = ( + base_url_host_matches(_host_src, "githubcopilot.com") + or base_url_host_matches(_host_src, "models.github.ai") + ) if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs: _cache_key = _content_cache_key(instructions, resp_kwargs.get("tools")) if _cache_key: resp_kwargs["prompt_cache_key"] = _cache_key + if "prompt_cache_retention" not in resp_kwargs: + _cache_retention = _default_prompt_cache_retention_for_request( + model, + _host_src, + ) + if _cache_retention: + resp_kwargs["prompt_cache_retention"] = _cache_retention except Exception: logger.debug( "Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True @@ -1151,6 +1208,10 @@ class _CodexCompletionsAdapter: def _on_each_event(_event: Any) -> None: # Re-check timeout/cancellation per event, matching the # cadence the old in-line ``_check_cancelled()`` used. + # Each SSE event is also forward progress for hosts watching + # a progress hook (gateway session hygiene): a reasoning + # model streaming a long summary must not look hung. + _notify_aux_progress() _check_cancelled() event_stream = self._client.responses.create(**stream_kwargs) @@ -1392,7 +1453,18 @@ class _AnthropicCompletionsAdapter: existing = {} anthropic_kwargs["extra_body"] = {**existing, **passthrough} - response = create_anthropic_message(self._client, anthropic_kwargs) + response = create_anthropic_message( + self._client, + anthropic_kwargs, + # Tick the aux forward-progress hook per streamed event so hosts + # watching liveness (gateway session hygiene) don't kill a + # slow-but-generating summary model. No-op when no hook is + # installed (None keeps the fast get_final_message path). + on_stream_event=( + (lambda _event: _notify_aux_progress()) + if _aux_progress_active() else None + ), + ) _transport = get_transport("anthropic_messages") _nr = _transport.normalize_response( response, strip_tool_prefix=self._is_oauth @@ -1705,7 +1777,7 @@ def _read_nous_auth() -> Optional[dict]: try: if not _AUTH_JSON_PATH.is_file(): return None - data = json.loads(_AUTH_JSON_PATH.read_text()) + data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8")) if data.get("active_provider") != "nous": return None provider = data.get("providers", {}).get("nous", {}) @@ -2489,16 +2561,18 @@ def _relay_sync_completion( *, provider: str | None = None, api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, ) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) if route is None: - return client.chat.completions.create(**kwargs) + return callback(kwargs) provider_name, fallback_model, metadata = route from agent import relay_llm return relay_llm.execute_current( kwargs, - lambda request: client.chat.completions.create(**request), + callback, name=provider_name, model_name=str(kwargs.get("model") or fallback_model), metadata=metadata, @@ -2512,16 +2586,18 @@ async def _relay_async_completion( *, provider: str | None = None, api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, ) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) if route is None: - return await client.chat.completions.create(**kwargs) + return await callback(kwargs) provider_name, fallback_model, metadata = route from agent import relay_llm return await relay_llm.execute_current_async( kwargs, - lambda request: client.chat.completions.create(**request), + callback, name=provider_name, model_name=str(kwargs.get("model") or fallback_model), metadata=metadata, @@ -3807,6 +3883,7 @@ def _retry_same_provider_sync( effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict], + extra_headers: Optional[Dict[str, str]] = None, ) -> Any: if task == "vision": _, retry_client, retry_model = resolve_vision_provider_client( @@ -3842,7 +3919,13 @@ def _retry_same_provider_sync( extra_body=effective_extra_body, reasoning_config=reasoning_config, base_url=retry_base or resolved_base_url, + task=task, ) + # Preserve per-request attribution headers (e.g. Copilot's + # ``x-initiator: user``) across the rebuilt-client retry — dropping them + # here would let a recovery retry silently lose capability gating (#60293). + if extra_headers: + retry_kwargs["extra_headers"] = dict(extra_headers) if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( @@ -3872,6 +3955,7 @@ async def _retry_same_provider_async( effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict], + extra_headers: Optional[Dict[str, str]] = None, ) -> Any: if task == "vision": _, retry_client, retry_model = resolve_vision_provider_client( @@ -3907,7 +3991,12 @@ async def _retry_same_provider_async( extra_body=effective_extra_body, reasoning_config=reasoning_config, base_url=retry_base or resolved_base_url, + task=task, ) + # Preserve per-request attribution headers across the rebuilt-client + # retry — see the sync variant above (#60293). + if extra_headers: + retry_kwargs["extra_headers"] = dict(extra_headers) if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( @@ -3988,6 +4077,24 @@ def _refresh_provider_credentials(provider: str) -> bool: return False _evict_cached_clients(normalized) return True + if normalized == "vertex": + # Mirrors run_agent.py's _try_refresh_vertex_client_credentials + # for the main conversation loop. Without this branch, an + # auxiliary Vertex client (vision, title generation, reflection, + # context compression, ...) that 401s on its ~1h token expiry + # falls through to the final `return False` below: the stale + # client is never evicted from _client_cache (whose cache key + # ignores the rotating bearer token), so every subsequent + # auxiliary Vertex call keeps 401ing until process restart. + from agent.vertex_adapter import get_vertex_config + + token, base_url = get_vertex_config() + if not isinstance(token, str) or not token.strip(): + return False + if not isinstance(base_url, str) or not base_url.strip(): + return False + _evict_cached_clients(normalized) + return True except Exception as exc: logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc) return False @@ -4101,7 +4208,7 @@ def _call_fallback_candidate_sync( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base) + base_url=fb_base, task=task) try: return _validate_llm_response( _relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task) @@ -4118,7 +4225,7 @@ def _call_fallback_candidate_sync( tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( _relay_sync_completion( @@ -4173,7 +4280,7 @@ async def _call_fallback_candidate_async( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base) + base_url=fb_base, task=task) try: return _validate_llm_response( await _relay_async_completion( @@ -4197,7 +4304,7 @@ async def _call_fallback_candidate_async( tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( await _relay_async_completion( @@ -6965,6 +7072,7 @@ def _build_call_kwargs( extra_body: Optional[dict] = None, reasoning_config: Optional[dict] = None, base_url: Optional[str] = None, + task: Optional[str] = None, ) -> dict: """Build kwargs for .chat.completions.create() with model/provider adjustments.""" kwargs: Dict[str, Any] = { @@ -7019,11 +7127,32 @@ def _build_call_kwargs( _provider_norm in {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"} or base_url_host_matches(_effective_base, "integrate.api.nvidia.com") ) + _is_moa = bool(task) and str(task) == "moa_reference" + # Gemini's native generateContent maps max_tokens → maxOutputTokens and, + # when it is omitted, applies a fixed 65,535-token ceiling rather than + # "the model's full budget" (see gemini_native_adapter.build_gemini_request). + # So an explicit cap is both safe and the ONLY way to honor it here — + # dropping max_tokens silently makes MoA's reference_max_tokens a no-op + # for gemini advisors (they run effectively uncapped). + _is_gemini_native = _provider_norm in { + "gemini", "google", "google-gemini", "google-ai-studio", + } + if not _is_gemini_native and _effective_base: + try: + from agent.gemini_native_adapter import is_native_gemini_base_url + _is_gemini_native = is_native_gemini_base_url(_effective_base) + except Exception: + pass if ( _is_anthropic_compat_endpoint(provider, _effective_base) or _is_nvidia_nim + or _is_moa + or _is_gemini_native ): - kwargs["max_tokens"] = max_tokens + # Use auxiliary_max_tokens_param() so models that require + # max_completion_tokens (GPT-5 family, Copilot) get the right + # parameter name instead of a hardcoded max_tokens that 400s. + kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model)) if tools: # Defensive dedup: providers like Google Vertex, Azure, and Bedrock @@ -7266,6 +7395,346 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any: return value +# ── Streamed aggregation for progress-hooked auxiliary calls ───────────── +# When a forward-progress hook is installed (aux_progress_hook — today only +# by context compression), the primary chat.completions attempt is upgraded +# to a streamed request that is aggregated back into a complete response. +# Two effects, both deliberate: +# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead +# of a total budget (httpx applies the read timeout per stream read), so +# a slow-but-generating summary model is never killed mid-generation +# while tokens are moving — only a genuinely silent connection dies. +# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs +# (gateway session hygiene) extend their deadlines on liveness instead +# of guessing with a fixed wall clock. +# A total ceiling still bounds the pathological 1-token-per-idle-window +# stream; see _aux_stream_total_ceiling(). + +_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0 +_AUX_STREAM_CEILING_MULTIPLIER = 4.0 + + +def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float: + """Absolute wall-clock bound for a progress-hooked streamed aux call. + + Generous by design — the idle timeout is the real guard; this only stops + a degenerate stream that trickles one token per idle window forever. + """ + try: + timeout = float(effective_timeout) if effective_timeout is not None else 0.0 + except (TypeError, ValueError): + timeout = 0.0 + return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, + _AUX_STREAM_CEILING_MULTIPLIER * timeout) + + +def _client_streams_internally(client: Any) -> bool: + """Wire adapters that consume a stream inside .create() already tick the + progress hook themselves (Codex per SSE event, Anthropic per stream + event); Bedrock's Converse shim cannot stream at all. None of them + accept chat-completions ``stream=True`` semantics from us.""" + return isinstance(client, ( + CodexAuxiliaryClient, + AnthropicAuxiliaryClient, + BedrockAuxiliaryClient, + )) + + +def _is_streaming_rejected_error(exc: Exception) -> bool: + """Provider explicitly refused a streamed chat.completions request.""" + err = str(exc).lower() + if "stream_options" in err: + return True + return "stream" in err and ( + "not supported" in err + or "unsupported" in err + or "not allowed" in err + or "disabled" in err + ) + + +def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool: + """Detect providers that only accept streaming (non-stream = HTTP 400). + + Some OpenAI-compatible endpoints reject non-streaming chat requests + outright — e.g. Tencent Copilot returns + ``{"code": 11101, "msg": "Non-stream chat request is currently not + supported"}``. The main conversation loop already streams, so interactive + chat works; auxiliary tasks (title generation, compression, web extract) + used the non-streaming path and failed on every call. When this returns + True the auxiliary client sends ``stream=True`` and aggregates the chunks + itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686). + + Beyond the known-host list, users can mark ANY custom endpoint as + stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml + (list of substrings matched against the endpoint URL). + """ + _url = str(base_url or "").lower() + if not _url: + return False + # Tencent Copilot — "Non-stream chat request is currently not supported" + if base_url_host_matches(_url, "copilot.tencent.com"): + return True + try: + from hermes_cli.config import load_config + aux_cfg = (load_config() or {}).get("auxiliary", {}) + markers = aux_cfg.get("stream_only_base_urls") or [] + if isinstance(markers, (list, tuple)): + for marker in markers: + if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url: + return True + except Exception: + # Config read is best-effort; never break an aux call over it. + pass + return False + + +def _create_with_progress( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, + *, + force_stream: bool = False, +) -> Any: + """chat.completions.create() that streams when a progress hook is active + or the provider only accepts streamed requests. + + Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when + neither trigger applies (every existing caller/task) or when the client's + wire adapter streams internally. With a hook + a chunk-capable client, + the request is sent with ``stream=True`` and aggregated, ticking the hook + per chunk — so the configured ``timeout`` acts per stream read (idle) + rather than as a total budget, and outer liveness watchdogs see tokens + moving. ``force_stream=True`` (stream-only providers such as Tencent + Copilot — credit @kudi88, PR #60686) takes the same streamed path even + without a hook. Providers that reject the streamed request fall back to + the plain non-streaming call — except under ``force_stream``, where a + stream-only provider rejects the plain call by definition, so the + original error is surfaced to the normal recovery chains instead. + """ + _notify_aux_progress() # request dispatched counts as progress + if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client): + return client.chat.completions.create(**kwargs) + + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + try: + chunks = client.chat.completions.create(**stream_kwargs) + except Exception as exc: + # Genuine provider failures (auth, credit, rate limit, network) are + # not streaming's fault — surface them unchanged so the existing + # recovery chains (credential refresh, pool rotation, provider + # fallback) see the same error they would on a plain call. + if ( + force_stream + or _is_transient_transport_error(exc) + or _is_auth_error(exc) + or _is_payment_error(exc) + or _is_rate_limit_error(exc) + ): + raise + # Anything else may be a streaming-specific rejection (explicit + # "stream not supported", stream_options 400, or an idiosyncratic + # 4xx). Retry non-streaming once; if the request itself is bad the + # plain call reproduces the real error for the normal except-chains. + logger.debug( + "Auxiliary %s: streamed request failed (%s); retrying " + "non-streaming", task or "call", exc, + ) + return client.chat.completions.create(**kwargs) + + # Some shims (MoA virtual provider under quiet mode, defensive adapters) + # return a complete response even when stream=True was requested. + if hasattr(chunks, "choices"): + _notify_aux_progress() + return chunks + return _aggregate_chat_stream( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + +def _aggregate_chat_stream( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Consume a chat.completions chunk stream into a complete response. + + Ticks the thread-local aux progress hook on every chunk. Raises + TimeoutError when *total_ceiling* seconds elapse before the stream + finishes — phrased with "timed out" so existing timeout classification + (``_is_timeout_error``) treats it exactly like a request timeout. + Accumulation is shared with the async mirror via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + pass + return acc.finish() + + +class _ChatStreamAccumulator: + """Shared per-chunk accumulation for sync and async stream aggregation. + + Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async + consumer below cannot drift from the sync one (same content/reasoning/ + tool-call delta reassembly, same "timed out" ceiling phrasing). + """ + + def __init__(self, model: str = "", total_ceiling: Optional[float] = None): + self._started = time.monotonic() + self._total_ceiling = total_ceiling + self.content_parts: List[str] = [] + self.reasoning_parts: List[str] = [] + self.tool_calls_acc: Dict[int, Dict[str, Any]] = {} + self.finish_reason = None + self.usage = None + self.resp_id = "" + self.resp_model = model or "" + + def feed(self, chunk: Any) -> None: + _notify_aux_progress() + if ( + self._total_ceiling is not None + and (time.monotonic() - self._started) >= self._total_ceiling + ): + raise TimeoutError( + f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s " + "total ceiling (stream still open but over budget)" + ) + self.resp_id = getattr(chunk, "id", None) or self.resp_id + self.resp_model = getattr(chunk, "model", None) or self.resp_model + chunk_usage = getattr(chunk, "usage", None) + if chunk_usage: + self.usage = chunk_usage + choices = getattr(chunk, "choices", None) or [] + if not choices: + return + choice = choices[0] + self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason + delta = getattr(choice, "delta", None) + if delta is None: + return + piece = getattr(delta, "content", None) + if piece: + self.content_parts.append(piece) + reasoning_piece = ( + getattr(delta, "reasoning", None) + or getattr(delta, "reasoning_content", None) + ) + if reasoning_piece and isinstance(reasoning_piece, str): + self.reasoning_parts.append(reasoning_piece) + for tc in (getattr(delta, "tool_calls", None) or []): + idx = getattr(tc, "index", 0) or 0 + acc = self.tool_calls_acc.setdefault( + idx, {"id": "", "name": "", "arguments": []} + ) + if getattr(tc, "id", None): + acc["id"] = tc.id + fn = getattr(tc, "function", None) + if fn is not None: + if getattr(fn, "name", None): + acc["name"] = fn.name + if getattr(fn, "arguments", None): + acc["arguments"].append(fn.arguments) + + def finish(self) -> Any: + tool_calls = None + if self.tool_calls_acc: + tool_calls = [ + SimpleNamespace( + id=acc["id"], + type="function", + function=SimpleNamespace( + name=acc["name"], + arguments="".join(acc["arguments"]), + ), + ) + for _idx, acc in sorted(self.tool_calls_acc.items()) + ] + message = SimpleNamespace( + role="assistant", + content="".join(self.content_parts), + tool_calls=tool_calls, + reasoning="".join(self.reasoning_parts) or None, + ) + choice = SimpleNamespace( + index=0, + message=message, + finish_reason=self.finish_reason or "stop", + ) + return SimpleNamespace( + id=self.resp_id, + model=self.resp_model, + object="chat.completion", + choices=[choice], + usage=self.usage, + ) + + +async def _aggregate_chat_stream_async( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer). + + The AsyncOpenAI stream contract is an async iterator — consuming it with + the sync helper raises. Same accumulation and ceiling semantics via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + async for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None) + if callable(close_fn): + try: + result = close_fn() + if inspect.isawaitable(result): + await result + except Exception: + pass + return acc.finish() + + +async def _acreate_with_stream( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, +) -> Any: + """Async chat.completions.create() for stream-only providers. + + Sends ``stream=True`` and aggregates the async chunk stream into a + complete response (credit @kudi88, PR #60686 — async contract fixed to + ``async for`` and tool-call deltas preserved per sweeper review). + """ + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + chunks = await client.chat.completions.create(**stream_kwargs) + # Defensive: shims may hand back a complete response despite stream=True. + if hasattr(chunks, "choices"): + return chunks + return await _aggregate_chat_stream_async( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + @_relay_auxiliary_call def call_llm( task: str = None, @@ -7282,6 +7751,7 @@ def call_llm( timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None, + extra_headers: Optional[Dict[str, str]] = None, api_mode: str = None, stream: bool = False, stream_options: dict = None, @@ -7307,6 +7777,9 @@ def call_llm( extra_body: Additional request body fields. reasoning_config: Optional Hermes reasoning config for direct model calls such as MoA reference/aggregator slots. + extra_headers: Additional per-request HTTP headers. These override + client-level defaults for providers that gate capabilities on + request attribution (for example Copilot's ``x-initiator``). stream: When True, return the raw SDK streaming iterator instead of a validated complete response. The caller is responsible for consuming chunks (and for any fallback). Used by the MoA aggregator so its @@ -7424,7 +7897,9 @@ def call_llm( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=_base_info or resolved_base_url) + base_url=_base_info or resolved_base_url, task=task) + if extra_headers: + kwargs["extra_headers"] = dict(extra_headers) # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax) _client_base = str(getattr(client, "base_url", "") or "") @@ -7476,7 +7951,16 @@ def call_llm( kwargs, provider=resolved_provider, api_mode=resolved_api_mode, - ), task, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), + ), + ), + task, provider=resolved_provider, base_url=_base_info) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7514,7 +7998,17 @@ def call_llm( kwargs, provider=resolved_provider, api_mode=resolved_api_mode, - ), task) + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, + _base_info or resolved_base_url, + ), + ), + ), + task) except Exception as retry_transient: if not _is_transient_transport_error(retry_transient): raise @@ -7718,6 +8212,7 @@ def call_llm( effective_timeout=effective_timeout, effective_extra_body=effective_extra_body, reasoning_config=reasoning_config, + extra_headers=extra_headers, ) # ── Same-provider credential-pool recovery ───────────────────── @@ -7766,6 +8261,7 @@ def call_llm( effective_timeout=effective_timeout, effective_extra_body=effective_extra_body, reasoning_config=reasoning_config, + extra_headers=extra_headers, ) except Exception as retry2_err: # The rotated key also hit a quota/auth wall. Mark it @@ -8091,7 +8587,7 @@ async def async_call_llm( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=_client_base or resolved_base_url) + base_url=_client_base or resolved_base_url, task=task) # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax) if _is_anthropic_compat_endpoint(resolved_provider, _client_base): @@ -8101,6 +8597,22 @@ async def async_call_llm( # Retry ONCE on the same provider for a transient transport blip # before the except-chain escalates to fallback — see call_llm() # for the rationale. (PR #16587) + _force_stream_async = ( + _provider_requires_stream( + resolved_provider, _client_base or resolved_base_url, + ) + and not isinstance(client, ( + AsyncCodexAuxiliaryClient, + AsyncAnthropicAuxiliaryClient, + AsyncBedrockAuxiliaryClient, + )) + ) + + async def _acreate(_kwargs: Dict[str, Any]) -> Any: + if _force_stream_async: + return await _acreate_with_stream(client, _kwargs, task) + return await client.chat.completions.create(**_kwargs) + try: return _validate_llm_response( await _relay_async_completion( @@ -8108,7 +8620,9 @@ async def async_call_llm( kwargs, provider=resolved_provider, api_mode=resolved_api_mode, - ), task, + create=_acreate, + ), + task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -8134,7 +8648,9 @@ async def async_call_llm( kwargs, provider=resolved_provider, api_mode=resolved_api_mode, - ), task) + create=_acreate, + ), + task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) diff --git a/agent/background_review.py b/agent/background_review.py index c2ea87bd94e..a0dbd4a99e2 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -209,7 +209,10 @@ _SKILL_REVIEW_PROMPT = ( "conversation for skills the user loaded via /skill-name or you " "read via skill_view. If any of them covers the territory of the " "new learning, PATCH that one first. It is the skill that was in " - "play, so it's the right one to extend.\n" + "play, so it's the right one to extend — but only if it is " + "curator-managed. Bundled, hub, pinned, and user-owned skills are " + "off-limits to you no matter how relevant (see Protected skills " + "below); for those, fall through to the next option.\n" " 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). " "If no loaded skill fits but an existing class-level skill does, " "patch it. Add a subsection, a pitfall, or broaden a trigger.\n" @@ -251,10 +254,18 @@ _SKILL_REVIEW_PROMPT = ( "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). You are an " + "autonomous no-user-present actor, so pin blocks your writes too — " + "content updates included. Only the user, in a foreground session, " + "can change a pinned skill.\n" + " • USER-OWNED skills — anything not curator-managed. A skill the " + "user hand-wrote, installed by URL, or asked a foreground agent to " + "create is theirs, not yours; your writes to it WILL be refused. " + "This includes skills that were loaded or consulted this session: " + "being in play does not make one yours to edit. If such a skill is " + "wrong or outdated, say so in your reply and recommend " + "'hermes curator adopt ' — do not try to patch it.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture (these become persistent self-imposed constraints " @@ -309,7 +320,9 @@ _COMBINED_REVIEW_PROMPT = ( " 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were " "loaded via /skill-name or skill_view in the conversation. If one " "of them covers the learning, PATCH it first. It was in play; " - "it's the right place.\n" + "it's the right place — provided it is curator-managed. Protected " + "and user-owned skills are off-limits however relevant; fall " + "through when one of those is the best fit.\n" " 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to " "find the right one). Patch it.\n" " 3. ADD A SUPPORT FILE under an existing umbrella via " @@ -337,10 +350,15 @@ _COMBINED_REVIEW_PROMPT = ( "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). Pin blocks " + "autonomous writes entirely — content updates included — because no " + "user is present to consent. Only a foreground session can change one.\n" + " • USER-OWNED skills — anything not curator-managed (hand-written, " + "URL-installed, or created by a foreground agent at the user's " + "request). Your writes to these WILL be refused, including to skills " + "loaded or consulted this session. If one is wrong, say so in your " + "reply and recommend 'hermes curator adopt ' instead.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture as skills (these become persistent self-imposed " diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 409df6604a6..15161ea11d4 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1083,6 +1083,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: tools=tools_for_api, reasoning_config=agent.reasoning_config, session_id=getattr(agent, "session_id", None), + base_url=agent.base_url, max_tokens=agent.max_tokens, timeout=agent._resolved_api_call_timeout(), request_overrides=agent.request_overrides, @@ -1520,6 +1521,45 @@ def _fallback_entry_key(fb: dict) -> tuple[str, str, str]: ) +def _fallback_entry_is_same_backend_by_base_url( + *, + current_provider: str, + fb_provider: str, + current_base_url: str, + fb_base_url: str, + current_model: str, + fb_model: str, +) -> bool: + """True when base_url+model identity means the fallback is the same backend. + + Issue #22548: two ``custom_providers`` aliases that point at the same shim + URL with the same model must be skipped, or failover loops on the dead + backend. First-class providers that share a host while using different + auth (``xai-oauth`` vs ``xai``, ``openai-codex`` vs ``openai-api``) are + distinct credential surfaces — skipping them strands configured failover + when primary and fallback reuse the same model slug on that host. + """ + if not ( + fb_base_url + and current_base_url + and fb_base_url == current_base_url + and fb_model == current_model + ): + return False + if fb_provider == current_provider: + return True + try: + from hermes_cli.auth import PROVIDER_REGISTRY + + # Both sides are registered first-class providers → different auth + # identities even when the inference host matches. Allow failover. + if current_provider in PROVIDER_REGISTRY and fb_provider in PROVIDER_REGISTRY: + return False + except Exception: + pass + return True + + def _fallback_entry_unavailable_without_network(agent, fb: dict) -> Optional[str]: """Return a skip reason for fallback entries known to be unusable locally.""" fb_provider = (fb.get("provider") or "").strip().lower() @@ -1608,7 +1648,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # Skip entries that resolve to the current (provider, model) — falling # back to the same backend that just failed loops the failure. Compare # base_url too so two distinct custom_providers entries pointing at the - # same shim/proxy URL also dedup. See issue #22548. + # same shim/proxy URL also dedup. See issue #22548. Do NOT treat + # first-class providers that share a host (xai-oauth vs xai) as the same + # backend — they use different credentials. current_provider = (getattr(agent, "provider", "") or "").strip().lower() current_model = (getattr(agent, "model", "") or "").strip() current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower() @@ -1619,11 +1661,13 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_provider, fb_model, ) return agent._try_activate_fallback(reason) - if ( - fb_base_url_for_dedup - and current_base_url - and fb_base_url_for_dedup == current_base_url - and fb_model == current_model + if _fallback_entry_is_same_backend_by_base_url( + current_provider=current_provider, + fb_provider=fb_provider, + current_base_url=current_base_url, + fb_base_url=fb_base_url_for_dedup, + current_model=current_model, + fb_model=fb_model, ): logger.warning( "Fallback skip: chain entry base_url %s matches current backend", @@ -1746,6 +1790,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_provider, fb_model, _pool_provider, ) agent._credential_pool = None + agent._credential_pool_entry_id = None if getattr(agent, "_credential_pool", None) is None: try: from agent.credential_pool import load_pool @@ -1808,6 +1853,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # not only after a later credential-rotation rebuild. agent._replace_primary_openai_client(reason="fallback_timeout_apply") + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) + # Re-evaluate prompt caching for the new provider/model agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( @@ -1975,7 +2023,17 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: for internal_key in [k for k in api_msg if isinstance(k, str) and k.startswith("_")]: api_msg.pop(internal_key, None) if _needs_sanitize: - agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model) + # In MoA mode, agent.model is the virtual preset name, + # not the actual aggregator model. Resolve the real + # aggregator model so Gemini preserves thought_signature. + _sanitize_model = agent.model + if agent.provider == "moa": + _moa_client = getattr(agent, "client", None) + if _moa_client is not None: + _agg_slot = getattr(_moa_client, "last_aggregator_slot", None) + if _agg_slot and _agg_slot.get("model"): + _sanitize_model = _agg_slot["model"] + agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model) api_messages.append(api_msg) effective_system = agent._cached_system_prompt or "" @@ -2568,7 +2626,11 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= request_client_holder = {"client": None, "diag": None, "owner_tid": None} # Transport kind of the registered request client — see the non-streaming # variant. Routes _close_request_client_once to anthropic vs openai abort/ - # close helpers (#67142). + # close helpers (#67142). ``kind="stream"`` registers a per-request + # *stream handle* instead of a client — used under the MoA facade, whose + # singleton client has no per-request sockets to abort + # (_abort_request_openai_client is a no-op on it), so interrupts must + # close the stream object itself (#57354). request_client_kind = {"value": "openai"} request_client_lock = threading.Lock() # Request-local cancellation flag — see interruptible_api_call for the full @@ -2588,6 +2650,44 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= request_client_holder["owner_tid"] = threading.get_ident() return client + def _stream_close_callable(stream): + close = getattr(stream, "close", None) + if callable(close): + return close + response = getattr(stream, "response", None) + close = getattr(response, "close", None) + if callable(close): + return close + return None + + def _set_request_stream_handle(stream): + # Register the per-request *stream* under kind="stream" so an + # interrupt closes the stream handle itself. Under the MoA facade the + # registered "client" is the shared facade singleton whose + # per-request abort helpers are no-ops, leaving the underlying HTTP + # stream open until the provider drained it (#57354). + if _stream_close_callable(stream) is None: + return stream + with request_client_lock: + request_client_holder["client"] = stream + request_client_kind["value"] = "stream" + request_client_holder["owner_tid"] = threading.get_ident() + return stream + + def _close_request_stream_handle(stream, reason: str) -> None: + close = _stream_close_callable(stream) + if close is None: + return + try: + close() + logger.info("Streaming response handle closed (%s)", reason) + except Exception as exc: + logger.debug( + "Streaming response handle close failed (%s): %s", + reason, + exc, + ) + def _close_request_client_once(reason: str) -> None: # See #29507 explanation in the non-streaming variant above. A # stranger thread (the interrupt-check / stale-stream detector loop) @@ -2595,9 +2695,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # so the worker thread retains ownership of the FD release. with request_client_lock: request_client = request_client_holder.get("client") + request_kind = request_client_kind.get("value", "openai") owner_tid = request_client_holder.get("owner_tid") + # A registered stream handle (kind="stream", MoA facade path) is + # safe to close from any thread — closing IS the abort — so the + # stranger-thread ownership carve-out only applies to real + # per-request clients (#57354). stranger_thread = ( - request_client is not None + request_kind != "stream" + and request_client is not None and owner_tid is not None and owner_tid != threading.get_ident() ) @@ -2606,8 +2712,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= request_client_holder["owner_tid"] = None if request_client is None: return - kind = request_client_kind.get("value", "openai") - if kind == "anthropic_messages": + if request_kind == "stream": + _close_request_stream_handle(request_client, reason) + elif request_kind == "anthropic_messages": if stranger_thread: agent._abort_request_anthropic_client(request_client, reason=reason) else: @@ -2897,6 +3004,10 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= defer_logical_completion=True, ) ) + if agent.provider == "moa": + # Hermes interrupts the managed stream; Relay retains sole + # ownership of closing the underlying provider stream. + _set_request_stream_handle(stream) for chunk in stream: last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") @@ -3567,13 +3678,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # already worker-owned-closed by _close_request_client_once # above; the next attempt builds a fresh one. The shared # _anthropic_client is never closed from inside a request. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_mid_tool_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # SSE error events from proxies (e.g. OpenRouter sends @@ -3632,13 +3739,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # above; next attempt builds fresh), so the shared # _anthropic_client is never closed from inside a # request — only the OpenAI-wire primary is refreshed. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # Retries exhausted. Log the final failure with # full diagnostic detail (chain, headers, @@ -3882,10 +3985,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # FD-recycle corruption vector. Nothing further is needed. pass else: - try: - agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup") - except Exception: - pass + # #70773: same FD-recycle corruption vector as #67142. + # The shared OpenAI client's connection pool must NOT be + # closed from this watchdog/poll thread — worker threads + # from previous stale-killed attempts may still be + # unwinding their SSL BIOs. The request-local client is + # already closed above via _close_request_client_once. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next request. + pass # Reset the timer so we don't kill repeatedly while # the inner thread processes the closure. last_chunk_time["t"] = time.time() diff --git a/agent/codex_responses_adapter.py b/agent/codex_responses_adapter.py index bce372ebb5d..ee75f4190e6 100644 --- a/agent/codex_responses_adapter.py +++ b/agent/codex_responses_adapter.py @@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs( allowed_keys = { "model", "instructions", "input", "tools", "store", "reasoning", "include", "max_output_tokens", "temperature", - "tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier", + "tool_choice", "parallel_tool_calls", "prompt_cache_key", + "prompt_cache_retention", "service_tier", "extra_headers", "extra_body", "timeout", } normalized: Dict[str, Any] = { @@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs( if isinstance(temperature, (int, float)): normalized["temperature"] = float(temperature) - # Pass through tool_choice, parallel_tool_calls, prompt_cache_key - for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"): + # Pass through cache routing/retention and tool-dispatch hints. + for passthrough_key in ( + "tool_choice", + "parallel_tool_calls", + "prompt_cache_key", + "prompt_cache_retention", + ): val = api_kwargs.get(passthrough_key) if val is not None: normalized[passthrough_key] = val diff --git a/agent/coding_context.py b/agent/coding_context.py index 4a0cb841030..fabbdb48e07 100644 --- a/agent/coding_context.py +++ b/agent/coding_context.py @@ -520,30 +520,46 @@ class RuntimeMode: return None return [self.profile.toolset, *_enabled_mcp_servers(config)] - def system_blocks(self) -> list[str]: - """Stable system-prompt blocks for this posture (brief + workspace). + def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]: + """Return prefix, workspace, and trailing posture blocks separately. The operating brief carries a model-family edit-format nudge appended to it (one cached string, not a separate block) so the model is steered toward the `patch` mode it handles best — see ``_edit_format_line``. + + The three lists preserve the historical flat prompt order: the brief, + the live workspace snapshot, then configured operator instructions. + Prompt assembly can therefore put a cache boundary before the snapshot + without changing the persisted system-prompt bytes. """ if not self.is_coding: - return [] - blocks: list[str] = [] + return [], [], [] + prefix: list[str] = [] + workspace_parts: list[str] = [] + trailing: list[str] = [] if self.profile.guidance: brief = self.profile.guidance edit_line = _edit_format_line(self.model) if edit_line: brief = f"{brief}\n{edit_line}" - blocks.append(brief) + prefix.append(brief) workspace = build_coding_workspace_block(self.cwd) if workspace: - blocks.append(workspace) + workspace_parts.append(workspace) # Operator instructions ride their own block so the brief (block 0) stays # byte-stable and cache-keyed independently of user config. if self.instructions: - blocks.append(f"Operator instructions (from config):\n{self.instructions}") - return blocks + trailing.append(f"Operator instructions (from config):\n{self.instructions}") + return prefix, workspace_parts, trailing + + def system_blocks(self) -> list[str]: + """Return posture blocks in their historical display order. + + ``system_prompt_parts`` is the cache-aware API. This compatibility + helper retains the public flat list for callers outside prompt assembly. + """ + prefix, workspace, trailing = self.system_prompt_parts() + return [*prefix, *workspace, *trailing] def compact_skill_categories(self) -> frozenset[str]: """Skill categories to demote to names-only in the prompt's skill index. @@ -644,6 +660,19 @@ def coding_system_blocks( ).system_blocks() +def coding_system_prompt_parts( + *, + platform: Optional[str] = None, + cwd: Optional[str | Path] = None, + config: Optional[dict[str, Any]] = None, + model: Optional[str] = None, +) -> tuple[list[str], list[str], list[str]]: + """Return coding prefix, workspace snapshot, and trailing guidance.""" + return resolve_runtime_mode( + platform=platform, cwd=cwd, config=config, model=model + ).system_prompt_parts() + + def coding_compact_skill_categories( *, platform: Optional[str] = None, diff --git a/agent/context_compressor.py b/agent/context_compressor.py index e282a6c7a45..73fa1e36f21 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -90,9 +90,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: HISTORICAL_TASK_HEADING = "## Historical Task Snapshot" -HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State" -HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks" -HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work" SUMMARY_PREFIX = ( @@ -107,9 +104,7 @@ SUMMARY_PREFIX = ( "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -219,8 +214,45 @@ _MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW] # stale directive it carried (e.g. "resume exactly from Active Task") survives # embedded in the body and keeps hijacking replies. Keep newest-first; entries # are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes. +# NEVER mutate or reorder an existing entry — each one is the exact wire text a +# shipped build persisted, so editing it silently un-normalizes every summary +# written by that build generation; prepend only. tests/agent/ +# test_summary_prefix_semantics.py byte-pins every entry to enforce this. _HISTORICAL_SUMMARY_PREFIXES = ( - # Jul 2026 (#65848 class): identical to the current prefix except it + # Pre-#69619: identical to the current prefix except the stale-item + # discard clause named all four historical headings (the three + # section headers removed by #69619 were still in the template). + # Summaries persisted by builds immediately before #69619 carry this + # exact text and must remain detectable/strippable on resume. + "[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted " + "into the summary below. This is a handoff from a previous context " + "window — treat it as background reference, NOT as active instructions. " + "Do NOT answer questions or fulfill requests mentioned in this summary; " + "they were already addressed. " + "Respond ONLY to the latest user message that appears AFTER this " + "summary — that message is the single source of truth for what to do " + "right now. " + "Topic overlap with the summary does NOT mean you should resume its " + "task: even on similar topics, the latest user message WINS. Treat ONLY " + "the latest message as the active task and discard stale items from " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " + "'finish' work described there unless the latest message explicitly " + "asks for it. " + "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " + "back', 'just verify', 'don't do that anymore', 'never mind', a new " + "topic) must immediately end any in-flight work described in the " + "summary; do not re-surface it in later turns. " + "IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system " + "prompt is ALWAYS authoritative and active — never ignore or deprioritize " + "memory content due to this compaction note. " + "None of the above restricts HOW you work: your tools remain fully " + "active — keep calling them normally for the active task (edit files, " + "run commands, search) instead of merely narrating what you would do. " + "The current session state (files, config, etc.) may reflect work " + "described here — avoid repeating it:", + # Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it # lacked the explicit "tools remain fully active" clause — the strong # REFERENCE ONLY framing bled into general tool-use suppression # (observed: 7 consecutive narration-only turns immediately after a @@ -236,9 +268,9 @@ _HISTORICAL_SUMMARY_PREFIXES = ( "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -303,9 +335,238 @@ _SUMMARY_RATIO = 0.20 # itself a context-pressure source and slows every compaction. _SUMMARY_TOKENS_CEILING = 10_000 +# Aggregate cap on the serialized turn block fed to the summarizer prompt +# (chars). Per-message truncation (_CONTENT_MAX / _TOOL_ARGS_MAX) alone is +# not enough: a compression window with hundreds of already-truncated turns +# can still produce a multi-hundred-KB prompt that blows past slow auxiliary +# backends' context limits or timeouts (Codex Responses fallback paths +# especially). 160K chars ≈ 40K tokens — comfortably inside every supported +# aux model's window while leaving room for the template + previous summary. +# Applied AFTER per-message truncation, with head+tail retention and an +# explicit omitted-middle marker (see _bound_summary_input). This is a +# prompt-side bound only — NEVER add a max_tokens wire cap on the summary +# call (see the no-wire-cap contract test in +# test_compression_small_ctx_threshold_floor.py). +_SUMMARY_INPUT_MAX_CHARS = 160_000 + # Placeholder used when pruning old tool results _PRUNED_TOOL_PLACEHOLDER = "[Old tool output cleared to save context space]" +# Ghost-skill defense (#32106): when compaction reduces an old ``skill_view`` +# result to a 1-line metadata summary, the model still believes the skill is +# loaded even though its instructions are gone. The marker below is the ONE +# canonical prune signal — ``_skill_pruned_marker()`` builds it and every +# presence check matches against the same string, so the emit side and the +# check side can never drift apart (the original PR #44166 emitted +# ``[SKILL_PRUNED:`` but presence-checked ``[SKILL_PRUNED]``, making +# re-injection fire even when the marker had survived). +SKILL_PRUNED_MARKER_PREFIX = "[SKILL_PRUNED:" +# skill_view results at or below this size stay verbatim in pruned +# summaries — small skills are cheap to keep and their loss is unlikely to +# ghost the model. Shared by the emit site and the summarizer-input scan. +_SKILL_VIEW_PRUNE_MIN_CHARS = 5000 +# Cap for the deterministic marker re-injection list — keeps a very long +# session from growing an unbounded "## Pruned Skills" block in every +# iterative summary update. Newest-referenced skills win. +_MAX_PRUNED_SKILL_MARKERS = 20 + + +def _skill_pruned_marker(skill_name: str) -> str: + """Return the canonical prune marker for *skill_name*. + + Used verbatim by BOTH the emit sites (tool-result summarization, + summary re-injection) and the survival check in + ``_reinject_pruned_skill_markers`` — one string, no drift. + """ + return ( + f"{SKILL_PRUNED_MARKER_PREFIX} content lost in compression; " + f"reload with skill_view(name='{skill_name}')]" + ) + + +# Matches the canonical marker and captures the skill name. Anchored on the +# shared prefix constant so a wording change to the marker body updates the +# emit helper and this extractor together. +_SKILL_PRUNED_MARKER_RE = re.compile( + re.escape(SKILL_PRUNED_MARKER_PREFIX) + + r"[^\]]*?reload with skill_view\(name='([^']+)'\)" +) + + +def _extract_pruned_skill_names(text: str) -> list[str]: + """Return skill names referenced by prune markers in *text*, in order.""" + names: list[str] = [] + for match in _SKILL_PRUNED_MARKER_RE.finditer(text or ""): + name = match.group(1) + if name not in names: + names.append(name) + return names + + +def _collect_ghosted_skill_names(turns: List[Dict[str, Any]]) -> list[str]: + """Skill names whose instructions are about to be lost in compaction. + + Covers BOTH shapes a compacted middle window can carry: + + - a ``skill_view`` result already demoted by Phase-1 pruning — the + canonical ``[SKILL_PRUNED: ...]`` marker is in the row content; + - a RAW ``skill_view`` body that was never demoted (it sat inside the + protected tail of an earlier prune, then aged into the compression + window). The summarizer will paraphrase the instructions away, which + is exactly the ghost-skill failure — so it needs a marker too. + """ + names: list[str] = [] + + def _add(name: str) -> None: + if name and name not in names: + names.append(name) + + call_id_to_skill: dict[str, str] = {} + for idx, skill in _skill_view_call_sites(turns): + msg = turns[idx] + for tc in msg.get("tool_calls") or []: + tc_fn = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", None) + tc_name = tc_fn.get("name", "") if isinstance(tc_fn, dict) else getattr(tc_fn, "name", "") + if tc_name != "skill_view": + continue + cid = tc.get("id", "") if isinstance(tc, dict) else (getattr(tc, "id", "") or "") + if cid: + call_id_to_skill[cid] = skill + for msg in turns: + content = msg.get("content") + text = content if isinstance(content, str) else _content_text_for_contains(content) + for name in _extract_pruned_skill_names(text): + _add(name) + if ( + msg.get("role") == "tool" + and isinstance(content, str) + and len(content) > _SKILL_VIEW_PRUNE_MIN_CHARS + ): + skill = call_id_to_skill.get(str(msg.get("tool_call_id") or "")) + if skill: + _add(skill) + return names + + +_PRUNED_SKILLS_SECTION_HEADING = "## Pruned Skills" + + +def _reinject_pruned_skill_markers(summary: str, skill_names: list[str]) -> str: + """Deterministically restore prune markers the summarizer dropped. + + ``skill_names`` was extracted from the summarizer INPUT before the LLM + call. For every skill whose canonical marker (``_skill_pruned_marker``) + is absent from the model's output, append it under a ``## Pruned + Skills`` section. Presence is checked against the SAME canonical string + the emit sites produce — a paraphrased or renamed marker counts as + dropped and is restored (the original PR checked the literal + ``[SKILL_PRUNED]``, which never matches the emitted ``[SKILL_PRUNED:`` + form, so it duplicated markers that HAD survived). + + The appended block is plain body text: it never carries a handoff + prefix, the merged-summary delimiter, or a start-of-content scaffolding + marker, so ``classify_summary_content`` / todo-snapshot flag handling + are unaffected. The block is routed through ``_redact_compaction_text`` + like every other compaction-boundary text. + """ + if not skill_names: + return summary + missing = [ + name for name in skill_names + if _skill_pruned_marker(name) not in summary + ] + if not missing: + return summary + lines = [_skill_pruned_marker(name) for name in missing] + block = ( + "\n\n" + _PRUNED_SKILLS_SECTION_HEADING + "\n" + + "\n".join(lines) + + "\n(The listed skills' instructions were pruned during context " + "compression. Reload with the skill_view call in each marker before " + "relying on that skill; one reload per skill is enough — ignore any " + "older markers for the same skill.)" + ) + return summary + _redact_compaction_text(block) + + +# A skill_view call within this many trailing messages counts as "just +# loaded": its full instruction body must survive the Phase-1 prune even when +# the token-budget boundary would otherwise demote it (#32106). Distinct from +# the protected-tail boundary, which is token-based and can land immediately +# after a bulky just-loaded skill body. +_SKILL_PRUNE_RECENT_WINDOW = 10 + + +def _skill_view_call_sites( + messages: List[Dict[str, Any]], +) -> list[tuple[int, str]]: + """Yield ``(message_index, skill_name)`` for every skill_view tool call.""" + sites: list[tuple[int, str]] = [] + for i, msg in enumerate(messages): + if msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if isinstance(tc, dict): + fn = tc.get("function", {}) + name = fn.get("name", "") if isinstance(fn, dict) else "" + args_str = fn.get("arguments", "") if isinstance(fn, dict) else "" + else: + fn = getattr(tc, "function", None) + name = getattr(fn, "name", "") if fn else "" + args_str = getattr(fn, "arguments", "") if fn else "" + if name != "skill_view" or not isinstance(args_str, str) or not args_str: + continue + try: + args = json.loads(args_str) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(args, dict): + skill = args.get("name", "") + if isinstance(skill, str) and skill: + sites.append((i, skill)) + return sites + + +def _collect_protected_skill_names( + messages: List[Dict[str, Any]], prune_boundary: int, +) -> set[str]: + """Skill names whose skill_view bodies must survive Phase-1 demotion. + + A skill is protected (lower-cased set) when any of these hold: + + - its most recent ``skill_view`` call sits within the last + ``_SKILL_PRUNE_RECENT_WINDOW`` messages (just loaded / just reloaded); + - its most recent ``skill_view`` call sits inside the protected tail + (at or after *prune_boundary*); + - its name is mentioned in a user message inside the protected tail + (the user is actively steering work that depends on it). + + Protection applies to the ordinary Phase-1/2 prune only. The Pass-4 + pressure demotion deliberately ignores it: when the protected region + itself exceeds the soft budget, exempting skill bodies would recreate + the #61932 dead-end shape. + """ + total = len(messages) + if not total: + return set() + recent_start = max(0, total - _SKILL_PRUNE_RECENT_WINDOW) + tail_start = max(0, prune_boundary) + tail_user_texts: list[str] = [] + for msg in messages[tail_start:]: + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, str) and content: + tail_user_texts.append(content.lower()) + protected: set[str] = set() + for idx, skill in _skill_view_call_sites(messages): + key = skill.lower() + if idx >= recent_start or idx >= tail_start: + protected.add(key) + elif any(key in text for text in tail_user_texts): + protected.add(key) + return protected + # Chars per token rough estimate _CHARS_PER_TOKEN = 4 # Flat token cost per attached image part. Real cost varies by provider and @@ -877,7 +1138,19 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten code_preview += "..." return f"[execute_code] `{code_preview}` ({line_count} lines output)" - if tool_name in {"skill_view", "skills_list", "skill_manage"}: + if tool_name == "skill_view": + name = args.get("name", "?") + if content_len > _SKILL_VIEW_PRUNE_MIN_CHARS: + # Ghost-skill defense (#32106): a metadata-only summary makes the + # model believe the skill is still loaded. The canonical marker + # tells it the instructions are gone AND how to get them back. + return ( + f"[skill_view] name={name} ({content_len:,} chars) " + + _skill_pruned_marker(str(name)) + ) + return f"[skill_view] name={name} ({content_len:,} chars)" + + if tool_name in {"skills_list", "skill_manage"}: name = args.get("name", "?") return f"[{tool_name}] name={name} ({content_len:,} chars)" @@ -972,6 +1245,7 @@ class ContextCompressor(ContextEngine): self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -1109,6 +1383,7 @@ class ContextCompressor(ContextEngine): self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -1135,6 +1410,7 @@ class ContextCompressor(ContextEngine): self._consecutive_timeout_failures = 0 self._fallback_compression_streak = 0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self.get_active_compression_failure_cooldown() self._load_fallback_compression_streak() self._load_ineffective_compression_count() @@ -1506,6 +1782,15 @@ class ContextCompressor(ContextEngine): # rationale as the gpt-5.5/Codex 85% autoraise. _MIN_CTX_TRIGGER_RATIO = 0.85 + # Anti-thrash recovery window (#14694): once the ineffective/fallback + # breaker trips, automatic compaction stays blocked for this long, then + # ONE probe attempt is allowed (counters drop to 1 strike, so another + # ineffective pass re-trips immediately). Long enough that a genuinely + # incompressible session isn't compacting in a loop; short enough that a + # session which has since grown real compressible material recovers well + # before it rides into the provider's hard context limit. + _ANTI_THRASH_RECOVERY_SECONDS = 300.0 + @staticmethod def _coerce_max_tokens(value: Any) -> int | None: """Normalize a max_tokens value to a positive int or None. @@ -1629,6 +1914,10 @@ class ContextCompressor(ContextEngine): max_tokens: int | None = None, model_thresholds: dict[str, float] | None = None, threshold_tokens_cap: Any = None, + proactive_prune_tokens: int = 0, + proactive_prune_min_result_chars: int = 8000, + proactive_prune_min_reclaim_tokens: int = 4096, + min_tail_user_messages: int = 1, ): self.model = model self.base_url = base_url @@ -1658,6 +1947,32 @@ class ContextCompressor(ContextEngine): ) self.protect_first_n = protect_first_n self.protect_last_n = protect_last_n + # Proactive tool-result pruning (cost-oriented; runs INDEPENDENTLY of the + # full-compression trigger, via prune_tool_results_only()). 0 = disabled. + self.proactive_prune_tokens = int(proactive_prune_tokens or 0) + # Floor the summarize threshold at 200 chars (matching + # _prune_old_tool_results' dedup floor). Below ~200 a generated summary + # can be longer than the floor it replaces, so Pass 2 would re-summarize + # its own output every turn (corrupting it and never converging); a + # negative value would strip every non-tail tool result outright. A + # configured 0 keeps the 8000 default via `or`. Keep the floor well above + # typical summary length (default 8000) to stay idempotent. + self.proactive_prune_min_result_chars = max( + 200, int(proactive_prune_min_result_chars or 8000) + ) + # Minimum estimated token reclaim before a proactive prune COMMITS. + # Every commit rewrites messages the provider has already seen, which + # invalidates the prompt-cache prefix from the earliest rewritten + # message forward. Without this gate a busy tool loop would re-fire + # the prune nearly every iteration (each new tool pair ages an old one + # out of the protected tail), breaking the cache per turn. Requiring a + # meaningful batch of reclaimable tokens makes fires episodic and + # amortized — the same way full compression is the one sanctioned + # cache break. 0 disables the gate (commit any non-zero prune). + self.proactive_prune_min_reclaim_tokens = max( + 0, int(proactive_prune_min_reclaim_tokens or 0) + ) + self.min_tail_user_messages = min_tail_user_messages self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80)) self.quiet_mode = quiet_mode # Output-token reservation: the provider carves max_tokens out of the @@ -1745,6 +2060,12 @@ class ContextCompressor(ContextEngine): # Anti-thrashing: track whether last compression was effective self._last_compression_savings_pct: float = 100.0 self._ineffective_compression_count: int = 0 + # Monotonic deadline after which a tripped anti-thrash guard grants + # one probation probe (#14694). 0.0 = clock not armed. Armed lazily on + # the first blocked evaluation; deliberately NOT durable, so a process + # restart with a persisted tripped counter (#69872) waits a full fresh + # window before probing (#54923: restart must never disarm a guard). + self._anti_thrash_recovery_deadline: float = 0.0 # Consecutive completed deterministic-fallback boundaries. Unlike the # real-usage effectiveness counter, ordinary fitting responses must not # reset this breaker; only a healthy completed summary does. @@ -1856,6 +2177,16 @@ class ContextCompressor(ContextEngine): self._verify_compaction_cleared_threshold = False self.awaiting_real_usage_after_compression = False + def snapshot_preflight_display_tokens(self) -> int: + """Capture the display token count before a speculative preflight seed.""" + return self.last_prompt_tokens + + def rollback_interrupted_preflight_display_tokens(self, snapshot: int) -> None: + """Restore a speculative display seed without touching compaction state.""" + if self.awaiting_real_usage_after_compression and self.last_prompt_tokens == -1: + return + self.last_prompt_tokens = snapshot + def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool: """Return True when a high rough preflight estimate is known-noisy. @@ -2025,21 +2356,66 @@ class ContextCompressor(ContextEngine): _cooldown_remaining, ) return True - # Anti-thrashing: back off if recent compressions were ineffective + # Anti-thrashing: back off if recent compressions were ineffective. + # The back-off must not be permanent (#14694): the tripped state was + # judged against the transcript as it existed THEN (e.g. a middle + # region too small to matter), but the conversation keeps growing and + # can accumulate plenty of compressible material later. Without a + # recovery path the session never auto-compacts again and rides into + # the provider's hard context limit. Recovery is a probation probe: + # after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE + # attempt by dropping the tripped counter(s) to 1 strike (persisted, + # so sibling agents on the same session row unblock too). If the probe + # is ineffective again the very next verdict re-trips the guard, so + # the worst case in the truly-incompressible state is one compaction + # attempt per recovery window — bounded, not thrash. + # + # The clock is armed lazily on the first BLOCKED evaluation rather + # than persisted at trip time: a fresh process that loads a durable + # tripped counter (#69872) therefore starts a full window blocked, + # preserving the restart-must-not-disarm contract (#54923). if ( self._ineffective_compression_count >= 2 or self._fallback_compression_streak >= 2 ): + _now = time.monotonic() + if self._anti_thrash_recovery_deadline <= 0.0: + self._anti_thrash_recovery_deadline = ( + _now + self._ANTI_THRASH_RECOVERY_SECONDS + ) + elif _now >= self._anti_thrash_recovery_deadline: + self._anti_thrash_recovery_deadline = 0.0 + if self._ineffective_compression_count >= 2: + self._record_ineffective_compression_verdict(1) + if self._fallback_compression_streak >= 2: + self._fallback_compression_streak = 1 + self._persist_fallback_compression_streak() + if not self.quiet_mode: + logger.info( + "Anti-thrashing recovery: %.0fs elapsed since the " + "guard tripped — allowing one compaction probe " + "(ineffective=%d fallback=%d).", + self._ANTI_THRASH_RECOVERY_SECONDS, + self._ineffective_compression_count, + self._fallback_compression_streak, + ) + return False if not self.quiet_mode: logger.warning( "Compression skipped — repeated compaction attempts did not " "restore healthy context. ineffective=%d fallback=%d. " - "Consider /new to start fresh, or /compress for " - "focused compression.", + "Auto-compaction will retry once in %.0fs. Consider /new " + "to start fresh, or /compress for focused " + "compression.", self._ineffective_compression_count, self._fallback_compression_streak, + max(0.0, self._anti_thrash_recovery_deadline - _now), ) return True + # Guard not tripped (counters were cleared by an effective compaction + # or a fitting real-usage reading) — disarm any pending recovery clock + # so a LATER trip starts its own full window. + self._anti_thrash_recovery_deadline = 0.0 return False # ------------------------------------------------------------------ @@ -2049,6 +2425,7 @@ class ContextCompressor(ContextEngine): def _prune_old_tool_results( self, messages: List[Dict[str, Any]], protect_tail_count: int, protect_tail_tokens: int | None = None, + min_prune_chars: int = 200, ) -> tuple[List[Dict[str, Any]], int]: """Replace old tool result contents with informative 1-line summaries. @@ -2158,7 +2535,14 @@ class ContextCompressor(ContextEngine): else: content_hashes[h] = (i, msg.get("tool_call_id", "?")) - def _demote_tool_result_at(idx: int) -> bool: + # Ghost-skill defense (#32106): skills just loaded (or actively + # referenced in the protected tail) keep their full skill_view + # bodies through the ordinary prune passes. Without this, a skill + # loaded moments before a compaction can be demoted to metadata + # while the model still believes its instructions are in context. + protected_skills = _collect_protected_skill_names(result, prune_boundary) + + def _demote_tool_result_at(idx: int, *, spare_protected_skills: bool = True) -> bool: """Replace a bulky tool result at ``idx`` with a 1-line summary. Returns True when the message was modified. @@ -2191,10 +2575,22 @@ class ContextCompressor(ContextEngine): return False if content.startswith("[screenshot removed"): return False - if len(content) <= 200: + # Only prune if the content is substantial (default >200 chars; the + # proactive path raises this floor via min_prune_chars). + if len(content) <= min_prune_chars: return False call_id = msg.get("tool_call_id", "") tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", "")) + if spare_protected_skills and tool_name == "skill_view" and protected_skills: + # Just-loaded / actively-referenced skills survive verbatim + # (#32106). Pass-4 pressure demotion overrides this. + try: + _args = json.loads(tool_args) if tool_args else {} + except (json.JSONDecodeError, TypeError): + _args = {} + _skill = _args.get("name", "") if isinstance(_args, dict) else "" + if isinstance(_skill, str) and _skill.lower() in protected_skills: + return False summary = _summarize_tool_result(tool_name, tool_args, content) result[idx] = {**msg, "content": summary} pruned += 1 @@ -2259,7 +2655,10 @@ class ContextCompressor(ContextEngine): if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling: pressure_hits = 0 for i in range(max(0, prune_boundary), demote_end): - if _demote_tool_result_at(i): + # Pressure passes override the just-loaded-skill guard: + # when the protected region itself blows the soft budget, + # sparing skill bodies would recreate the #61932 dead-end. + if _demote_tool_result_at(i, spare_protected_skills=False): pressure_hits += 1 if _truncate_tool_call_args_at(i): pressure_hits += 1 @@ -2279,7 +2678,7 @@ class ContextCompressor(ContextEngine): if last_tool_idx is not None and i == last_tool_idx: continue if result[i].get("role") == "tool": - if _demote_tool_result_at(i): + if _demote_tool_result_at(i, spare_protected_skills=False): pressure_hits += 1 elif result[i].get("role") == "assistant": if _truncate_tool_call_args_at(i): @@ -2293,7 +2692,9 @@ class ContextCompressor(ContextEngine): and last_tool_idx >= prune_boundary and _protected_region_tokens() > soft_ceiling ): - if _demote_tool_result_at(last_tool_idx): + if _demote_tool_result_at( + last_tool_idx, spare_protected_skills=False + ): pressure_hits += 1 if pressure_hits and not self.quiet_mode: logger.info( @@ -2307,6 +2708,75 @@ class ContextCompressor(ContextEngine): return result, pruned + def prune_tool_results_only( + self, messages: List[Dict[str, Any]], current_tokens: int | None = None, + ) -> tuple[List[Dict[str, Any]], int]: + """Deterministic, no-LLM tool-result prune for the cost-oriented path. + + Runs the Phase-1 prune (``_prune_old_tool_results``) WITHOUT the + compression summary phase, gated on ``proactive_prune_tokens`` rather + than the (much higher) full-compression threshold. On large-window + models ``should_compress()`` (≈50% of the window) rarely fires, so old + tool outputs otherwise ride in history and are re-sent verbatim on every + subsequent turn; this reclaims them early with no quality-risky LLM + summarization. + + Protects the recent tail by message COUNT (``protect_last_n``), never by + ``tail_token_budget`` — the latter is derived from the 50% compression + threshold (≈100K tokens on a 1M window) and would protect the entire + session, pruning nothing. + + ``_prune_old_tool_results`` runs all three deterministic passes: + (1) dedup byte-identical tool results — keeps the newest full copy and + back-references older exact duplicates ANYWHERE in the list (including + the protected tail), so no unique content is ever lost; (2) summarize + non-tail tool results larger than ``min_prune_chars``; (3) truncate + oversized tool_call arguments on non-tail assistant messages. Only + pass (2)'s floor is raised by ``proactive_prune_min_result_chars``; + passes (1) and (3) keep their own fixed floors. The recent-tail + protection applies to passes (2) and (3); pass (1) is tail-agnostic by + design because dedup is lossless. + + PROMPT-CACHE CONTRACT: a committed prune rewrites message bodies the + provider has already seen, invalidating the cached prefix from the + earliest rewritten message forward — exactly like a compression + boundary. To keep that break episodic rather than per-turn, the prune + only COMMITS when the estimated reclaim meets + ``proactive_prune_min_reclaim_tokens`` (measured on the actual pruned + output, not guessed up front). Below the gate the INPUT list object is + returned unchanged — the standard no-op caller contract (callers gate + bookkeeping on ``result is not input``). + + Returns ``(messages, 0)`` — the input object — when disabled, below + the trigger, or when the reclaim gate rejects the commit. + """ + if self.proactive_prune_tokens <= 0: + return messages, 0 + if current_tokens is not None and current_tokens < self.proactive_prune_tokens: + return messages, 0 + # Nothing to reclaim until there are messages outside the protected tail. + if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1: + return messages, 0 + pruned_msgs, pruned_count = self._prune_old_tool_results( + messages, + protect_tail_count=self.protect_last_n, + protect_tail_tokens=None, + min_prune_chars=self.proactive_prune_min_result_chars, + ) + if not pruned_count: + # Standard no-op contract: hand back the INPUT object so callers + # can gate bookkeeping on `result is not input`. + return messages, 0 + # Measured-savings gate (prompt-cache hysteresis): only commit when + # the prune reclaims a meaningful batch of tokens. Estimated on the + # real before/after messages so dedup + arg truncation count too. + if self.proactive_prune_min_reclaim_tokens > 0: + before = sum(_estimate_msg_budget_tokens(m) for m in messages) + after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs) + if (before - after) < self.proactive_prune_min_reclaim_tokens: + return messages, 0 + return pruned_msgs, pruned_count + # ------------------------------------------------------------------ # Summarization # ------------------------------------------------------------------ @@ -2330,6 +2800,10 @@ class ContextCompressor(ContextEngine): _CONTENT_TAIL = 1500 # chars kept from the end _TOOL_ARGS_MAX = 1500 # tool call argument chars _TOOL_ARGS_HEAD = 1200 # kept from the start of tool args + # Aggregate cap over the whole serialized block, applied AFTER the + # per-message limits above. Alias of the module-level constant (which + # carries the full rationale) so subclasses/tests can override per-class. + _SUMMARY_INPUT_MAX_CHARS = _SUMMARY_INPUT_MAX_CHARS def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: """Serialize conversation turns into labeled text for the summarizer. @@ -2589,12 +3063,6 @@ Recovered from a deterministic fallback because the LLM context summarizer was u ## Active State Unknown from deterministic fallback. Inspect current repository/session state if needed. -{HISTORICAL_IN_PROGRESS_HEADING} -Unknown from deterministic fallback — the latest user ask is recorded once under -"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an -unfulfilled instruction to re-answer; verify current state and continue from the -protected recent messages after this summary. - ## Blocked {_bullets(blockers, limit=5)} @@ -2604,27 +3072,62 @@ None recoverable from deterministic fallback. ## Resolved Questions None recoverable from deterministic fallback. -{HISTORICAL_PENDING_ASKS_HEADING} -None recoverable from deterministic fallback. (The latest user ask is preserved once -under "{HISTORICAL_TASK_HEADING}" as historical context — it is NOT necessarily -outstanding.) - ## Relevant Files {_bullets(relevant_files, limit=12)} -{HISTORICAL_REMAINING_WORK_HEADING} -Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims. - ## Last Dropped Turns {_bullets(last_dropped_turns, limit=8)} ## Critical Context Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}""" + # Ghost-skill defense (#32106): the fallback's per-turn truncation + # (``_FALLBACK_TURN_MAX_CHARS``) routinely cuts [SKILL_PRUNED: ...] + # markers out of the compacted turns. Re-derive the ghosted skills + # from the raw turn contents and re-inject deterministically, + # exactly like the LLM-summary path. + _pruned_names = _collect_ghosted_skill_names(turns_to_summarize) + del _pruned_names[_MAX_PRUNED_SKILL_MARKERS:] summary = self._with_summary_prefix(_redact_compaction_text(body.strip())) if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS: summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]" + # Re-inject AFTER the size cap: the markers live at the end of the + # body, exactly where the truncation above cuts. + summary = _reinject_pruned_skill_markers(summary, _pruned_names) return summary + @classmethod + def _bound_summary_input(cls, content: str) -> str: + """Cap total summarizer input while preserving beginning and recent tail. + + Per-message truncation alone is not enough for very long sessions: a + compression window with hundreds of messages can still produce a huge + single prompt that slow auxiliary backends time out on. Keep both edges + because the beginning often has task setup and the tail has the most + recent state; explicitly mark the omitted middle so the summarizer knows + context was intentionally compressed before it saw the prompt. + """ + if len(content) <= cls._SUMMARY_INPUT_MAX_CHARS: + return content + + marker_template = ( + "\n\n...[summary input truncated: omitted " + "{omitted:,} chars from the middle to keep compression prompt bounded]...\n\n" + ) + # Estimate once, then rebuild with the exact omitted span after the + # head/tail split is known. The second marker can differ by a few chars + # if the comma-formatted number changes width, so recompute once. + marker = marker_template.format(omitted=len(content)) + remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0) + head_chars = int(remaining * 0.45) + tail_chars = remaining - head_chars + omitted = max(len(content) - head_chars - tail_chars, 0) + marker = marker_template.format(omitted=omitted) + remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0) + head_chars = int(remaining * 0.45) + tail_chars = remaining - head_chars + tail = content[-tail_chars:].lstrip() if tail_chars else "" + return content[:head_chars].rstrip() + marker + tail + def _fallback_to_main_for_compression(self, e: Exception, reason: str) -> None: """Switch from a separate ``summary_model`` back to the main model. @@ -2698,6 +3201,23 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb summary_budget = self._compute_summary_budget(turns_to_summarize) content_to_summarize = self._serialize_for_summary(turns_to_summarize) + # P2 ghost-skill defense (#32106): [SKILL_PRUNED: ...] markers entering + # the summarizer are prompt INPUT only — LLMs routinely paraphrase them + # into vague prose ("some skills were loaded"), which erases the reload + # instruction. Collect the ghosted skills deterministically BEFORE the + # call (both already-pruned marker rows AND raw skill_view bodies whose + # instructions are about to be summarized away); + # ``_reinject_pruned_skill_markers`` restores any marker the model + # dropped AFTER the call. Markers already carried by the previous + # summary must survive iterative rewrites the same way. Collection + # walks the turn LIST, so the serialized input bound below cannot + # hide a marker in its omitted middle. + _pruned_skill_names = _collect_ghosted_skill_names(turns_to_summarize) + for _name in _extract_pruned_skill_names(self._previous_summary or ""): + if _name not in _pruned_skill_names: + _pruned_skill_names.append(_name) + del _pruned_skill_names[_MAX_PRUNED_SKILL_MARKERS:] + content_to_summarize = self._bound_summary_input(content_to_summarize) _sanitized_memory_context = sanitize_memory_context(memory_context) _serialized_memory_context = json.dumps( _sanitized_memory_context, @@ -2865,9 +3385,6 @@ Be specific with file paths, commands, line numbers, and results.] - Any running processes or servers - Environment details that matter] -{HISTORICAL_IN_PROGRESS_HEADING} -[Work currently underway — what was being done when compaction fired] - ## Blocked [Any blockers, errors, or issues not yet resolved. Include exact error messages.] @@ -2877,30 +3394,39 @@ Be specific with file paths, commands, line numbers, and results.] ## Resolved Questions {_resolved_questions_instructions} -{HISTORICAL_PENDING_ASKS_HEADING} -{_pending_asks_instructions} - ## Relevant Files [Files read, modified, or created — with brief note on each] -{HISTORICAL_REMAINING_WORK_HEADING} -[What remains to be done — framed as STALE context for reference only. The agent must NOT resume this work unless the latest user message explicitly asks for it.] - ## Critical Context [Any specific values, error messages, configuration details, or data that would be lost without explicit preservation. NEVER include API keys, tokens, passwords, or credentials — write [REDACTED] instead.] +{_PRUNED_SKILLS_SECTION_HEADING} +[If any [SKILL_PRUNED: ...reload with skill_view(...)] markers appear in the input, +repeat each one verbatim here — copy the exact text, do NOT paraphrase, summarize, +or describe them. These markers tell the agent which skills must be reloaded before +use. If none appear, omit this section entirely.] + Target ~{summary_budget} tokens. Be CONCRETE — include file paths, command outputs, error messages, line numbers, and specific values. Avoid vague descriptions like "made some changes" — say exactly what changed. {_temporal_anchoring_rule} Write only the summary body. Do not include any preamble or prefix.""" if self._previous_summary: - # Iterative update: preserve existing info, add new progress + # Iterative update: preserve existing info, add new progress. + # Bound the previous-summary block with the same aggregate cap as + # the serialized new turns: a normal summary is far below the cap + # (the output side is held to a ~10K-token ceiling), but a + # pathological handoff rehydrated from a persisted session can be + # arbitrarily large — the iterative prompt (previous summary + + # new turns) must stay bounded too. + _bounded_previous_summary = self._bound_summary_input( + self._previous_summary + ) prompt = f"""{_summarizer_preamble} You are updating a context compaction summary. A previous compaction produced the summary below. New conversation turns have occurred since then and need to be incorporated. PREVIOUS SUMMARY: -{self._previous_summary} +{_bounded_previous_summary} NEW TURNS TO INCORPORATE: {content_to_summarize}{_memory_section} @@ -3031,6 +3557,9 @@ This compaction should PRIORITISE preserving all information related to the focu # Redact the summary output as well — the summarizer LLM may # ignore prompt instructions and echo back secrets verbatim. summary = _redact_compaction_text(content.strip()) + # P2 ghost-skill defense (#32106): deterministically restore any + # [SKILL_PRUNED: ...] marker the summarizer paraphrased away. + summary = _reinject_pruned_skill_markers(summary, _pruned_skill_names) summary = self._ground_historical_task_snapshot(summary, turns_to_summarize) self._validate_summary_user_provenance(summary, has_user_turn) # Store for iterative updates on next compaction @@ -4091,6 +4620,69 @@ This compaction should PRIORITISE preserving all information related to the focu return max(pair_end, head_end + 1) return adjusted + def _ensure_last_n_user_messages_in_tail( + self, + messages: List[Dict[str, Any]], + cut_idx: int, + head_end: int, + n: int, + ) -> int: + """Guarantee the last N actionable user messages are in the protected tail. + + Generalizes ``_ensure_last_user_message_in_tail`` to preserve an + arbitrary number of recent user messages. This prevents the token- + budget-based tail cut from consuming recent conversation turns + when large tool outputs fill the budget. + + When *n* <= 1, delegates directly to the existing single-message + method for byte-identical regression safety. + + If the conversation has fewer than *n* user messages, the earliest + available user message is used without error. + + Only REAL actionable user turns count toward N — the collector uses + the same ``_is_actionable_user_turn`` / + ``_is_synthetic_compression_user_turn`` pair as + ``_find_last_user_message_idx``, so blank platform echoes, compaction + handoffs, continuation markers, and todo-snapshot rows never consume + a slot (#69291 bug class). + + A user message is already a clean boundary — there is no + tool_call/result group that spans across it, so + ``_align_boundary_backward`` is intentionally NOT called. + Calling it can pull the cut past the user message into the + preceding assistant(tool_calls)→tool group and split it (#22566). + """ + if n <= 1: + return self._ensure_last_user_message_in_tail(messages, cut_idx, head_end) + + # Collect real user message indices walking backward from end. + # Mirror _find_last_user_message_idx's filters: compaction handoffs, + # blank platform echoes, and synthetic continuation/todo rows are + # continuity artifacts, not real user turns. + user_indices = [] + for i in range(len(messages) - 1, head_end - 1, -1): + msg = messages[i] + if ( + self._is_actionable_user_turn(msg) + and not self._is_synthetic_compression_user_turn(msg) + ): + user_indices.append(i) + + if len(user_indices) == 0: + return cut_idx + + if len(user_indices) < n: + target_idx = user_indices[-1] + else: + target_idx = user_indices[n - 1] + + if target_idx >= cut_idx: + return cut_idx + + cut_idx = target_idx + return max(cut_idx, head_end + 1) + def _find_turn_pair_end( self, messages: List[Dict[str, Any]], @@ -4220,6 +4812,26 @@ This compaction should PRIORITISE preserving all information related to the focu # monotonic — the tail can only grow, never shrink. cut_idx = self._ensure_last_assistant_message_in_tail(messages, cut_idx, head_end) + # Extend to the last N actionable user messages when configured + # (compression.min_tail_user_messages > 1). This prevents the + # token-budget tail from consuming recent turns when large tool + # outputs fill the budget. The anchor only walks ``cut_idx`` + # backward (monotonic — the tail can only grow, never shrink), and + # a user message is a clean boundary, so the forward re-alignment + # below remains a no-op for the anchored index. Gated at the call + # site so the default (1) path is byte-identical to the historical + # single-anchor pipeline — the single-user anchor already ran above, + # and re-invoking it here could re-trigger the causal-coupling + # forward push (#22523) after the assistant anchor adjusted the cut. + # getattr-guarded: bare ``ContextCompressor.__new__`` test doubles + # (and plugin engines) skip __init__, so the attribute may be absent + # (see the compression-path test-double pitfall). + _min_tail_users = getattr(self, "min_tail_user_messages", 1) + if isinstance(_min_tail_users, int) and not isinstance(_min_tail_users, bool) and _min_tail_users > 1: + cut_idx = self._ensure_last_n_user_messages_in_tail( + messages, cut_idx, head_end, _min_tail_users, + ) + # The floor guarantees forward progress — compression must always claim # at least one message or the caller's compress_start >= compress_end # guard turns the pass into a no-op that re-runs forever (the same loop @@ -4521,6 +5133,39 @@ This compaction should PRIORITISE preserving all information related to the focu ) telemetry["chunk_count"] = 1 if turns_to_summarize else 0 + if not turns_to_summarize: + # The newest handoff summary consumed the entire compressible + # window (every window row was a standalone handoff that strips + # to None, and nothing follows it before compress_end) — there + # is nothing new to summarize. Skip the summary call entirely: + # without this guard the empty window still reached + # _generate_summary, wasting an aux LLM call that aborts + # noisily on empty input (#59496). Mirrors the sibling + # "no compressable window" guard above (#40803): record an + # ineffective strike through the durable write-through helper + # so the anti-thrash breaker in should_compress() can stop the + # loop — this shape cannot shrink, so every subsequent turn + # would otherwise re-fire the same no-op. The rehydrated + # _previous_summary is deliberately KEPT (not rolled back as + # the summary-abort path does for #57835): it came from a + # handoff genuinely present in this transcript, which is + # returned unchanged. + telemetry["failure_class"] = "empty_post_handoff_window" + self._record_ineffective_compression_verdict( + self._ineffective_compression_count + 1, + ) + self._last_compression_savings_pct = 0.0 + if not self.quiet_mode: + logger.warning( + "Compression skipped: latest context summary leaves no " + "new turns to summarize in window %d-%d. " + "ineffective_compression_count=%d", + compress_start, + compress_end, + self._ineffective_compression_count, + ) + return messages + if not self.quiet_mode: logger.info( "Context compression triggered (%d tokens >= %d threshold)", diff --git a/agent/context_engine.py b/agent/context_engine.py index 28d41e43161..b772125c0cb 100644 --- a/agent/context_engine.py +++ b/agent/context_engine.py @@ -189,6 +189,144 @@ class ContextEngine(ABC): host filters unsupported optional arguments by signature. """ + # -- Optional: proactive tool-result prune ----------------------------- + + def prune_tool_results_only( + self, + messages: List[Dict[str, Any]], + current_tokens: int | None = None, + ) -> tuple[List[Dict[str, Any]], int]: + """Deterministically trim old tool-result payloads without an LLM call. + + Runs on a low, cost-oriented trigger independent of ``should_compress`` + so large-window engines can reclaim re-sent tool output long before full + compaction would fire. Returns ``(messages, n_pruned)``. + + Default is a safe no-op: the list is returned unchanged with ``0`` + pruned. Engines that don't implement a cheap prune — and any engine that + predates this hook — inherit this default, so the agent loop's + post-tool-call prune path never raises ``AttributeError`` on them. The + built-in ContextCompressor overrides this with the real implementation. + """ + return messages, 0 + + # -- Optional: per-turn context selection (distinct from compression) -- + + def select_context( + self, + request_messages: List[Dict[str, Any]], + *, + conversation_messages: List[Dict[str, Any]] = None, + incoming_message: Dict[str, Any] = None, + budget_tokens: int = 0, + ) -> List[Dict[str, Any]]: + """Optionally choose/replace the context for THIS request, pre-generation. + + Called every turn after the request message list is assembled and + before it is dispatched to the provider — independent of + ``should_compress()``. This lets an engine *select* which context + enters the prompt (retrieval, topic routing, role/branch switching) + rather than *shrink* context that is already there. The two verbs are + orthogonal: + + - ``compress()`` : context is too long -> make it shorter. + - ``select_context()``: this turn belongs to a different context + -> use that one instead. + + Without this hook, engines that need per-turn access to the message + list have to force ``should_compress()`` to return ``True`` so that + ``compress()`` is invoked every turn purely as a callback — which + conflates selection with compression and degrades behaviour when the + engine's backend is unavailable. ``select_context()`` removes the need + for that workaround. + + The returned list is request-only: it replaces the messages sent to + the provider for this single call and MUST NOT be treated as persisted + transcript state. The conversation history in the session DB is left + untouched, so nothing leaks across turns. Return ``None`` to leave the + request unchanged. + + Unlike the ``pre_llm_call`` plugin hook (which appends to the user + message and intentionally never rewrites the list, to preserve the + cache prefix), ``select_context()`` may *replace* the message list. + + Ordering / cache contract: the host runs this hook **before** prompt + cache-control and **before** every request sanitizer (orphaned-tool + cleanup, thinking-only/role normalization, whitespace/JSON + normalization). So (a) whatever the hook returns still passes through + the same validation as any request — a malformed replacement cannot + reach the provider — and (b) prompt-cache stability (an AGENTS.md + invariant) is preserved: the default no-op leaves the request + byte-identical, so cache behaviour is unchanged for the built-in + compressor and any non-implementing engine. An engine that *does* + replace the list changes its own cache prefix by definition; that is + the engine's concern, and cache-control breakpoints are re-derived on + the selected list. The hook is evaluated per provider request (so it + re-runs on retries within a turn), consistent with "select the context + for THIS request". + + Args: + request_messages: The assembled request message list (system + prompt + history + any ephemeral prefill), in OpenAI format. + conversation_messages: The unmodified persisted conversation + history, for reference only (do not mutate). + incoming_message: The current turn's user message, if available. + budget_tokens: The active model's context length, or 0 if unknown. + + Default returns ``None`` (no-op) — zero impact on the built-in + compressor or any existing engine. + """ + return None + + def on_turn_complete( + self, + messages: List[Dict[str, Any]], + usage: Dict[str, Any] = None, + **kwargs: Any, + ) -> None: + """Observe a finished user turn (post-turn ingestion / observation). + + Called from the standard turn-finalization path once the assistant/tool + loop completes, with the finalized in-memory transcript snapshot. This + is the complement to ``select_context()``: selection happens *before* + the request, while observation happens *after* the turn. It lets an + engine ingest, index, summarize, or update routing / topic / session + state from what actually happened — so the next ``select_context()`` + can act on it. + + Coverage: this fires from the normal finalization seam. Some abnormal + early-return paths in the loop (e.g. a content-policy block or a + provider terminal failure) persist and return without routing through + finalization, and therefore do not currently emit this hook. Treat it + as a best-effort post-turn observation for completed turns, not a + guaranteed callback for every possible early exit; unifying all + terminal paths behind one finalization seam is a separate follow-up. + + Together the two hooks remove the need to abuse ``should_compress()`` / + ``compress()`` as a generic per-turn callback just to observe history, + and they cover the case where a turn finishes and there may be no next + request from which to infer the previous turn. + + ``messages`` is a shallow copy and should be treated as read-only: + return values are ignored and this hook must not rely on transcript + mutation for persistence. ``kwargs`` may include ``turn_id``, + ``task_id``, ``api_call_count``, ``interrupted``, ``failed``, and + ``turn_exit_reason``. + + ``usage`` carries the completed turn's canonical token usage (the same + dict shape passed to ``update_from_response`` — ``prompt_tokens`` / + ``completion_tokens`` / ``total_tokens`` plus the canonical + ``input_tokens`` / ``output_tokens`` / ``cache_read_tokens`` / + ``cache_write_tokens`` / ``reasoning_tokens`` buckets) so an engine can + weigh how large/expensive the selected context actually was when + deciding the next ``select_context()``. It is ``None`` on finalized + turns that never reached a provider response (e.g. interrupt); engines + must treat it as optional. + + Default is a no-op. + """ + return None + # -- Optional: pre-flight check ---------------------------------------- def should_compress_preflight(self, messages: List[Dict[str, Any]]) -> bool: diff --git a/agent/context_references.py b/agent/context_references.py index eea16ae52b4..8981aa472f5 100644 --- a/agent/context_references.py +++ b/agent/context_references.py @@ -19,6 +19,7 @@ REFERENCE_PATTERN = re.compile( rf"(?diff|staged)\b|(?Pfile|folder|git|url):(?P{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))" ) TRAILING_PUNCTUATION = ",.;!?" +_NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""") _SENSITIVE_HOME_DIRS = (".ssh", ".aws", ".gnupg", ".kube", ".docker", ".azure", ".config/gh") _SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",) _SENSITIVE_HOME_FILES = ( @@ -60,6 +61,21 @@ class ContextReferenceResult: blocked: bool = False +def format_reference_value(value: str) -> str: + """Quote a reference value so ``REFERENCE_PATTERN`` reads it back whole. + + The unquoted alternative in the pattern is ``\\S+``, so a path containing a + space parses as a truncated ref with the tail left behind as loose text. + Mirrors ``formatRefValue`` in the desktop's directive-text.tsx. + """ + if not _NEEDS_QUOTING.search(value): + return value + for quote in ("`", '"', "'"): + if quote not in value: + return f"{quote}{value}{quote}" + return value + + def parse_context_references(message: str) -> list[ContextReference]: refs: list[ContextReference] = [] if not message: @@ -308,7 +324,7 @@ def _expand_git_reference( ["git", *args], cwd=cwd, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=30, stdin=subprocess.DEVNULL, **_popen_kwargs, @@ -534,7 +550,7 @@ def _rg_files(path: Path, cwd: Path, limit: int) -> list[Path] | None: ["rg", "--files", str(path.relative_to(cwd))], cwd=cwd, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=10, stdin=subprocess.DEVNULL, **_popen_kwargs, diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index b2357d26be5..d2df6f25a39 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -40,7 +40,7 @@ import uuid import threading from datetime import datetime from pathlib import Path -from typing import Any, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from agent.context_engine import ( automatic_compaction_status_message, @@ -221,6 +221,25 @@ class CompressionCommitFence: self._lock = threading.Lock() self._cancelled = False self._commit_started = False + # Forward-progress telemetry: the compression worker touches this + # whenever the streamed summary call produces a token (see + # ContextCompressor._call_summary_llm). Waiters use it to distinguish + # a SLOW-but-alive summary model from a HUNG one, so slow models are + # not killed by a fixed wall-clock deadline while tokens are moving. + self._last_progress = time.monotonic() + + def touch_progress(self) -> None: + """Record forward progress (e.g. a streamed summary token arriving). + + Called from the compression worker thread; read by async waiters via + :meth:`seconds_since_progress`. A bare float store is atomic in + CPython, so no lock is needed. + """ + self._last_progress = time.monotonic() + + def seconds_since_progress(self) -> float: + """Seconds since the worker last reported forward progress.""" + return max(0.0, time.monotonic() - self._last_progress) def cancel_before_commit(self) -> bool: """Cancel a pending commit, or wait for an active commit to finish. @@ -355,6 +374,143 @@ def _emit_compression_attempt_telemetry( logger.debug("failed to emit compression attempt telemetry: %s", exc) +def compression_skipped_due_to_lock(agent: Any) -> bool: + """Type-pinned read of the #69870 lock-skip signal. + + ``agent._compression_skipped_due_to_lock`` is set by ``compress_context`` + when a compression pass no-ops because another path holds the per-session + compression lock (holder string when the holder was confirmed, ``True`` + otherwise) and cleared to ``None`` at the entry of every call. + + The read MUST be type-pinned (``is True or isinstance(x, str)``), never + bare truthiness: MagicMock test-double agents auto-create truthy + attributes, and a bare ``if getattr(agent, ...)`` would hijack every + mocked agent in sibling suites into the lock-skip branch (the + #69870 × #69840 type-ahead incident). + """ + _sig = getattr(agent, "_compression_skipped_due_to_lock", None) + return _sig is True or isinstance(_sig, str) + + +def _adopt_live_compression_child( + agent: Any, + session_db: Any, + parent_session_id: str, +) -> Optional[List[Dict[str, Any]]]: + """Move a stale compression contender onto the unique durable child. + + Resolve and load first, then mutate the live agent. This ordering keeps the + stale contender fail-closed when lineage is ambiguous or the compacted + handoff cannot be read. + """ + finder = getattr(type(session_db), "find_live_compression_child", None) + loader = getattr(type(session_db), "get_messages_as_conversation", None) + if not callable(finder) or not callable(loader): + return None + child = finder(session_db, parent_session_id) + if not child or not child.get("id"): + return None + child_session_id = str(child["id"]) + recovered = loader(session_db, child_session_id) + if not isinstance(recovered, list) or not recovered: + return None + # Revalidate after loading: the child may have rotated or a competing + # continuation may have appeared between the two DB reads. + confirmed = finder(session_db, parent_session_id) + if not confirmed or str(confirmed.get("id") or "") != child_session_id: + return None + + agent.session_id = child_session_id + try: + from gateway.session_context import set_current_session_id + + set_current_session_id(child_session_id) + except Exception: + os.environ["HERMES_SESSION_ID"] = child_session_id + try: + from hermes_logging import set_session_context + + set_session_context(child_session_id) + except Exception: + pass + + agent._session_db_created = True + if child.get("system_prompt"): + agent._cached_system_prompt = child["system_prompt"] + agent._last_flushed_db_idx = len(recovered) + agent._flushed_db_message_session_id = child_session_id + agent._flushed_db_message_ids = { + id(message) for message in recovered if isinstance(message, dict) + } + + on_session_start = getattr(agent.context_compressor, "on_session_start", None) + if callable(on_session_start): + try: + on_session_start( + child_session_id, + boundary_reason="compression", + old_session_id=parent_session_id, + session_db=session_db, + platform=getattr(agent, "platform", None) or "cli", + conversation_id=getattr(agent, "_gateway_session_key", None), + ) + except Exception as exc: + logger.debug("context engine compression-child adoption failed: %s", exc) + else: + bind_state = getattr(agent.context_compressor, "bind_session_state", None) + if callable(bind_state): + try: + bind_state(session_db=session_db, session_id=child_session_id) + except Exception: + pass + try: + if agent._memory_manager: + agent._memory_manager.on_session_switch( + child_session_id, + parent_session_id=parent_session_id, + reset=False, + reason="compression", + ) + except Exception as exc: + logger.debug("memory manager compression-child adoption failed: %s", exc) + + return recovered + + +def recover_rotated_compression_session( + agent: Any, +) -> Optional[List[Dict[str, Any]]]: + """Recover a stale live agent before a new turn writes to its old parent.""" + session_db = getattr(agent, "_session_db", None) + session_id = getattr(agent, "session_id", None) or "" + if session_db is None or not session_id: + return None + try: + if not _session_was_rotated_by_compression(session_db, session_id): + return None + # Rotation publication holds the parent compression lease until the + # child handoff is durable. A concurrent turn waits briefly rather than + # observing the intentional parent-ended/child-empty intermediate state. + holder_getter = getattr(session_db, "get_compression_lock_holder", None) + for attempt in range(21): + recovered = _adopt_live_compression_child(agent, session_db, session_id) + if recovered is not None: + return recovered + holder = holder_getter(session_id) if callable(holder_getter) else None + if not holder or attempt == 20: + return None + time.sleep(0.05) + return None + except Exception as exc: + logger.warning( + "compression session recovery failed for session=%s (%s: %s)", + session_id, + type(exc).__name__, + exc, + ) + return None + + def _compression_lock_holder(agent: Any) -> str: """Build a unique holder id for the lock: pid:tid:agent-instance:uuid. @@ -515,7 +671,17 @@ class _CompressionLockLeaseRefresher: # by the TTL the acquirer set — the lock can never be held past its TTL # by a stuck refresher. consecutive_failures = 0 - while not self._stop.wait(self._refresh_interval_seconds): + # First refresh happens immediately, not one interval late. Everything + # between try_acquire() and start() (the rotation-ownership lookup, the + # durable-breaker re-read, thread startup) is charged against the very + # first lease, so on a short TTL under load the lock could already be + # expired — and reclaimable by a competing path — before tick #1. + first = True + while first or not self._stop.wait(self._refresh_interval_seconds): + if first: + first = False + if self._stop.is_set(): + break try: refreshed = self._db.refresh_compression_lock( self._session_id, @@ -874,6 +1040,7 @@ _SYNTHETIC_USER_FLAGS = ( "_empty_recovery_synthetic", "_verification_stop_synthetic", "_pre_verify_synthetic", + "_dropped_toolcall_nudge", ) @@ -1143,6 +1310,14 @@ def compress_context( # boundary, so the previous flush baseline remains authoritative. agent._last_compression_attempt_recorded = True agent._last_compression_attempt_in_place = None + # Clear the lock-skip signal at the VERY TOP, before the codex route and + # the breaker gates below can early-return (per-attempt state rule, + # #58630/#69853). A stale ``True``/holder value from a prior lock-skip + # must never make a later breaker/codex no-op look like lock contention + # to the automatic-path consumers (compression_deferred, #49874) — the + # second clear before lock acquisition below stays for the same reason + # it was added in #69870 and is simply idempotent now. + agent._compression_skipped_due_to_lock = None _attempt_started_at = time.monotonic() _attempt_id = uuid.uuid4().hex @@ -1226,7 +1401,7 @@ def compress_context( # parent_session_id child, no # `name #N` renumber, no contextvar/env/logging re-sync, no memory/context- # engine session-switch. The conversation keeps one durable id for life, - # eliminating the session-rotation bug cluster. Default False during rollout. + # eliminating the session-rotation bug cluster. Default True (2107b86024). in_place = bool(getattr(agent, "compression_in_place", False)) # Set True once the in-place DB write actually completes (the DB block can # raise and skip it). Surfaced to the gateway via agent._last_compaction_in_place. @@ -1434,6 +1609,8 @@ def compress_context( if _lock_released: return _lock_released = True + if getattr(agent, "_active_compression_lock_holder", None) == _lock_holder: + agent._active_compression_lock_holder = None if _lock_refresher is not None: try: _lock_refresher.stop() @@ -1445,6 +1622,9 @@ def compress_context( except Exception as _rel_err: logger.debug("compression lock release failed: %s", _rel_err) + if _lock_holder is not None: + agent._active_compression_lock_holder = _lock_holder + # A delayed contender can acquire the parent lock after the winning path # has released it and completed rotation. The lock serializes work but does # not by itself prove that this stale agent still owns a live parent. @@ -1467,15 +1647,25 @@ def compress_context( _existing_sp = agent._build_system_prompt(system_message) return messages, _existing_sp if _parent_already_rotated: - logger.info( - "compression skipped: session=%s was already rotated by " - "another compression path", - _lock_sid, + recovered_messages = _adopt_live_compression_child( + agent, _lock_db, _lock_sid ) _release_lock() _existing_sp = getattr(agent, "_cached_system_prompt", None) if not _existing_sp: _existing_sp = agent._build_system_prompt(system_message) + if recovered_messages is not None: + logger.warning( + "compression recovery: stale session=%s adopted live child=%s", + _lock_sid, + agent.session_id, + ) + return recovered_messages, _existing_sp + logger.warning( + "compression skipped: session=%s was already rotated by " + "another compression path, but no unique live child could be adopted", + _lock_sid, + ) return messages, _existing_sp # The agent may have been constructed before another path completed an @@ -1509,6 +1699,37 @@ def compress_context( ) _lock_refresher.start() + # The caller's history snapshot predates lease acquisition. Reload the + # durable parent after the lease is live; MORE durable rows than the + # snapshot carries means a frontend/background writer committed a turn + # in that window, so publishing from this snapshot would omit it. + # Deliberately a LENGTH check, not content equality: in-memory + # mutation of past turns is legal (multimodal compression, retry + # history replacement, think-tag stripping), and a content-equality + # abort would permanently wedge compression on such sessions — the + # #14694 failure shape. + # Rotation-only: in-place compaction (archive_and_compact) is + # non-destructive — pre-compaction rows are soft-archived (active=0, + # compacted=1), stay searchable and recoverable, so snapshot/durable + # drift cannot lose data there and must not abort compaction. + if not in_place and _lock_db is not None and _lock_sid: + durable_loader = getattr( + type(_lock_db), "get_messages_as_conversation", None + ) + if callable(durable_loader): + durable_parent = durable_loader(_lock_db, _lock_sid) + if isinstance(durable_parent, list) and len(durable_parent) > len(messages): + logger.warning( + "compression aborted: session=%s changed before lease " + "acquisition; preserving newer durable messages", + _lock_sid, + ) + _release_lock() + existing_prompt = getattr(agent, "_cached_system_prompt", None) + if not existing_prompt: + existing_prompt = agent._build_system_prompt(system_message) + return messages, existing_prompt + # Notify external memory provider before compression discards context. # The provider's on_pre_compress() may return a string of insights it # wants surfaced inside the compression summary; capture and forward it @@ -1549,7 +1770,29 @@ def compress_context( messages_before_compression = copy.deepcopy(messages) _activity_heartbeat = _CompressionActivityHeartbeat(agent).start() - compressed = compress_fn(messages, **compress_kwargs) + # Publish forward progress to the commit fence while the summary LLM + # call streams. Async hosts (gateway session hygiene) poll + # ``commit_fence.seconds_since_progress()`` to extend their deadline + # while tokens are moving — so a SLOW summary model is only killed + # when it is actually silent, not merely thorough. The hook is + # thread-local and the compress call is synchronous on this thread, + # so it cannot leak into unrelated auxiliary calls. + # + # Fenceless callers (CLI /compress, in-loop auto-compress) install a + # no-op hook: nobody polls their progress, but an ACTIVE hook is what + # switches the summary call onto the streamed path — giving every + # compression path the same two guarantees: the configured timeout + # acts on inactivity (slow models finish), and a byte-trickling + # provider that keeps the connection alive forever is cut off at the + # streamed total ceiling (see _aux_stream_total_ceiling) instead of + # outliving the SDK's inactivity timeout indefinitely. + from agent.auxiliary_client import aux_progress_hook + _progress_hook = ( + commit_fence.touch_progress if commit_fence is not None + else (lambda: None) + ) + with aux_progress_hook(_progress_hook): + compressed = compress_fn(messages, **compress_kwargs) except BaseException as _compress_exc: # ANY exception after lock acquisition — memory hook, capability # inspection, engine lookup, or compress() — must release the lock so @@ -1777,6 +2020,21 @@ def compress_context( ): new_system_prompt = cached_system_prompt agent._cached_system_prompt = cached_system_prompt + # _invalidate_system_prompt() above also cleared the + # cross-session-stable prefix marker boundary. The kept prompt + # is byte-identical, so reconstruct the stable tier and reuse + # it ONLY when the kept prompt still literally starts with it + # (same startswith gate as the restore path); otherwise the + # request layer falls back to the legacy single-breakpoint + # layout with the prompt bytes untouched. + try: + from agent.system_prompt import build_system_prompt_parts as _build_parts + + _static = _build_parts(agent, system_message=system_message)["stable"] + if _static and cached_system_prompt.startswith(_static): + agent._cached_system_prompt_static = _static + except Exception: + pass else: new_system_prompt = agent._build_system_prompt(system_message) agent._cached_system_prompt = new_system_prompt @@ -1853,81 +2111,54 @@ def compress_context( ) except Exception: pass # best-effort — don't block compression on a flush error - # Propagate title to the new session with auto-numbering + # Publish parent closure + child row + compacted handoff in + # one transaction. No reader can observe a missing/empty child. + # The rotation child must stay on the parent's profile — + # mirror _ensure_db_session's stamp ("default" persists as + # NULL). publish_compression_child additionally COALESCEs + # from the parent row, covering app-global remote sessions + # whose thread lacks the HERMES_HOME context. + try: + from hermes_cli.profiles import get_active_profile_name + + _profile_for_child = get_active_profile_name() + if _profile_for_child == "default": + _profile_for_child = None + except Exception: + _profile_for_child = None old_title = agent._session_db.get_session_title(agent.session_id) - agent._session_db.end_session(agent.session_id, "compression") old_session_id = agent.session_id - agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}" - # Ordering contract: the agent thread updates the contextvar here; - # the gateway propagates to SessionEntry after run_in_executor returns. + new_session_id = ( + f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_" + f"{uuid.uuid4().hex[:6]}" + ) + agent._session_db.publish_compression_child( + parent_session_id=old_session_id, + child_session_id=new_session_id, + source=agent.platform + or os.environ.get("HERMES_SESSION_SOURCE", "cli"), + model=agent.model, + model_config=agent._session_init_model_config, + system_prompt=new_system_prompt, + messages=compressed, + cwd=getattr(agent, "working_directory", None), + profile_name=_profile_for_child, + compression_lock_holder=_lock_holder, + require_compression_lease=_lock_holder is not None, + ) + agent.session_id = new_session_id try: from gateway.session_context import set_current_session_id set_current_session_id(agent.session_id) except Exception: os.environ["HERMES_SESSION_ID"] = agent.session_id - # The gateway/tools session context (ContextVar + env) and the - # logging session context are SEPARATE mechanisms. The call above - # moves the former; the ``[session_id]`` tag on log lines comes - # from ``hermes_logging._session_context`` (set once per turn in - # conversation_loop.py). Without this, post-rotation log lines in - # the same turn keep the STALE old id while the message/DB/gateway - # state carry the new one — breaking log correlation exactly at the - # compaction boundary (see #34089). Guarded separately so a logging - # failure can never regress the routing update above. try: from hermes_logging import set_session_context set_session_context(agent.session_id) except Exception: pass - agent._session_db_created = False - try: - agent._session_db.create_session( - session_id=agent.session_id, - source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), - model=agent.model, - model_config=agent._session_init_model_config, - parent_session_id=old_session_id, - ) - except Exception as _cs_err: - # The child row could not be created (e.g. FK constraint, - # contended write). Previously the outer handler simply - # warned and let the agent continue on the NEW id — which - # has no row in state.db, producing an orphan: the parent - # is ended, the child is never indexed, and every - # subsequent message is attributed to a session that - # doesn't exist (#33906/#33907). Roll the live id back to - # the parent so the conversation stays attached to a real, - # indexed session instead of a phantom. - logger.warning( - "Compression child session create failed (%s) — " - "rolling back to parent session %s to avoid an orphan.", - _cs_err, old_session_id, - ) - agent.session_id = old_session_id - try: - from gateway.session_context import set_current_session_id - set_current_session_id(agent.session_id) - except Exception: - os.environ["HERMES_SESSION_ID"] = agent.session_id - try: - from hermes_logging import set_session_context - set_session_context(agent.session_id) - except Exception: - pass - # Re-open the parent: it was ended above, but we're - # continuing on it, so it must not stay closed. - try: - agent._session_db.reopen_session(old_session_id) - except Exception: - pass - old_session_id = None # no rotation happened - # The parent row already exists in state.db, so mark the - # session as created — _ensure_db_session would otherwise - # retry a (harmless INSERT OR IGNORE) create next turn. - agent._session_db_created = True - raise agent._session_db_created = True split_status = "rotated_committed" # Carry a persistent /goal onto the continuation session. @@ -1947,18 +2178,14 @@ def compress_context( except (ValueError, Exception) as e: logger.debug("Could not propagate title on compression: %s", e) - # Shared post-write steps (both modes target agent.session_id, which - # in-place keeps and rotation has already reassigned to the new id): - # refresh the stored system prompt and reset the flush cursor so the - # next turn re-bases its append diff. - agent._session_db.update_system_prompt(agent.session_id, new_system_prompt) + # In-place mode still updates/replaces the current row here. + # Rotation already published prompt + compacted handoff atomically. if in_place: + agent._session_db.update_system_prompt( + agent.session_id, new_system_prompt + ) agent._last_flushed_db_idx = 0 else: - # A headless turn can be killed before its finalizer. Persist - # the rotated child's compacted handoff at the boundary so - # the new session is immediately resumable. - agent._session_db.replace_messages(agent.session_id, compressed) agent._last_flushed_db_idx = len(compressed) agent._flushed_db_message_session_id = agent.session_id agent._flushed_db_message_ids = { @@ -1968,7 +2195,22 @@ def compress_context( } _session_commit_succeeded = True except Exception as e: - split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed" + if ( + not in_place + and locals().get("old_session_id") + and agent.session_id == old_session_id + ): + # Atomic publication failed (including lease loss): keep the + # parent live and discard the stale compacted snapshot. + old_session_id = None + messages[:] = copy.deepcopy(messages_before_compression) + compressed = messages + _compression_made_progress = False + split_status = ( + "aborted" + if locals().get("old_session_id") is None and not in_place + else "failed_not_indexed" + ) # If the rotation rolled back to the parent (orphan-avoidance # above), agent.session_id is the still-indexed parent and # old_session_id was cleared — so this is recovery, not an diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index d952f092795..fb3d7978b93 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -34,6 +34,7 @@ from agent.conversation_compression import ( COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE, COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE, PRE_API_COMPRESSION_STATUS_TEMPLATE, + compression_skipped_due_to_lock, conversation_history_after_compression, ) from agent.context_engine import automatic_compaction_status_message @@ -47,6 +48,7 @@ from agent.turn_context import ( reanchor_current_turn_user_idx, ) from agent.turn_retry_state import TurnRetryState +from agent.runtime_cwd import resolve_agent_cwd from agent.message_sanitization import ( close_interrupted_tool_sequence, _repair_tool_call_arguments, @@ -435,6 +437,37 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # Continuing session — reuse the exact system prompt from the # previous turn so the Anthropic cache prefix matches. agent._cached_system_prompt = stored_prompt + # Reconstruct the cross-session-stable prefix for the early cache + # breakpoint. The static prefix is not persisted (only the full + # prompt is), so gateway surfaces that build a fresh AIAgent per + # turn would otherwise lose the two-block system layout after the + # first turn — flip-flopping the wire shape mid-conversation and + # silently degrading to the legacy single-breakpoint layout. + # + # Safety: the rebuilt stable tier is used ONLY when the restored + # prompt literally starts with it (checked here AND re-checked by + # ``_apply_system_cache_markers``'s ``startswith`` gate). If any + # stable-tier input changed since the prompt was persisted (skills + # edited, identity changed), the prefix mismatches, ``_static`` + # stays None, and the request falls back to the legacy layout with + # the restored prompt bytes untouched — never a rewritten prompt. + # + # Gated on ``_use_prompt_caching`` so non-Anthropic routes skip the + # rebuild entirely (the static prefix is only consumed by + # ``apply_anthropic_cache_control``). + if getattr(agent, "_use_prompt_caching", False): + try: + from agent.system_prompt import build_system_prompt_parts as _build_parts + + _static = _build_parts(agent, system_message=system_message)["stable"] + if _static and stored_prompt.startswith(_static): + agent._cached_system_prompt_static = _static + except Exception: + # Fail-open: restore continues with the legacy cache layout. + logger.debug( + "static system-prefix reconstruction failed on restore", + exc_info=True, + ) return if stored_prompt: stored_state = "stale_runtime" @@ -507,9 +540,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) def _stored_prompt_matches_runtime(agent, prompt: str) -> bool: - """Return False when the persisted Model/Provider lines are stale.""" + """Return False when the persisted runtime-identity lines are stale.""" def line_value(label: str) -> str: + """Last matching line wins. + + Safe ONLY for fields emitted in the volatile tier at the very END of + the prompt (Model / Provider / Platform). User-supplied project + context (AGENTS.md / CLAUDE.md / .cursorrules) is embedded in the + middle context tier, so a last-match scan lets project prose shadow + any field emitted EARLIER — see ``host_info_value``. + """ prefix = f"{label}:" value = "" for line in prompt.splitlines(): @@ -517,6 +558,32 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool: value = line[len(prefix):].strip() return value + def host_info_value(label: str) -> str: + """Read a field from the prompt's own host-info block. + + The host-info block (``build_environment_hints``) sits in the STABLE + tier, ahead of the embedded project context files. A bare scan of the + whole prompt would therefore match a user's ``AGENTS.md`` that merely + contains a line starting with the same label, comparing runtime state + against project prose. That mismatch never clears, so the check would + reject the stored prompt on EVERY turn — rebuilding the system prompt + each message and destroying the prefix cache for the whole session, + which is far worse than the staleness this function guards against. + + Anchor on the ``User home directory:`` line that immediately precedes + the working-directory line in that block, and take the FIRST such + occurrence, so only Hermes' own emitted block can satisfy the read. + """ + prefix = f"{label}:" + lines = prompt.splitlines() + for idx, line in enumerate(lines): + if not line.startswith("User home directory:"): + continue + for candidate in lines[idx + 1: idx + 4]: + if candidate.startswith(prefix): + return candidate[len(prefix):].strip() + return "" + stored_model = line_value("Model") current_model = str(getattr(agent, "model", "") or "").strip() if stored_model and current_model and stored_model != current_model: @@ -527,6 +594,24 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool: if stored_provider and current_provider and stored_provider != current_provider: return False + # Detect cwd drift: if the stored prompt was built in a different working + # directory, reuse would silently inject a stale path into the prefix cache. + # Compare against resolve_agent_cwd() — the SAME resolver used to build the + # prompt — so gateway/TUI sessions that set TERMINAL_CWD are not falsely + # rejected (they would always differ from the launch dir's os.getcwd()). + stored_cwd = host_info_value("Current working directory") + if stored_cwd: + if stored_cwd != str(resolve_agent_cwd()): + return False + + # Detect runtime-surface drift: the stored prompt records which platform it + # was built for (e.g. "desktop" vs "cli"). Reusing a desktop-built prompt on + # a terminal session (or vice versa) would inject the wrong runtime hints. + stored_platform = line_value("Platform") + current_platform = str(getattr(agent, "platform", "") or "").strip() + if stored_platform and current_platform and stored_platform != current_platform: + return False + return True @@ -640,6 +725,55 @@ def _content_policy_blocked_result( } +def _compression_deferred_result( + agent, + messages: List[Dict], + api_call_count: int, +) -> Dict[str, Any]: + """Build the soft turn result for a lock-contended compression defer. + + Another path (a sibling turn, a background review fork, a manual + ``/compress``) holds this session's compression lock, so every + compression pass this turn no-oped and the request still does not fit. + This is a TEMPORARY condition — the lock winner is actively shrinking + the same session — so the turn must end as a soft defer + (``compression_deferred``), never as ``compression_exhausted``: the + gateway auto-resets (wipes) the session on exhaustion (#9893/#35809), + which would destroy a session that the concurrent compressor is about + to make healthy again. + + ``failed`` stays False so the gateway persists the user turn (transient + branch) and retry-next-message semantics apply. + """ + holder = getattr(agent, "_compression_skipped_due_to_lock", None) + logger.info( + "turn deferred: compression lock held by another path " + "(session=%s holder=%s) — not counting as compression exhaustion", + agent.session_id or "none", + holder if isinstance(holder, str) else "unconfirmed", + ) + try: + agent._flush_status_buffer() + except Exception: + pass + _final = ( + "Context compression is already running for this session. " + "Please retry in a moment — your next message will be processed " + "once the concurrent compression finishes." + ) + return { + "final_response": _final, + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": _final, + "partial": True, + "failed": False, + "compression_deferred": True, + "session_id": agent.session_id, + } + + def _sync_failover_system_message(agent, api_messages, active_system_prompt): """Refresh the in-flight system message after a provider failover. @@ -666,6 +800,136 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt): return sp +def _apply_context_engine_selection( + agent: Any, + api_messages: List[Dict[str, Any]], + conversation_messages: List[Dict[str, Any]], + incoming_message: Optional[Dict[str, Any]], + *, + logger: Any, +) -> List[Dict[str, Any]]: + """Run the optional per-turn ``ContextEngine.select_context()`` hook. + + Returns the (possibly replaced) request message list. The hook is for + context *selection / routing* (retrieval, topic routing, role switching), + which is distinct from compression and fires every turn independent of + ``should_compress()``. + + Fail-open by design: a missing hook, any exception, or an invalid return + value yields the unmodified ``api_messages``. The result is request-only — + persisted conversation history is never mutated here. + """ + engine = getattr(agent, "context_compressor", None) + if engine is None or not hasattr(engine, "select_context"): + return api_messages + + # Skip the no-op base implementation so non-implementing engines — + # including the built-in ContextCompressor — pay nothing per request: + # no history copies below, no call. ``hasattr`` alone is not enough, + # because the ABC defines a default ``select_context`` that every engine + # inherits. Mirrors the base-method short-circuit in + # ``_notify_context_engine_turn_complete``. Lazy import avoids any import + # cycle with agent.context_engine. + try: + from agent.context_engine import ContextEngine as _CE + if getattr(engine.select_context, "__func__", None) is _CE.select_context: + return api_messages + except Exception: + pass + + session_label = getattr(agent, "session_id", None) or "-" + # Pass shallow copies of the reference-only inputs so an engine that + # mutates them in place cannot alter persisted transcript state. Only + # ``request_messages`` (the per-call request list) is meant to be acted on, + # and it may be replaced wholesale via the return value — never mutated in + # place either. ``conversation_messages`` / ``incoming_message`` are + # read-only context; copying enforces the request-only contract rather than + # merely documenting it. + _conv_copy = [dict(m) if isinstance(m, dict) else m for m in conversation_messages] \ + if conversation_messages is not None else None + _incoming_copy = dict(incoming_message) if isinstance(incoming_message, dict) else incoming_message + try: + selected = engine.select_context( + api_messages, + conversation_messages=_conv_copy, + incoming_message=_incoming_copy, + budget_tokens=getattr(engine, "context_length", 0) or 0, + ) + except Exception: + logger.warning( + "Context engine select_context hook failed; using unmodified " + "request messages (session=%s)", + session_label, + exc_info=True, + ) + return api_messages + + if selected is None: + return api_messages + # Require a NON-EMPTY list of dicts. An empty list must fall open to the + # original request: ``all([])`` is ``True``, so without the emptiness check + # a ``[]`` returned by a buggy/failing engine would replace a valid request + # with an empty message list that the downstream sanitizers cannot restore, + # reaching the provider as an invalid request instead of failing open. + if isinstance(selected, list) and selected and all(isinstance(m, dict) for m in selected): + return selected + + logger.warning( + "Context engine select_context returned an invalid value " + "(not a non-empty list of dicts); ignoring (session=%s)", + session_label, + ) + return api_messages + + +def _notify_context_engine_turn_complete( + agent: Any, + messages: List[Dict[str, Any]], + *, + usage: Optional[Dict[str, Any]] = None, + logger: Any, + **meta: Any, +) -> None: + """Notify the active context engine that a user turn has finished. + + Calls the optional ``ContextEngine.on_turn_complete()`` observation hook + once per turn, after the assistant/tool loop has produced the finalized + transcript. The complement to ``select_context()`` (pre-request selection): + this lets an engine ingest / index / summarize the completed turn. + + Fail-open: a missing or no-op hook, or any exception, is swallowed. + ``messages`` is passed as a shallow copy so the engine cannot mutate the + persisted transcript. + """ + engine = getattr(agent, "context_compressor", None) + hook = getattr(engine, "on_turn_complete", None) + if engine is None or not callable(hook): + return + + # Skip the no-op base implementation so non-implementing engines (incl. + # the built-in compressor) pay nothing per turn. Lazy import avoids any + # import cycle with agent.context_engine. + try: + from agent.context_engine import ContextEngine as _CE + if getattr(hook, "__func__", None) is _CE.on_turn_complete: + return + except Exception: + pass + + try: + hook( + [dict(m) if isinstance(m, dict) else m for m in messages], + usage=usage, + **meta, + ) + except Exception: + logger.warning( + "Context engine on_turn_complete hook failed (session=%s)", + getattr(agent, "session_id", None) or "-", + exc_info=True, + ) + + def run_conversation( agent, user_message: Any, @@ -805,6 +1069,13 @@ def run_conversation( # over instead of spinning. Reset here so each turn starts fresh. See #26080. agent._auth_pool_refresh_counts = {} + # Reset the per-turn usage holder forwarded to the context engine's + # on_turn_complete() observation hook. Set after each successful provider + # response (see below); left as None on turns that never reach a response + # (early failure / interrupt) so the hook receives None rather than a + # stale prior turn's usage. + agent._last_turn_usage = None + # Optional opt-in runtime: if api_mode == codex_app_server, hand the # turn to the codex app-server subprocess (terminal/file ops/patching # all run inside Codex). Default Hermes path is bypassed entirely. @@ -1053,7 +1324,28 @@ def run_conversation( # Uses new dicts so the internal messages list retains the fields # for Codex Responses compatibility. if agent._should_sanitize_tool_calls(): - agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model) + # In MoA mode, agent.model is the virtual preset name + # (e.g. "closed"), not the actual aggregator model. Use + # the resolved aggregator model so Gemini aggregators + # correctly preserve thought_signature (extra_content). + _sanitize_model = agent.model + if agent.provider == "moa": + if moa_config: + _agg = moa_config.get("aggregator") or {} + if _agg.get("model"): + _sanitize_model = _agg["model"] + if _sanitize_model == agent.model: + # Virtual-provider mode: no moa_config is threaded + # through run_conversation — the facade resolves the + # preset internally. Ask the facade for the resolved + # aggregator slot from the previous create() instead + # (set before any history replay that could carry + # thought_signature). + _moa_client = getattr(agent, "client", None) + _agg_slot = getattr(_moa_client, "last_aggregator_slot", None) + if _agg_slot and _agg_slot.get("model"): + _sanitize_model = _agg_slot["model"] + agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model) # Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context # The signature field helps maintain reasoning continuity api_messages.append(api_msg) @@ -1070,9 +1362,9 @@ def run_conversation( # # Hermes invariant: the system prompt is built ONCE per session # (cached on ``_cached_system_prompt``) and replayed verbatim on - # every turn. We send it as a single content string so the - # bytes are byte-stable across turns and upstream prompt caches - # stay warm. + # every turn. ``apply_anthropic_cache_control`` may split its stable + # prefix into content blocks on the wire, but the stored string and + # its byte-stability remain unchanged. effective_system = active_system_prompt or "" if agent.ephemeral_system_prompt: effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip() @@ -1099,7 +1391,18 @@ def run_conversation( aggregator=moa_config.get("aggregator") or {}, temperature=_preset_temperature(moa_config, "reference_temperature"), aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"), - max_tokens=moa_config.get("reference_max_tokens"), + reference_max_tokens=moa_config.get("reference_max_tokens"), + # None = no per-preset override; inherit + # auxiliary.moa_reference.timeout via call_llm. + reference_timeout=( + float(moa_config["reference_timeout"]) + if moa_config.get("reference_timeout") + else None + ), + degraded_reference_policy=str( + moa_config.get("degraded_reference_policy") or "loud" + ), + agent=agent, ) if _moa_context: for _msg in reversed(api_messages): @@ -1126,18 +1429,25 @@ def run_conversation( for idx, pfm in enumerate(agent.prefill_messages): api_messages.insert(sys_offset + idx, pfm.copy()) - # Apply Anthropic prompt caching for Claude models on native - # Anthropic, OpenRouter, and third-party Anthropic-compatible - # gateways. Auto-detected: if ``_use_prompt_caching`` is set, - # inject cache_control breakpoints (system + last 3 messages) - # to reduce input token costs by ~75% on multi-turn - # conversations. - if agent._use_prompt_caching: - api_messages = apply_anthropic_cache_control( - api_messages, - cache_ttl=agent._cache_ttl, - native_anthropic=agent._use_native_cache_layout, - ) + # Per-turn context selection hook (additive, no-op by default). + # Lets a context engine select/replace which context enters the + # prompt for THIS call only — retrieval, topic routing, role/branch + # switching — distinct from compression and independent of + # should_compress(). Request-only: persisted history is untouched, so + # caching/sanitization below operate on whatever the engine selected. + # Fail-open (see _apply_context_engine_selection). + _sel_incoming = ( + messages[current_turn_user_idx] + if 0 <= current_turn_user_idx < len(messages) + else None + ) + api_messages = _apply_context_engine_selection( + agent, + api_messages, + messages, + _sel_incoming, + logger=request_logger, + ) # Safety net: strip orphaned tool results / add stubs for missing # results before sending to the API. Runs unconditionally — not @@ -1197,6 +1507,39 @@ def run_conversation( # the OpenAI SDK. Sanitizing here prevents the 3-retry cycle. _sanitize_messages_surrogates(api_messages) + # Apply Anthropic prompt caching for Claude models on native + # Anthropic, OpenRouter, and third-party Anthropic-compatible + # gateways. Auto-detected: if ``_use_prompt_caching`` is set, inject + # cache_control breakpoints for the static system prefix, full system + # prompt, and last two messages (or the legacy system-and-3 layout + # when no static prefix is available). + # + # Runs LAST, after every message mutation above. Marking earlier + # defeats the prefix stability the mutations exist to create: + # ``_apply_cache_marker`` rewrites ``content`` from a plain string + # into a ``[{"type": "text", ...}]`` block, so the marked messages + # no longer match the ``isinstance(content, str)`` test in the + # whitespace-normalization pass and silently keep their raw + # leading/trailing whitespace. A tool result ending in "\n" is + # therefore sent unstripped while it sits in the last-3 window and + # stripped once it rolls out of it — the same message, different + # bytes on consecutive turns, which breaks the prefix match at + # exactly the point the breakpoints were meant to protect. Marking + # last also keeps breakpoints off messages that the orphan sweep or + # the thinking-only drop is about to remove or merge away. + if agent._use_prompt_caching: + _static_system_prefix = getattr(agent, "_cached_system_prompt_static", None) + api_messages = apply_anthropic_cache_control( + api_messages, + cache_ttl=agent._cache_ttl, + native_anthropic=agent._use_native_cache_layout, + static_system_prefix=( + _static_system_prefix + if isinstance(_static_system_prefix, str) + else None + ), + ) + # Build a persistent-MoA request before measuring compression pressure. # MoA reference output is injected into the aggregator prompt, but it # is deliberately ephemeral and therefore absent from ``messages``. @@ -1354,36 +1697,52 @@ def run_conversation( if _pre_api_status: agent._emit_status(_pre_api_status) _last_preflight_pressure = request_pressure_tokens + _pre_api_input = messages messages, active_system_prompt = agent._compress_context( messages, system_message, approx_tokens=request_pressure_tokens, task_id=effective_task_id, ) - # Reset retry/empty-response state so the compacted request - # gets a fresh chance instead of inheriting stale recovery - # counters from the pre-compaction history. - agent._empty_content_retries = 0 - agent._thinking_prefill_retries = 0 - agent._last_content_with_tools = None - agent._last_content_tools_all_housekeeping = False - agent._mute_post_response = False - # Re-baseline the flush cursor for the compaction mode that just - # ran. Legacy session-rotation returns None (the child session has - # not seen the compacted transcript, so the next flush writes it - # whole); in-place compaction returns list(messages) because the - # compacted rows are already persisted under the same session id — - # leaving None there would re-append them, doubling the active - # context and retriggering compression. Mirrors the post-response - # and preflight compaction sites; see - # conversation_history_after_compression(). - conversation_history = conversation_history_after_compression( - agent, messages, conversation_history - ) - api_call_count -= 1 - agent._api_call_count = api_call_count - agent.iteration_budget.refund() - continue + if messages is _pre_api_input and compression_skipped_due_to_lock(agent): + # #69870 lock-skip: another path holds this session's + # compression lock, so this pass no-oped. That is a temporary + # DEFER, not evidence about compressibility — refund the + # attempt (it must not burn the shared overflow-recovery + # budget toward compression_exhausted → gateway auto-reset, + # #9893/#35809) and leave the insufficient-progress blocker + # unarmed. Proceed with the current request: if it truly does + # not fit, the provider's 413/overflow handler returns the + # soft compression_deferred result with that stronger signal. + compression_attempts -= 1 + _last_preflight_pressure = None + if pending_moa_prepared_request is _moa_prepared_request: + pending_moa_prepared_request = None + else: + # Reset retry/empty-response state so the compacted request + # gets a fresh chance instead of inheriting stale recovery + # counters from the pre-compaction history. + agent._empty_content_retries = 0 + agent._thinking_prefill_retries = 0 + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + agent._mute_post_response = False + # Re-baseline the flush cursor for the compaction mode that just + # ran. Legacy session-rotation returns None (the child session has + # not seen the compacted transcript, so the next flush writes it + # whole); in-place compaction returns list(messages) because the + # compacted rows are already persisted under the same session id — + # leaving None there would re-append them, doubling the active + # context and retriggering compression. Mirrors the post-response + # and preflight compaction sites; see + # conversation_history_after_compression(). + conversation_history = conversation_history_after_compression( + agent, messages, conversation_history + ) + api_call_count -= 1 + agent._api_call_count = api_call_count + agent.iteration_budget.refund() + continue elif ( agent.compression_enabled and len(messages) > 1 @@ -2021,6 +2380,8 @@ def run_conversation( ) continue # Retry the API call + agent._turn_received_provider_response = True + # Check finish_reason before proceeding if agent.api_mode == "codex_responses": status = getattr(response, "status", None) @@ -2542,6 +2903,14 @@ def run_conversation( "reasoning_tokens": canonical_usage.reasoning_tokens, } agent.context_compressor.update_from_response(usage_dict) + + # Stash this response's canonical usage so the post-turn + # on_turn_complete() observation hook can forward it (the + # same dict shape passed to update_from_response). A turn + # may make several API calls; the engine's per-turn signal + # of interest is the cost/size of the latest assembled + # request, so we keep the most recent call's usage. + agent._last_turn_usage = dict(usage_dict) elif getattr( agent.context_compressor, "awaiting_real_usage_after_compression", @@ -3869,10 +4238,23 @@ def run_conversation( original_len = len(messages) original_tokens = estimate_messages_tokens_rough(messages) + _overflow_input = messages messages, active_system_prompt = agent._compress_context( messages, system_message, approx_tokens=approx_tokens, task_id=effective_task_id, ) + if messages is _overflow_input and compression_skipped_due_to_lock(agent): + # #69870 lock-skip: the provider proved the request + # does not fit, but this compression pass no-oped only + # because another path holds the session's compression + # lock. Temporary defer, not exhaustion — refund the + # attempt and end the turn softly so the gateway does + # NOT auto-reset the session (#9893/#35809). + compression_attempts -= 1 + agent._persist_session(messages, conversation_history) + return _compression_deferred_result( + agent, messages, api_call_count + ) conversation_history = conversation_history_after_compression( agent, messages, conversation_history ) @@ -4110,10 +4492,23 @@ def run_conversation( original_len = len(messages) original_tokens = estimate_messages_tokens_rough(messages) + _overflow_input = messages messages, active_system_prompt = agent._compress_context( messages, system_message, approx_tokens=approx_tokens, task_id=effective_task_id, ) + if messages is _overflow_input and compression_skipped_due_to_lock(agent): + # #69870 lock-skip: the provider proved the request + # does not fit, but this compression pass no-oped only + # because another path holds the session's compression + # lock. Temporary defer, not exhaustion — refund the + # attempt and end the turn softly so the gateway does + # NOT auto-reset the session (#9893/#35809). + compression_attempts -= 1 + agent._persist_session(messages, conversation_history) + return _compression_deferred_result( + agent, messages, api_call_count + ) conversation_history = conversation_history_after_compression( agent, messages, conversation_history ) @@ -5354,6 +5749,10 @@ def run_conversation( # flag so it can fire again if the model goes empty on # a LATER tool round. agent._post_tool_empty_retried = False + # A landed tool call means any earlier dropped-tool-call stall + # was recovered — refresh that budget too so it guards each + # stall independently rather than capping the whole run. + agent._dropped_toolcall_retries = 0 previous_msg = messages[-1] if messages else None current_interim_visible = agent._interim_assistant_visible_text(assistant_msg) @@ -5516,14 +5915,27 @@ def run_conversation( if callable(_clear_warn): _clear_warn() agent._safe_print(" ⟳ compacting context…") + _post_tool_input = messages messages, active_system_prompt = agent._compress_context( messages, system_message, approx_tokens=agent.context_compressor.last_prompt_tokens, task_id=effective_task_id, ) - conversation_history = conversation_history_after_compression( - agent, messages, conversation_history - ) + if ( + messages is _post_tool_input + and compression_skipped_due_to_lock(agent) + ): + # #69870 lock-skip: this pass no-oped because another + # path holds the session's compression lock — a + # temporary defer, not evidence about compressibility. + # Refund the attempt so a lock-loser tool loop does not + # burn the shared per-turn budget toward + # compression_exhausted (#9893/#35809). + compression_attempts -= 1 + else: + conversation_history = conversation_history_after_compression( + agent, messages, conversation_history + ) elif agent.compression_enabled: # Over threshold but compression is blocked (summary-LLM # cooldown or anti-thrashing). Surface a deduped warning so @@ -5543,15 +5955,72 @@ def run_conversation( _real_tokens, int(getattr(_compressor, "threshold_tokens", 0) or 0), ) + # Proactive tool-result prune: reclaim re-sent history on + # large-window models long before should_compress() (≈50% of + # the window) would ever fire. Deterministic, no LLM call; + # protects the recent tail. No-op unless proactive_prune_tokens + # is configured and _real_tokens is above it — and even then + # the prune only commits when it reclaims at least + # proactive_prune_min_reclaim_tokens, so prompt-cache breaks + # stay episodic like compression's (the one sanctioned cache + # break) instead of firing every tool iteration. See + # ContextCompressor.prune_tool_results_only. + # getattr guard: plugin context engines predating the hook and + # minimal test doubles (SimpleNamespace compressors) lack the + # method — treat absence as a no-op. + _prune = getattr(_compressor, "prune_tool_results_only", None) + if callable(_prune): + try: + _pruned_msgs, _pruned_n = _prune( + messages, current_tokens=_real_tokens + ) + except Exception: + logger.debug( + "proactive tool-result prune failed; skipping", + exc_info=True, + ) + _pruned_msgs, _pruned_n = messages, 0 + # Standard no-op caller contract: only commit when the + # engine returned a NEW list object with a non-zero count. + if _pruned_n and _pruned_msgs is not messages: + # Do NOT rebuild conversation_history here. Unlike the + # compression branch, the prune neither rotates the session + # nor calls archive_and_compact(), so there is no new + # persistence baseline to establish. _prune_old_tool_results + # returns per-message copies that preserve the + # _DB_PERSISTED_MARKER, so the marker-based flush dedup (see + # _flush_messages_to_session_db) already prevents both + # duplicate writes and dropped rows. Calling + # conversation_history_after_compression (a compaction-only + # helper keyed on the _last_compaction_in_place flag) would be + # a no-op at best, and on a stale in-place flag could seed + # this turn's fresh, not-yet-persisted rows into history_ids + # and skip writing them. + messages = _pruned_msgs # Save session log incrementally (so progress is visible even if interrupted) agent._session_messages = messages + # Touch activity before continuing so the gateway's + # inactivity monitor never sees a stale timestamp + # between tool completion and the start of the next + # API call. Without this, a tool-call result (which + # takes ~0s to process) followed by slow post-tool + # processing (compression, persist) and a slow + # follow-up API call can exceed the gateway inactivity + # timeout (HERMES_AGENT_TIMEOUT, default 1800s) and the + # gateway kills the session before the next activity + # touch fires (#69559, #69131). + agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}") # Continue loop for next response continue else: - # No tool calls - this is the final response + # No tool calls - this is the final response. + # (Dropped tool-call recovery — finish_reason=="tool_calls" with + # an empty tool_calls array — is handled at the finalization + # chokepoint below, after final_msg is built, so it catches + # every path that reaches turn finalization, not just this one.) final_response = assistant_message.content or "" # Fix: unmute output when entering the no-tool-call branch @@ -5883,6 +6352,64 @@ def run_conversation( final_msg = agent._build_assistant_message(assistant_message, finish_reason) + # ── Dropped tool-call recovery (copilot/Claude) ──────── + # Some providers (observed: claude-opus-4.8 / claude-sonnet-4.5 + # on GitHub Copilot, ~2026-07) return finish_reason="tool_calls" + # while the parsed tool_calls array is empty — the model + # signalled it wanted to act but the payload shipped no call. + # Reaching finalization with that mismatch means the turn is + # about to end with the task unstarted (the narration, which may + # be in content or only in the reasoning field, gets treated as + # the final answer). Re-prompt (bounded to 3 CONSECUTIVE stalls; + # the budget resets after any successful tool round) to make the + # model emit the call instead of exiting. finish_reason="stop" + # text finishes never enter this guard. + if ( + finish_reason == "tool_calls" + and not assistant_message.tool_calls + and getattr(agent, "_dropped_toolcall_retries", 0) < 3 + ): + agent._dropped_toolcall_retries = getattr(agent, "_dropped_toolcall_retries", 0) + 1 + logger.warning( + "finish_reason=tool_calls with empty tool_calls array " + "(narration only) — re-prompting to emit the call " + "(retry %d/3, model=%s provider=%s)", + agent._dropped_toolcall_retries, agent.model, agent.provider, + ) + agent._emit_status( + "↻ Model signaled a tool call but sent none — " + f"re-prompting ({agent._dropped_toolcall_retries}/3)" + ) + # Both halves of the re-prompt pair are ephemeral recovery + # scaffolding (mirrors the empty-response nudge pattern): + # the interim narration-only assistant turn exists solely to + # keep role alternation valid for the nudge, and the nudge + # exists solely to drive the retry. Flag both so the + # persistence layer never writes them to the durable + # transcript and the finalization pop below can strip an + # unanswered tail pair. A recovered (answered) pair stays + # buried mid-list in live memory but is skipped by the + # flush regardless of position. + final_msg["_dropped_toolcall_nudge"] = True + messages.append(final_msg) + messages.append({ + "role": "user", + "content": ( + "Your previous turn indicated a tool call but none was " + "included. Do not narrate a plan or restate intent — issue " + "the actual tool call now to continue the task." + ), + "_dropped_toolcall_nudge": True, + }) + agent._session_messages = messages + final_response = None + continue + + # Reached finalization without the dropped-tool-call mismatch — + # a genuine turn end. Clear the consecutive-stall budget so the + # next turn starts fresh. + agent._dropped_toolcall_retries = 0 + # Pop thinking-only prefill and empty-response retry # scaffolding before appending either a final response or a # verification-stop follow-up. These internal turns are only @@ -5895,6 +6422,7 @@ def run_conversation( messages[-1].get("_thinking_prefill") or messages[-1].get("_empty_recovery_synthetic") or messages[-1].get("_empty_terminal_sentinel") + or messages[-1].get("_dropped_toolcall_nudge") ) ): messages.pop() diff --git a/agent/copilot_acp_client.py b/agent/copilot_acp_client.py index 662facc2dfe..9cbdcd34944 100644 --- a/agent/copilot_acp_client.py +++ b/agent/copilot_acp_client.py @@ -512,7 +512,7 @@ class CopilotACPClient: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + text=True, encoding='utf-8', errors='replace', bufsize=1, cwd=self._acp_cwd, env=_build_subprocess_env(), @@ -708,7 +708,7 @@ class CopilotACPClient: if block_error: raise PermissionError(block_error) try: - content = path.read_text() + content = path.read_text(encoding="utf-8") except FileNotFoundError: content = "" line = params.get("line") @@ -736,7 +736,7 @@ class CopilotACPClient: if denied: raise PermissionError(denied) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(str(params.get("content") or "")) + path.write_text(str(params.get("content") or ""), encoding="utf-8") response = { "jsonrpc": "2.0", "id": message_id, diff --git a/agent/credential_pool.py b/agent/credential_pool.py index d5d652ad741..08b0c0ea6b9 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -594,6 +594,14 @@ class CredentialPool: # Re-armed to None on every successful selection so a recover→re-exhaust # transition logs promptly instead of being swallowed by a stale window. self._last_no_entries_log_at: Optional[float] = None + # #70401: consecutive mark_exhausted_and_rotate() calls whose supplied + # credential identity matched no pool entry (OAuth wrappers whose + # runtime key rotates, entries pruned by another process, ...). These + # rotations mark nothing exhausted, so without a cap the pool can + # never converge to "no available entries" and the caller's 401 retry + # loop runs unbounded and non-interruptible. Reset whenever a real + # entry is identified or an escape path returns None. + self._unmatched_rotation_streak: int = 0 def has_credentials(self) -> bool: with self._lock: @@ -622,6 +630,28 @@ class CredentialPool: with self._lock: return self._current_unlocked() + def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]: + """Return the stable id for the runtime credential in use. + + Prefer the current selection when it still supplies ``api_key_hint``. + If the cursor was cleared, fall back to an unambiguous key match. + """ + with self._lock: + current = self._current_unlocked() + if current is not None and ( + api_key_hint is None + or current.runtime_api_key == api_key_hint + ): + return current.id + if api_key_hint is None: + return None + matches = [ + entry + for entry in self._entries + if entry.runtime_api_key == api_key_hint + ] + return matches[0].id if len(matches) == 1 else None + def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None: """Swap an entry in-place by id, preserving sort order.""" for idx, entry in enumerate(self._entries): @@ -1562,7 +1592,13 @@ class CredentialPool: def select(self) -> Optional[PooledCredential]: with self._lock: - return self._select_unlocked() + entry = self._select_unlocked() + if entry is not None: + # A normal (non-recovery) selection starts a fresh episode — + # don't let a leftover unmatched-rotation streak from an old + # failure trip the #70401 bound early next time. + self._unmatched_rotation_streak = 0 + return entry def _available_entries(self, *, clear_expired: bool = False, refresh: bool = False) -> List[PooledCredential]: """Return entries not currently in exhaustion cooldown. @@ -1763,10 +1799,17 @@ class CredentialPool: status_code: Optional[int], error_context: Optional[Dict[str, Any]] = None, api_key_hint: Optional[str] = None, + credential_id: Optional[str] = None, ) -> Optional[PooledCredential]: with self._lock: entry = None - if api_key_hint: + identity_supplied = bool(credential_id or api_key_hint) + if credential_id: + entry = next( + (e for e in self._entries if e.id == credential_id), + None, + ) + if entry is None and api_key_hint: # Prefer the specific entry whose API key matches the one that # actually failed. When this pool was freshly loaded from disk # (another process already rotated), current() is None and @@ -1775,20 +1818,59 @@ class CredentialPool: (e for e in self._entries if e.runtime_api_key == api_key_hint), None, ) - if entry is None: - # The failed key is identifiable but matches no entry - # (rotated away, or a wrapper whose runtime key differs). - # Falling through to current()/_select_unlocked() would - # mark an INNOCENT healthy key exhausted for the full - # cooldown TTL. Don't guess — just hand back a fresh - # selection so the caller can retry. - logger.info( - "credential pool: failed key hint matched no %s entry; " - "rotating without marking any credential exhausted", + if entry is None and identity_supplied: + # The failed credential is identifiable but matches no entry + # (rotated away, or a wrapper whose runtime key differs). + # Falling through to current()/_select_unlocked() would mark an + # innocent healthy key exhausted for the full cooldown TTL. + # + # #70401: this branch must still be BOUNDED. With OAuth-token + # auth the upstream 401's key hint never matches any entry's + # ``runtime_api_key``, so every retry lands here, nothing is + # ever marked exhausted, and the pool can never reach the + # "no available entries" state — the caller retries the same + # dead token forever (~6/sec, starving the event loop so chat + # interrupts are never processed). The single-entry case + # below already escapes; multi-entry pools could still + # ping-pong A→B→A indefinitely without marking anything. + # Cap consecutive no-mark rotations at one full lap of the + # available entries: past that, every candidate has been + # handed back at least once without recovery, so stop + # guessing and surface the error (no cooldown is written for + # anybody — healthy keys stay available for the next turn). + self._unmatched_rotation_streak += 1 + available_count = len(self._available_entries()) + if self._unmatched_rotation_streak > max(available_count, 1): + logger.warning( + "credential pool: failed credential identity matched no " + "%s entry for %d consecutive rotations (pool size %d) — " + "surfacing the error instead of rotating again", self.provider, + self._unmatched_rotation_streak, + available_count, ) + self._unmatched_rotation_streak = 0 self._current_id = None - return self._select_unlocked() + return None + logger.info( + "credential pool: failed credential identity matched no %s " + "entry; rotating without marking any credential exhausted", + self.provider, + ) + self._current_id = None + next_entry = self._select_unlocked() + if next_entry is not None and len(self._available_entries()) == 1: + # A single-entry pool cannot rotate. Returning its only + # entry reports a successful recovery without changing + # the credential, so the caller retries the same 401 + # indefinitely. Let fallback/error propagation proceed. + self._unmatched_rotation_streak = 0 + self._current_id = None + return None + return next_entry + # A real entry was identified — any prior unmatched-rotation + # streak is stale (this mark WILL advance pool state). + self._unmatched_rotation_streak = 0 if entry is None: entry = self._current_unlocked() or self._select_unlocked() if entry is None: @@ -1806,12 +1888,13 @@ class CredentialPool: # disconnects (a ~2.5min hang with no error surfaced to the user). # Mark every entry sharing the failed key so the pool can reach the # "no available entries" state and let the error propagate. - if api_key_hint: + failed_runtime_key = getattr(entry, "runtime_api_key", None) + if identity_supplied and failed_runtime_key: siblings_marked = False for sibling in self._entries: if sibling.id == entry.id: continue - if sibling.runtime_api_key == api_key_hint: + if sibling.runtime_api_key == failed_runtime_key: self._mark_exhausted( sibling, status_code, error_context, persist=False ) @@ -1885,9 +1968,11 @@ class CredentialPool: return self._try_refresh_current_unlocked() def try_refresh_matching( - self, api_key_hint: Optional[str] = None + self, + api_key_hint: Optional[str] = None, + credential_id: Optional[str] = None, ) -> Optional[PooledCredential]: - """Force-refresh the entry that supplied ``api_key_hint``. + """Force-refresh the entry that supplied the failed request. Direct provider integrations may reload the pool after a request has already failed, so they cannot rely on ``current_id`` identifying the @@ -1897,17 +1982,29 @@ class CredentialPool: """ with self._lock: entry = None - if api_key_hint: + if credential_id: entry = next( ( candidate for candidate in self._entries - if candidate.runtime_api_key == api_key_hint + if candidate.id == credential_id ), None, ) - else: - entry = self._current_unlocked() or self._select_unlocked(refresh=False) + if entry is None: + if api_key_hint: + entry = next( + ( + candidate + for candidate in self._entries + if candidate.runtime_api_key == api_key_hint + ), + None, + ) + else: + entry = self._current_unlocked() or self._select_unlocked( + refresh=False + ) if entry is None: return None self._current_id = entry.id diff --git a/agent/credential_sources.py b/agent/credential_sources.py index 18f0823ba84..32cd5e01a80 100644 --- a/agent/credential_sources.py +++ b/agent/credential_sources.py @@ -164,7 +164,7 @@ def _remove_env_source(provider: str, removed) -> RemovalResult: if env_path.exists(): env_in_dotenv = any( line.strip().startswith(f"{env_var}=") - for line in env_path.read_text(errors="replace").splitlines() + for line in env_path.read_text(errors="replace", encoding="utf-8").splitlines() ) except OSError: pass diff --git a/agent/curator.py b/agent/curator.py index ada93248e24..975ed102de0 100644 --- a/agent/curator.py +++ b/agent/curator.py @@ -325,7 +325,7 @@ def apply_automatic_transitions(now: Optional[datetime] = None) -> Dict[str, int counts = {"marked_stale": 0, "archived": 0, "reactivated": 0, "checked": 0, "seeded": 0} - for row in _u.agent_created_report(): + for row in _u.curated_report(): counts["checked"] += 1 name = row["name"] if row.get("pinned"): @@ -422,7 +422,9 @@ CURATOR_REVIEW_PROMPT = ( "INSTRUCTIONS AND EXPERIENTIAL KNOWLEDGE. A collection of hundreds of " "narrow skills where each one captures one session's specific bug is " "a FAILURE of the library — not a feature. An agent searching skills " - "matches on descriptions, not on exact names; one broad umbrella " + "matches on descriptions, not on exact names (note: long descriptions " + "are truncated to 57 chars in the system prompt skill index — keep the " + "trigger class in that window). One broad umbrella " "skill with labeled subsections beats five narrow siblings for " "discoverability, not the other way around.\n\n" "The right target shape is CLASS-LEVEL skills with rich SKILL.md " @@ -1470,15 +1472,16 @@ def _render_report_markdown(p: Dict[str, Any]) -> str: # --------------------------------------------------------------------------- def _render_candidate_list() -> str: - """Human/agent-readable list of agent-created skills with usage stats.""" - rows = skill_usage.agent_created_report() + """Human/agent-readable list of curator-managed skills with usage stats.""" + rows = skill_usage.curated_report() if not rows: - return "No agent-created skills to review." + return "No curator-managed skills to review." cron_referenced = _cron_referenced_skills() - lines = [f"Agent-created skills ({len(rows)}):\n"] + lines = [f"Curator-managed skills ({len(rows)}):\n"] for r in rows: lines.append( f"- {r['name']} " + f"provenance={r.get('provenance', 'agent')} " f"state={r['state']} " f"pinned={'yes' if r.get('pinned') else 'no'} " f"cron={'yes' if r['name'] in cron_referenced else 'no'} " @@ -1531,7 +1534,7 @@ def run_curator_review( if dry_run: # Count candidates without mutating state. try: - report = skill_usage.agent_created_report() + report = skill_usage.curated_report() counts = { "checked": len(report), "marked_stale": 0, @@ -1584,7 +1587,7 @@ def run_curator_review( nonlocal auto_summary # Snapshot skill state BEFORE the LLM pass so the report can diff. try: - before_report = skill_usage.agent_created_report() + before_report = skill_usage.curated_report() except Exception: before_report = [] before_names = {r.get("name") for r in before_report if isinstance(r, dict)} @@ -1610,7 +1613,7 @@ def run_curator_review( state2["last_run_duration_seconds"] = elapsed state2["last_run_summary"] = final_summary try: - after_report = skill_usage.agent_created_report() + after_report = skill_usage.curated_report() except Exception: after_report = [] try: @@ -1697,7 +1700,7 @@ def run_curator_review( try: rename_lines = _build_rename_summary( before_names=before_names, - after_report=skill_usage.agent_created_report(), + after_report=skill_usage.curated_report(), tool_calls=llm_meta.get("tool_calls", []) or [], model_final=llm_meta.get("final", "") or "", ) @@ -1715,7 +1718,7 @@ def run_curator_review( # reporting bug never breaks the curator itself. Report path is # recorded in state so `hermes curator status` can point at it. try: - after_report = skill_usage.agent_created_report() + after_report = skill_usage.curated_report() except Exception: after_report = [] try: diff --git a/agent/gemini_native_adapter.py b/agent/gemini_native_adapter.py index 1c25f1e6cf0..b4f6e6386e7 100644 --- a/agent/gemini_native_adapter.py +++ b/agent/gemini_native_adapter.py @@ -270,8 +270,12 @@ def _translate_tool_call_to_gemini(tool_call: Dict[str, Any]) -> Dict[str, Any]: } } thought_signature = _tool_call_extra_signature(tool_call) - if thought_signature: - part["thoughtSignature"] = thought_signature + # Fallback sentinel for cross-provider tool_calls (e.g. fallback from + # xAI/Anthropic to Gemini, where the original tool_call carries no + # Gemini thoughtSignature). Mirrors gemini_cloudcode_adapter.py:106. + # Without this, Gemini 3 thinking models reject replayed history with + # 400 INVALID_ARGUMENT on the missing thoughtSignature. + part["thoughtSignature"] = thought_signature or "skip_thought_signature_validator" return part diff --git a/agent/i18n.py b/agent/i18n.py index b55b8128c92..24f8ab0a023 100644 --- a/agent/i18n.py +++ b/agent/i18n.py @@ -25,7 +25,8 @@ Language resolution order: 3. ``display.language`` from config.yaml 4. ``"en"`` (baseline) -Supported languages: en, zh, ja, de, es, fr, tr, uk. Unknown values fall back to en. +Supported languages: en, zh, zh-hant, ja, de, es, fr, tr, uk, af, ko, it, ga, +pt, ru, hu, ar. Unknown values fall back to en. """ from __future__ import annotations @@ -41,7 +42,7 @@ logger = logging.getLogger(__name__) SUPPORTED_LANGUAGES: tuple[str, ...] = ( "en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk", - "af", "ko", "it", "ga", "pt", "ru", "hu", + "af", "ko", "it", "ga", "pt", "ru", "hu", "ar", ) DEFAULT_LANGUAGE = "en" @@ -78,6 +79,9 @@ _LANGUAGE_ALIASES: dict[str, str] = { "russian": "ru", "русский": "ru", "ru-ru": "ru", # Hungarian "hungarian": "hu", "magyar": "hu", "hu-hu": "hu", + # Arabic — bare "arabic"/endonym plus the common regional BCP-47 tags. + "arabic": "ar", "العربية": "ar", + "ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar", } _catalog_cache: dict[str, dict[str, str]] = {} diff --git a/agent/lsp/client.py b/agent/lsp/client.py index 9207cb75bda..3411f5300b3 100644 --- a/agent/lsp/client.py +++ b/agent/lsp/client.py @@ -56,6 +56,8 @@ from pathlib import Path from typing import Any, Awaitable, Callable, Dict, List, Optional, Set from urllib.parse import quote, unquote +from hermes_cli._subprocess_compat import windows_hide_flags + from agent.lsp.protocol import ( ERROR_CONTENT_MODIFIED, ERROR_METHOD_NOT_FOUND, @@ -294,6 +296,12 @@ class LSPClient: cmd = self._command if sys.platform == "win32": cmd = self._win_wrap_cmd(cmd) + # Suppress the cmd.exe console window that would otherwise flash + # every time we launch a ``.cmd``-wrapped language server + # (e.g. pyright-langserver.CMD) from a console-less host such as + # a VS Code/Zed extension running the ACP adapter. + # windows_hide_flags() is CREATE_NO_WINDOW on Windows, 0 on POSIX. + creationflags = windows_hide_flags() try: # start_new_session=True detaches the LSP server into its own @@ -312,6 +320,7 @@ class LSPClient: env=env, cwd=self._cwd, start_new_session=True, + creationflags=creationflags, ) except FileNotFoundError as e: raise LSPProtocolError( diff --git a/agent/lsp/install.py b/agent/lsp/install.py index 079033b772a..da8f1fd7788 100644 --- a/agent/lsp/install.py +++ b/agent/lsp/install.py @@ -35,6 +35,8 @@ import threading from pathlib import Path from typing import Any, Dict, Optional +from hermes_cli._subprocess_compat import windows_hide_flags + logger = logging.getLogger("agent.lsp.install") # Package-name → install-strategy hint registry. Each entry is a @@ -265,9 +267,10 @@ def _install_npm( [npm, "install", "--prefix", str(staging), "--silent", "--no-fund", "--no-audit", *install_targets], check=False, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=300, stdin=subprocess.DEVNULL, + creationflags=windows_hide_flags(), ) if proc.returncode != 0: logger.warning( @@ -313,10 +316,11 @@ def _install_go(pkg: str, bin_name: str) -> Optional[str]: [go, "install", pkg], check=False, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=600, env=env, stdin=subprocess.DEVNULL, + creationflags=windows_hide_flags(), ) if proc.returncode != 0: logger.warning( diff --git a/agent/memory_manager.py b/agent/memory_manager.py index 3909f2ebcd3..6f3bbadd6f0 100644 --- a/agent/memory_manager.py +++ b/agent/memory_manager.py @@ -80,8 +80,17 @@ def normalize_tool_schema(schema: Any) -> Optional[Dict[str, Any]]: return schema -def memory_provider_tools_enabled(enabled_toolsets: Optional[List[str]]) -> bool: +def memory_provider_tools_enabled( + enabled_toolsets: Optional[List[str]], + disabled_toolsets: Optional[List[str]] = None, + *, + memory_tool_present: bool = False, +) -> bool: """Return whether external memory-provider tools should be exposed.""" + if disabled_toolsets and "memory" in disabled_toolsets: + return False + if memory_tool_present: + return True if enabled_toolsets is None: return True if not enabled_toolsets: @@ -110,9 +119,10 @@ def inject_memory_provider_tools(agent: Any) -> int: for tool in tools if isinstance(tool, dict) } - if ( - "memory" not in existing_tool_names - and not memory_provider_tools_enabled(getattr(agent, "enabled_toolsets", None)) + if not memory_provider_tools_enabled( + getattr(agent, "enabled_toolsets", None), + getattr(agent, "disabled_toolsets", None), + memory_tool_present="memory" in existing_tool_names, ): return 0 diff --git a/agent/moa_loop.py b/agent/moa_loop.py index e075b001807..ed0f41a817a 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -10,7 +10,9 @@ from __future__ import annotations import hashlib import logging -from concurrent.futures import ThreadPoolExecutor +import re +import threading +from concurrent.futures import ThreadPoolExecutor, wait as _futures_wait from typing import Any from agent.auxiliary_client import call_llm @@ -19,6 +21,136 @@ from agent.transports import get_transport logger = logging.getLogger(__name__) +# --- MoA privacy filter (config: moa.privacy_filter — '' | display | full) --- +# +# Advisor (reference) outputs can echo PII from the conversation — emails, +# phone numbers, credentials pasted by the user — into surfaces the user may +# not expect: the labelled reference blocks rendered in the UI, saved MoA +# trace files, and (in `full` mode) the guidance block injected into the +# aggregator prompt (issue #59959). Secret/credential shapes (API-key +# prefixes, JWTs, private keys, DB connection strings, E.164 phone numbers) +# are handled by the repo's central redactor, ``agent.redact +# .redact_sensitive_text`` — the MoA filter never re-implements those. The +# two patterns below cover the PII classes the central redactor deliberately +# leaves alone for log/tool output (emails and formatted phone numbers). +# +# Pattern safety: advisory text is frequently code-review-shaped — line +# numbers, timestamps, git SHAs, IDs, IP addresses. A bare 10-digit match +# would mangle all of those, so the phone pattern requires clearly delimited +# formatting: a parenthesized area code and/or explicit `-`/`.` separators +# between groups ((555) 123-4567, 555-123-4567, 555.123.4567, +1 555-123-4567). +# Undelimited digit runs (5551234567), dates (2026-07-12), times (12:34:56), +# hex IDs, and dotted quads never match. International numbers in E.164 form +# (+14155551234) are already masked by the central redactor. +_MOA_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") +_MOA_PHONE_RE = re.compile( + r"(? Any: + """Redact secrets + PII from one advisor/reference text surface. + + Centralized secret shapes first (force=True: the MoA privacy filter is + its own explicit opt-in, independent of the global log-redaction toggle; + code_file=True: advisory text is prose/code, so the ENV/JSON assignment + heuristics that mangle source snippets stay off), then the MoA-specific + email/formatted-phone patterns. Non-string inputs pass through unchanged. + """ + if not isinstance(text, str) or not text: + return text + from agent.redact import redact_sensitive_text + + text = redact_sensitive_text(text, force=True, code_file=True) + text = _MOA_EMAIL_RE.sub("[redacted email]", text) + text = _MOA_PHONE_RE.sub("[redacted phone]", text) + return text + + +def _moa_privacy_mode(moa_raw: Any) -> str: + """Resolve the normalized privacy-filter mode from a raw ``moa`` config.""" + from hermes_cli.moa_config import coerce_privacy_filter + + raw = moa_raw if isinstance(moa_raw, dict) else {} + return coerce_privacy_filter(raw.get("privacy_filter")) + + +def _redact_reference_outputs( + reference_outputs: list[tuple[str, str, Any]], +) -> list[tuple[str, str, Any]]: + """Return reference-output tuples with their advisor text redacted. + + The ``_RefAccounting`` third slot is left as-is — accounting fields carry + no advisor text; the full-output/input trace fields are redacted + separately at trace-stash time (see create()) so the LIVE cache keeps raw + accounting objects untouched. + """ + return [ + (label, _redact_reference_text(text), acct) + for label, text, acct in reference_outputs + ] + + +def _redact_trace_messages(messages: Any) -> Any: + """Redact message copies destined for trace persistence. + + Handles both string content and structured content-part lists (e.g. + cache_control-decorated text parts). Unknown shapes pass through. + """ + if not isinstance(messages, list): + return messages + out: list[Any] = [] + for m in messages: + if not isinstance(m, dict): + out.append(m) + continue + content = m.get("content") + if isinstance(content, str): + out.append({**m, "content": _redact_reference_text(content)}) + elif isinstance(content, list): + out.append( + { + **m, + "content": [ + {**p, "text": _redact_reference_text(p.get("text"))} + if isinstance(p, dict) and isinstance(p.get("text"), str) + else p + for p in content + ], + } + ) + else: + out.append(m) + return out + + +def _redact_trace_accounting(acct: Any) -> Any: + """Return a copy of a ``_RefAccounting`` with its trace text redacted. + + Traces persist the advisor's FULL input messages and output to disk, so a + privacy-filtered run must not write raw PII there. Usage/cost fields are + copied verbatim (numbers, no text). Non-accounting objects pass through. + """ + if not isinstance(acct, _RefAccounting): + return acct + return _RefAccounting( + acct.usage, + acct.cost_usd, + acct.cost_status, + acct.cost_source, + messages=_redact_trace_messages(acct.messages), + output=_redact_reference_text(acct.output), + model=acct.model, + provider=acct.provider, + temperature=acct.temperature, + ) + + + # Upper bound on concurrent reference-model calls. References are independent # advisory calls (no tools, no inter-dependence), so we fan them out the same # way delegate_task runs a batch: all in flight at once, results collected when @@ -105,6 +237,16 @@ _REFERENCE_SYSTEM_PROMPT = ( "you should not try to or apologize for being unable to. A separate " "aggregator/orchestrator model holds those capabilities and will take the " "actual actions.\n\n" + "CRITICAL: You must NEVER claim or imply that you have executed a command, " + "downloaded a file, accessed a URL, or performed any action. You can only " + "analyze and advise based on the conversation context. Examples of what to " + "avoid:\n" + "- Bad: \"I ran curl and got 404.\"\n" + "- Bad: \"I downloaded the file successfully.\"\n" + "- Bad: \"I checked the repository and found...\"\n" + "- Good: \"Based on the error pattern, a curl request to that URL would likely return 404.\"\n" + "- Good: \"The conversation suggests downloading this file may help.\"\n" + "- Good: \"From the context, checking the repository would reveal...\"\n\n" "The conversation below is the current state of a task handled by that " "acting agent. Your job is to give your most intelligent analysis of that " "state: understand the goal, reason about the problem, and advise on what " @@ -115,7 +257,8 @@ _REFERENCE_SYSTEM_PROMPT = ( "asking for access.\n\n" "Respond with your advice directly — no preamble, no disclaimers about " "tools or access. Your response is private guidance handed to the " - "aggregator, not an answer shown to the user." + "aggregator, not an answer shown to the user. NEVER claim to have executed " + "anything." ) @@ -245,10 +388,10 @@ def _maybe_apply_moa_cache_control( Reuses the SAME policy function as the main agent loop (``anthropic_prompt_cache_policy``) resolved against the slot's own - provider/base_url/api_mode/model, and the SAME breakpoint layout - (``apply_anthropic_cache_control``, system_and_3). This keeps advisor and - aggregator calls decorated exactly like an acting agent on that provider - would be — no MoA-specific caching logic to drift. + provider/base_url/api_mode/model and shared marker helper + (``apply_anthropic_cache_control``). MoA has no per-session static prefix, + so it uses the helper's legacy system-and-3 fallback without carrying a + separate caching strategy. Returns the messages unchanged on any resolution error or when the policy says the route doesn't honor markers. @@ -281,13 +424,15 @@ def _maybe_apply_moa_cache_control( def _run_reference( - slot: dict[str, str], + slot: dict[str, Any], ref_messages: list[dict[str, Any]], *, temperature: float | None = None, max_tokens: int | None = None, + reference_timeout: float | None = None, + context_length_cache: Any = None, ) -> tuple[str, str, Any]: - """Call one reference model and return ``(label, text, usage)``. + """Call one reference model and return ``(label, text, accounting)``. The slot is resolved to its provider's real runtime (via ``_slot_runtime``) and called through the same ``call_llm`` request-building path any model @@ -321,10 +466,25 @@ def _run_reference( # trimmed view (_reference_messages) already strips the agent's own # system prompt, so this is the only system message the reference sees. messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] - # Apply the same Anthropic-style prompt-caching decoration the main - # agent loop applies (system_and_3 breakpoints). The advisory view is - # append-only across iterations (new turns append before the trailing - # synthetic marker), so on cache-honoring routes (Claude via + # Trim to fit THIS reference model's context window. Reference models + # may have a smaller window than the aggregator (e.g. kimi-k2.7-code + # @ 262K advising a glm-5.2 @ 1M conversation); without this trim the + # provider returns a hard HTTP 400 which the except below silently + # converts to a [failed: …] note (issue #60345). Estimated AFTER the + # advisory system prompt is prepended so its tokens count against the + # budget too. + messages = _trim_messages_for_reference( + messages, + slot, + runtime, + reserve_output_tokens=max_tokens, + context_length_cache=context_length_cache, + ) + # Apply the Anthropic-style prompt-caching decoration used by the main + # agent loop. This fixed reference prompt has no session-specific + # prefix split, so the helper uses its legacy system-and-3 fallback. + # The advisory view is append-only across iterations (new turns append + # before the trailing synthetic marker), so on cache-honoring routes (Claude via # OpenRouter/native, MiniMax, Qwen/DashScope) iteration N+1's prefix # replays iteration N's cached prefix. Without this, Claude advisors # served ZERO cache reads across an entire benchmark run (measured: @@ -333,12 +493,38 @@ def _run_reference( # (their caching is automatic; markers are ignored harmlessly, but we # only decorate when the policy says the route honors them). messages = _maybe_apply_moa_cache_control(messages, runtime) + # Per-slot max_tokens takes precedence over the preset-level + # reference_max_tokens passed in by the caller. This lets each + # reference model have its own output cap independently. + _slot_max_tokens: int | None = slot.get("max_tokens") + _effective_max_tokens = _slot_max_tokens if _slot_max_tokens is not None else max_tokens + extra_headers = None + # Normalize provider aliases (github, github-copilot, github-models, + # ...) through the auxiliary client's canonical alias table so slot + # configs that spell Copilot differently still get the header. + from agent.auxiliary_client import _normalize_aux_provider + + if _normalize_aux_provider(str(runtime.get("provider") or "")) in ( + "copilot", + "copilot-acp", + ): + # Copilot Pro/Pro+ gates some premium chat models on request + # attribution. The main agent marks the first API request of a + # user turn as ``x-initiator: user``; MoA reference fan-out is also + # directly serving the user's current turn, not a background agent + # task, so mirror that header here. Without it, Claude/Gemini + # Copilot advisors can be rejected as unavailable to the + # ``copilot-language-server`` integrator even though standalone + # Copilot calls work. + extra_headers = {"x-initiator": "user"} response = call_llm( task="moa_reference", messages=messages, temperature=temperature, - max_tokens=max_tokens, + max_tokens=_effective_max_tokens, + timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot), + extra_headers=extra_headers, **runtime, ) usage = CanonicalUsage() @@ -397,12 +583,162 @@ def _run_reference( ) +# Output-token headroom reserved inside the reference's context window when +# the preset does not cap advisor output (reference_max_tokens=None). Roughly +# one long-form advisory answer; generous enough for thinking models' visible +# output without starving the input budget. +_REFERENCE_DEFAULT_OUTPUT_RESERVE = 8192 + +# Additional estimation slack: estimate_messages_tokens_rough is a rough +# chars/4 heuristic and providers tokenize less favorably on code/JSON-heavy +# transcripts, so keep a safety fraction of the window unbudgeted. +_REFERENCE_TRIM_SAFETY_FRACTION = 0.10 + + +def _trim_messages_for_reference( + messages: list[dict[str, Any]], + slot: dict[str, str], + runtime: dict[str, Any], + *, + reserve_output_tokens: int | None = None, + context_length_cache: Any = None, +) -> list[dict[str, Any]]: + """Trim an advisory request to fit within a reference model's context window. + + Reference models may have a smaller context window than the aggregator or + the main conversation. Without this trim, a reference whose window is + exceeded gets a hard HTTP 400 from the provider, which ``_run_reference``'s + try/except silently converts to a ``[failed: …]`` note — the MoA turn + silently degrades to fewer references (issue #60345). + + ``messages`` is the FULL request as it will be sent — the advisory system + prompt already prepended — so the estimate covers everything the provider + will count. The budget reserves ``reserve_output_tokens`` (the preset's + ``reference_max_tokens`` when set, else a sane constant) for the model's + response plus a safety fraction for estimator error. + + Trimming drops the OLDEST conversation frames (right after the system + prompt) and preserves two invariants of the advisory view, which is + text-only user/assistant turns (``_reference_messages`` renders tool + calls/results inline, so there are no tool-result frames to orphan): + + - the system prompt (index 0) is always kept; + - the first non-system message stays ``user``-first — after each pop, + any now-leading assistant turns are popped too, so no provider ever + sees an assistant-first conversation; + - the trailing user turn (the synthetic judge-the-state marker) and at + least one preceding turn are always kept, even if still over budget — + a too-long-but-recent view beats an empty request. + + ``context_length_cache`` is an optional per-turn dict keyed by + ``(provider, model)`` so one fan-out (and every iteration reusing the + cache) resolves each model's window at most once instead of re-probing + metadata sources per-reference-per-iteration. When the window cannot be + resolved, messages are returned unchanged. + """ + if not messages: + return messages + + from agent.model_metadata import ( + estimate_messages_tokens_rough, + get_model_context_length, + ) + + model = str(slot.get("model") or "") + provider = str(runtime.get("provider") or slot.get("provider") or "") + if not model: + return messages + + cache_key = (provider, model) + context_length: int | None = None + if isinstance(context_length_cache, dict) and cache_key in context_length_cache: + context_length = context_length_cache[cache_key] + else: + try: + context_length = get_model_context_length( + model=model, + base_url=str(runtime.get("base_url") or ""), + api_key=str(runtime.get("api_key") or ""), + provider=provider, + ) + except Exception: + logger.debug( + "MoA reference context-length resolution failed for %s", + _slot_label(slot), + ) + context_length = None + if isinstance(context_length_cache, dict): + # Cache failures too (as None) — a flaky metadata source should + # not be re-probed for every reference of every iteration. + context_length_cache[cache_key] = context_length + + if not isinstance(context_length, int) or context_length <= 0: + return messages + + reserve = ( + int(reserve_output_tokens) + if isinstance(reserve_output_tokens, int) and reserve_output_tokens > 0 + else _REFERENCE_DEFAULT_OUTPUT_RESERVE + ) + budget = int(context_length * (1.0 - _REFERENCE_TRIM_SAFETY_FRACTION)) - reserve + if budget <= 0: + return messages + + estimated = estimate_messages_tokens_rough(messages) + if estimated <= budget: + return messages + + has_system = bool(messages) and messages[0].get("role") == "system" + head = [messages[0]] if has_system else [] + body = list(messages[1:] if has_system else messages) + + # Keep the trailing user turn plus at least one preceding turn. + while len(body) > 2 and estimate_messages_tokens_rough(head + body) > budget: + body.pop(0) + # Preserve the user-first invariant: never leave the advisory + # conversation starting on an assistant turn after a pop. + while len(body) > 2 and body[0].get("role") == "assistant": + body.pop(0) + # The loop can stop with two frames left where the first is an + # assistant turn — enforce user-first even then (a lone trailing user + # turn is a valid request; an assistant-first one is not). + while len(body) > 1 and body[0].get("role") == "assistant": + body.pop(0) + + trimmed = head + body + dropped = len(messages) - len(trimmed) + if dropped: + logger.info( + "MoA reference %s: estimated %d tokens exceeds budget %d " + "(window %d, output reserve %d); dropped %d oldest message(s).", + _slot_label(slot), + estimated, + budget, + context_length, + reserve, + dropped, + ) + return trimmed + + +_REFERENCE_POLL_INTERVAL_S = 5.0 + +# Sentinel text for a reference slot whose wait was aborted by a user +# interrupt. Shared by _run_references_parallel (which writes it) and the +# facade cache logic (which must never cache it as real advice). +_INTERRUPTED_REFERENCE_NOTE = "[skipped: interrupted by user]" + + def _run_references_parallel( - reference_models: list[dict[str, str]], + reference_models: list[dict[str, Any]], ref_messages: list[dict[str, Any]], *, temperature: float | None = None, max_tokens: int | None = None, + progress_callback: Any = None, + reference_timeout: float | None = None, + agent: Any = None, + late_accounting_sink: Any = None, ) -> list[tuple[str, str, Any]]: """Fan out all reference models in parallel, returning outputs in order. @@ -412,8 +748,30 @@ def _run_references_parallel( ``Reference {idx}`` labelling stays stable. MoA presets that reference another MoA preset are skipped here (recursion guard) with a labelled note. - Each element is ``(label, text, usage)`` where usage is a - ``CanonicalUsage`` (zeroed for skipped/failed references). + If ``progress_callback`` is provided it is invoked as each reference + completes: ``progress_callback(refs_done, refs_total, label)``. The total + matches ``len(reference_models)`` so listeners can render a status-bar + progress like ``MOA: 2/3 refs done``. Best-effort — failures are logged + but never break the fan-out (display must never block a turn). + + Each element is ``(label, text, accounting)`` where accounting is a + ``_RefAccounting`` object (zeroed for skipped/failed/interrupted + references). + + When *agent* is given, the fan-out is interruptible: waiting for the + batch is broken into ``_REFERENCE_POLL_INTERVAL_S``-second polls (instead + of one blocking ``future.result()`` per reference) so a user interrupt + mid-turn can abort the wait — mirroring the same interrupt check + ``agent.tool_executor`` already applies to its own concurrent tool + batch. This does not add or change any per-reference *timeout* (that is + ``reference_timeout`` / ``auxiliary.moa_reference.timeout``, resolved + elsewhere) — it only lets the caller stop waiting early. References + already in flight cannot be forcibly killed (``call_llm`` is a blocking + HTTP call with no interrupt hook of its own, same limitation + tool_executor has for tools without an interrupt check); an interrupted + reference's own timeout still reaps its thread independently. *agent* is + optional and defaults to ``None``, preserving the uninterruptible + blocking behavior for any caller that doesn't pass it. """ from agent.usage_pricing import CanonicalUsage @@ -421,7 +779,7 @@ def _run_references_parallel( return [] results: list[tuple[str, str, Any] | None] = [None] * len(reference_models) - futures = {} + futures: dict[Any, int] = {} workers = min(_MAX_REFERENCE_WORKERS, len(reference_models)) # Reference slots run on bare executor threads, which start with an empty # contextvars.Context — propagate the parent turn's context (approval @@ -429,7 +787,16 @@ def _run_references_parallel( # advisor calls attribute to the same conversation as the acting turn. from tools.thread_context import propagate_context_to_thread - with ThreadPoolExecutor(max_workers=workers) as executor: + total = len(reference_models) + completed = 0 + executor = ThreadPoolExecutor(max_workers=workers) + interrupted = False + # Per-fan-out context-length cache shared by every reference worker, so + # duplicate (provider, model) slots resolve their window once per turn + # instead of re-probing metadata sources per reference (dict get/set is + # GIL-atomic; a rare duplicate probe on a first-use race is harmless). + _ctx_len_cache: dict[tuple[str, str], int | None] = {} + try: for idx, slot in enumerate(reference_models): if slot.get("provider") == "moa": results[idx] = ( @@ -445,12 +812,79 @@ def _run_references_parallel( ref_messages, temperature=temperature, max_tokens=max_tokens, + reference_timeout=reference_timeout, + context_length_cache=_ctx_len_cache, ) ] = idx + # Collect every reference before returning — the aggregator needs the - # complete set, so there is no early-exit / first-completed path here. - for future, idx in futures.items(): - results[idx] = future.result() + # complete set, so there is no early-exit / first-completed path + # here, other than a user interrupt. Progress callbacks fire as each + # reference completes so frontends can render "MOA: k/n refs done". + pending = set(futures) + while pending: + done, pending = _futures_wait(pending, timeout=_REFERENCE_POLL_INTERVAL_S) + for future in done: + idx = futures[future] + results[idx] = future.result() + completed += 1 + if progress_callback is not None: + try: + label = _slot_label(reference_models[idx]) + progress_callback(completed, total, label) + except Exception as exc: # pragma: no cover - display must never break + logger.debug("MoA progress_callback failed: %s", exc) + if not pending: + break + if agent is not None and getattr(agent, "_interrupt_requested", False): + interrupted = True + break + + if interrupted: + for future, idx in futures.items(): + if results[idx] is not None: + continue + if future.cancel(): + # Never dispatched — genuinely nothing was billed. + results[idx] = ( + _slot_label(reference_models[idx]), + _INTERRUPTED_REFERENCE_NOTE, + _RefAccounting(CanonicalUsage()), + ) + elif future.done(): + # Finished between the interrupt check and now — the call + # completed and billed, so keep its REAL output and + # accounting rather than zeroing it with a placeholder. + results[idx] = future.result() + else: + # Already running — cannot be force-killed (see + # docstring); leave it be so the caller isn't blocked, + # and note that its output was abandoned. The provider + # call is still in flight and WILL bill when it + # completes, so hand its eventual accounting to the + # caller's sink instead of silently dropping it. + label = _slot_label(reference_models[idx]) + results[idx] = ( + label, + _INTERRUPTED_REFERENCE_NOTE, + _RefAccounting(CanonicalUsage()), + ) + if late_accounting_sink is not None: + def _record_late(f: Any, _label: str = label) -> None: + try: + _lbl, _txt, _acct = f.result() + except Exception: # pragma: no cover - defensive + return + try: + late_accounting_sink(_label, _acct) + except Exception: # pragma: no cover - defensive + logger.debug( + "MoA: late accounting sink failed for %s", + _label, + ) + future.add_done_callback(_record_late) + finally: + executor.shutdown(wait=not interrupted, cancel_futures=interrupted) return [r for r in results if r is not None] @@ -475,21 +909,35 @@ def _render_tool_calls(tool_calls: Any) -> str: The advisory view cannot carry real ``tool_calls`` payloads (strict providers reject tool_calls the reference never produced), so the agent's actions are flattened to text the reference can read and reason about. + + Tolerates both dict-shaped and ``SimpleNamespace``-shaped entries (with a + nested ``function`` of either kind), so the helper works uniformly against + an OpenAI-style transport and against SDK-style stream-stitched responses. + Without this shape tolerance, a SimpleNamespace-sourced entry rendered as + ``[called tool: tool]`` and silently lost the function name. """ lines: list[str] = [] for tc in tool_calls or []: - fn = (tc.get("function") or {}) if isinstance(tc, dict) else {} - name = fn.get("name") or (tc.get("name") if isinstance(tc, dict) else "") or "tool" - args = fn.get("arguments") - if isinstance(args, str): - args_text = args - elif args is not None: + if isinstance(tc, dict): + fn = tc.get("function") or {} + fn_name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None) + fn_args = fn.get("arguments") if isinstance(fn, dict) else getattr(fn, "arguments", None) + top_name = tc.get("name") + else: + fn = getattr(tc, "function", None) + fn_name = getattr(fn, "name", None) if fn is not None else None + fn_args = getattr(fn, "arguments", None) if fn is not None else None + top_name = getattr(tc, "name", None) + name = fn_name or top_name or "tool" + if isinstance(fn_args, str): + args_text = fn_args + elif fn_args is not None: try: import json - args_text = json.dumps(args, ensure_ascii=False) + args_text = json.dumps(fn_args, ensure_ascii=False) except Exception: - args_text = str(args) + args_text = str(fn_args) else: args_text = "" lines.append(f"[called tool: {name}({args_text})]" if args_text else f"[called tool: {name}]") @@ -679,45 +1127,128 @@ def _preset_temperature(preset: dict[str, Any], key: str) -> float | None: return None +def _is_failed_reference(text: str) -> bool: + """Return whether a reference output is an internal failure/skip sentinel. + + Covers both the ``[failed: …]`` notes produced when a reference call + raises (which may embed raw provider error text) and the + ``[skipped: …]`` recursion-guard notes — neither is real advice, so + neither belongs in the aggregator prompt. + """ + sentinel = text.lstrip().lower() + return sentinel.startswith("[failed:") or sentinel.startswith("[skipped:") + + +def _successful_references( + reference_outputs: list[tuple[str, str, Any]], +) -> list[tuple[str, str, Any]]: + """Filter failed advice while preserving each accounting payload.""" + return [output for output in reference_outputs if not _is_failed_reference(output[1])] + + +def _failed_reference_labels( + reference_outputs: list[tuple[str, str, Any]], +) -> list[str]: + return [label for label, text, _accounting in reference_outputs if _is_failed_reference(text)] + + +def _degraded_notice(failed_labels: list[str], policy: str) -> str: + if not failed_labels or policy.strip().lower() == "silent": + return "" + return f"[Reference models unavailable: {', '.join(failed_labels)}]" + + def aggregate_moa_context( *, user_prompt: str, api_messages: list[dict[str, Any]], - reference_models: list[dict[str, str]], - aggregator: dict[str, str], + reference_models: list[dict[str, Any]], + aggregator: dict[str, Any], temperature: float | None = None, aggregator_temperature: float | None = None, - max_tokens: int | None = None, + reference_max_tokens: int | None = None, + reference_timeout: float | None = None, + degraded_reference_policy: str = "loud", + agent: Any = None, ) -> str: """Run configured reference models and synthesize their advice. Failures are returned as model-specific notes instead of aborting the normal agent loop; the main model can still act with partial context. - ``max_tokens`` is ``None`` by default: MoA does not cap reference or - aggregator output, so each model uses its own maximum. ``call_llm`` omits - the parameter entirely when it is ``None`` (see its docstring), which also - sidesteps providers that reject ``max_tokens`` outright. A hardcoded cap - here previously truncated long aggregator syntheses. + ``reference_max_tokens`` applies ONLY to the reference fan-out — the + aggregator's own synthesis call is never capped, so it always uses its + model's own maximum. ``call_llm`` omits the parameter entirely when it + is ``None`` (see its docstring), which also sidesteps providers that + reject ``max_tokens`` outright. A hardcoded cap on the aggregator call + previously truncated long aggregator syntheses (#53580) — passing + ``reference_max_tokens`` to both calls here would silently reintroduce + that regression. ``temperature`` / ``aggregator_temperature`` are ``None`` by default: - like max_tokens, ``call_llm`` omits temperature when None so the - provider default applies — matching single-model agent behavior. Presets - may still pin explicit values. + like ``reference_max_tokens``, ``call_llm`` omits temperature when None + so the provider default applies — matching single-model agent behavior. + Presets may still pin explicit values. + + ``agent``, when passed, lets the reference fan-out be aborted early on a + user interrupt — see ``_run_references_parallel``'s docstring. """ + reference_models = [slot for slot in reference_models if slot.get("enabled", True)] reference_outputs: list[tuple[str, str, Any]] = [] ref_messages = _reference_messages(api_messages) reference_outputs = _run_references_parallel( reference_models, ref_messages, temperature=temperature, - max_tokens=max_tokens, + max_tokens=reference_max_tokens, + reference_timeout=reference_timeout, + agent=agent, ) + successful_outputs = _successful_references(reference_outputs) + failed_labels = _failed_reference_labels(reference_outputs) + + # 'full' privacy mode (moa.privacy_filter) also covers this one-shot /moa + # synthesis path: advisor text is redacted before it reaches the + # synthesizing aggregator. 'display' does not apply here — this path has + # no user-visible reference blocks or trace records of its own. Redaction + # runs on the successful outputs only (failed refs are already filtered + # into the degraded notice). + try: + from hermes_cli.config import load_config as _load_config + + if _moa_privacy_mode((_load_config() or {}).get("moa")) == "full": + successful_outputs = _redact_reference_outputs(successful_outputs) + except Exception: # pragma: no cover - privacy filter must never break a turn + logger.debug("MoA privacy filter check failed", exc_info=True) + joined = "\n\n".join( f"Reference {idx} — {label}:\n{text}" - for idx, (label, text, _usage) in enumerate(reference_outputs, start=1) + for idx, (label, text, _accounting) in enumerate(successful_outputs, start=1) ) + degraded = _degraded_notice(failed_labels, degraded_reference_policy) + if degraded: + joined = f"{joined}\n\n{degraded}" if joined else degraded + + # Skip the aggregator call when every reference failed or was skipped — + # synthesising over zero real advice wastes tokens and can block for the + # full provider timeout (observed: ~6 min on SenseNova) before returning + # a non-retryable error that leaves the session hanging. The early return + # carries only the sanitized unavailability notice (never raw provider + # error text) so the main agent loop can still act in single-model mode. + if reference_outputs and not successful_outputs: + logger.warning( + "MoA: all %d reference(s) failed — skipping aggregator synthesis", + len(reference_outputs), + ) + notice = degraded or "[Reference models unavailable]" + return ( + "[Mixture of Agents context — all reference models failed. " + "Proceeding without aggregated guidance.]\n" + f"References: {', '.join(_slot_label(slot) for slot in reference_models)}\n\n" + f"{notice}" + ) + synth_prompt = ( "You are the aggregator in a Mixture of Agents process. Synthesize the " "reference responses into concise, actionable guidance for the main " @@ -748,7 +1279,6 @@ def aggregate_moa_context( task="moa_aggregator", messages=agg_messages, temperature=aggregator_temperature, - max_tokens=max_tokens, reasoning_config=_aggregator_reasoning_config(aggregator), **agg_runtime, ) @@ -810,7 +1340,7 @@ def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str class MoAChatCompletions: """OpenAI-chat-compatible facade where the aggregator is the acting model.""" - def __init__(self, preset_name: str, reference_callback: Any = None): + def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.preset_name = preset_name or "default" # Optional display hook. Called as reference outputs become available so # frontends can show each reference model's answer as a labelled block @@ -818,9 +1348,22 @@ class MoAChatCompletions: # reference_callback(event, **kwargs) # where event is one of: # "moa.reference" kwargs: index, count, label, text + # "moa.progress" kwargs: refs_done, refs_total, label + # (fired once per reference completion — drives + # status-bar progress like ``MOA: 2/3 refs done``) + # "moa.phase" kwargs: phase, refs_done, refs_total, aggregator + # (fired on phase transitions, currently + # phase="aggregator" right before the aggregator + # acts; phase="reference" mirrors ``moa.progress`` + # so listeners can rely on a single event family) # "moa.aggregating" kwargs: aggregator (label), ref_count # Never raises into the model call — display is best-effort. self.reference_callback = reference_callback + # Back-reference to the owning AIAgent, so the reference fan-out can + # check agent._interrupt_requested (see _run_references_parallel). + # Optional — a caller that doesn't pass it just keeps the fan-out + # uninterruptible, as it was before. + self._agent = agent # State-scoped reference cache. The agent loop calls create() once per # tool-loop iteration; references should re-run whenever the task STATE # advances — i.e. on every new user message AND every new tool result — @@ -842,6 +1385,10 @@ class MoAChatCompletions: self._pending_reference_usage: Any = CanonicalUsage() self._pending_reference_cost: Any = None + # Guards pending usage/cost against concurrent late-accounting + # callbacks (see _record_late_reference_accounting), which fire on + # executor worker threads after an interrupted fan-out returns. + self._accounting_lock = threading.Lock() # Resolved aggregator slot ({provider, model, ...}) from the most recent # create(); read by session cost accounting to price the aggregator's # acting turn at its real model instead of the virtual preset name. @@ -850,6 +1397,18 @@ class MoAChatCompletions: # caller to stitch in the live session_id + resolved aggregator output # and flush to the trace file (only when moa.save_traces is on). self._pending_trace: Any = None + # every_n fan-out cadence state. The iteration counter is scoped to a + # single USER TURN (not the facade lifetime): it counts create() calls + # since the last new user message and resets whenever the user-turn + # signature changes, so cadence position never leaks across turns — + # iteration 1 of every turn is always on-cadence (fresh advice for a + # fresh request). See the fanout handling in create(). + self._fanout_iteration_count = 0 + self._fanout_turn_sig: str | None = None + self._fanout_last_state_sig: str | None = None + # Normalized moa.privacy_filter mode for the current turn ('' | + # 'display' | 'full'), refreshed from config on every create(). + self._privacy_mode: str = "" def consume_reference_usage(self) -> tuple[Any, Any]: """Pop pending reference-fan-out usage + cost, resetting both to empty. @@ -862,12 +1421,42 @@ class MoAChatCompletions: """ from agent.usage_pricing import CanonicalUsage - usage = self._pending_reference_usage or CanonicalUsage() - cost = self._pending_reference_cost - self._pending_reference_usage = CanonicalUsage() - self._pending_reference_cost = None + with self._accounting_lock: + usage = self._pending_reference_usage or CanonicalUsage() + cost = self._pending_reference_cost + self._pending_reference_usage = CanonicalUsage() + self._pending_reference_cost = None return usage, cost + def _record_late_reference_accounting(self, label: str, accounting: Any) -> None: + """Fold a late-completing interrupted reference's real spend in. + + When a user interrupt aborts the fan-out wait, references already in + flight keep running (they cannot be force-killed) and DO bill when + they complete. Their placeholder results carry zeroed accounting, so + without this hook that spend would vanish from session accounting. + The fan-out registers this as a done-callback on abandoned futures; + it folds the eventual real usage/cost into the pending totals, where + the next ``consume_reference_usage`` pick-up records it. Thread-safe: + done-callbacks fire on executor worker threads. + """ + from agent.usage_pricing import CanonicalUsage + + if not isinstance(accounting, _RefAccounting): + return + with self._accounting_lock: + if isinstance(accounting.usage, CanonicalUsage): + self._pending_reference_usage = ( + self._pending_reference_usage or CanonicalUsage() + ) + accounting.usage + if accounting.cost_usd is not None: + self._pending_reference_cost = ( + self._pending_reference_cost or 0 + ) + accounting.cost_usd + logger.debug( + "MoA: recorded late accounting for interrupted reference %s", label + ) + def consume_and_save_trace( self, session_id: Any = None, aggregator_output_fallback: Any = None ) -> None: @@ -965,9 +1554,17 @@ class MoAChatCompletions: extra_body: Any = agg_kwargs.get("extra_body") # Record the exact aggregator INPUT (incl. the injected reference # context) into the pending trace so a trace captures what the - # aggregator actually saw, not a reconstruction. + # aggregator actually saw, not a reconstruction. Traces are a + # persisted surface: when the privacy filter is active, the stored + # COPY is redacted ('display' mode's live aggregator input stays raw — + # only the on-disk record is filtered; 'full' mode's input is already + # redacted upstream, so this is a near no-op there). if self._pending_trace is not None: - self._pending_trace["aggregator_input_messages"] = agg_messages + self._pending_trace["aggregator_input_messages"] = ( + _redact_trace_messages([dict(m) for m in agg_messages]) + if getattr(self, "_privacy_mode", "") + else agg_messages + ) self._pending_trace["aggregator_label"] = _slot_label(aggregator) # The aggregator is the acting model. Resolve its slot to the provider's # real runtime (base_url/api_key/api_mode) and call it through the same @@ -1043,9 +1640,20 @@ class MoAChatCompletions: from hermes_cli.config import load_config from hermes_cli.moa_config import resolve_moa_preset - preset = resolve_moa_preset(load_config().get("moa") or {}, self.preset_name) + _moa_raw = load_config().get("moa") or {} + preset = resolve_moa_preset(_moa_raw, self.preset_name) + # Privacy filter mode: '' (off, default) | 'display' | 'full'. See + # coerce_privacy_filter / the pattern block at the top of this module. + # Remembered on self so _call_prepared_aggregator (which may run on a + # later prepared-request call without re-reading config) redacts the + # trace's aggregator input consistently with this turn's fan-out. + privacy_mode = _moa_privacy_mode(_moa_raw) + self._privacy_mode = privacy_mode messages = list(api_kwargs.get("messages") or []) - reference_models = preset.get("reference_models") or [] + reference_models = [ + slot for slot in (preset.get("reference_models") or []) + if slot.get("enabled", True) + ] aggregator = preset.get("aggregator") or {} # Expose the resolved aggregator slot so session cost accounting can # price the aggregator's acting turn at its REAL model/provider. The @@ -1070,6 +1678,16 @@ class MoAChatCompletions: # explicit values. See _preset_temperature. temperature = _preset_temperature(preset, "reference_temperature") aggregator_temperature = _preset_temperature(preset, "aggregator_temperature") + # None (the default) = no per-preset override; the fan-out inherits + # auxiliary.moa_reference.timeout (900s default) via call_llm's own + # per-task timeout resolution. Explicit per-preset values are honored. + raw_reference_timeout = preset.get("reference_timeout") + reference_timeout = ( + float(raw_reference_timeout) if raw_reference_timeout else None + ) + degraded_reference_policy = str( + preset.get("degraded_reference_policy") or "loud" + ) if aggregator_temperature is None and api_kwargs.get("temperature") is not None: # The acting agent's own configured temperature (if any) still # applies to the aggregator, which IS the acting model. @@ -1086,17 +1704,39 @@ class MoAChatCompletions: reference_outputs: list[tuple[str, str, Any]] = [] ref_messages = _reference_messages(messages) - # Fan-out cadence. "per_iteration" (default): advisors re-run whenever - # the advisory view changes — i.e. every tool iteration, since the - # view grows with each tool result. "user_turn": advisors run ONCE per - # user turn; subsequent tool iterations reuse that turn's advice and - # the aggregator acts alone (the original MoA shape: synthesize at the - # start, then let the acting model work). Implemented by hashing only - # the prefix up to the LAST USER message so mid-turn growth doesn't - # change the signature — iteration 2+ becomes a cache HIT. - fanout_mode = str(preset.get("fanout") or "per_iteration").strip().lower() + # Fan-out cadence. "user_turn" (default — cheapest cadence, #67199): + # advisors run ONCE per user turn; subsequent tool iterations reuse + # that turn's advice and the aggregator acts alone (the original MoA + # shape: synthesize at the start, then let the acting model work). + # Implemented by hashing only the prefix up to the LAST USER message + # so mid-turn growth doesn't change the signature — iteration 2+ + # becomes a cache HIT. "per_iteration": advisors re-run whenever the + # advisory view changes — i.e. every tool iteration, since the view + # grows with each tool result; advice tracks live task state at the + # cost of multiplying advisor latency/spend by tool-loop depth. + # "every_n:" (N >= 2): the middle ground (issue #63393 — advisor + # fan-out multiplies latency/cost by the tool-iteration count). + # Advisors run on iteration 1 of a user turn and then every Nth tool + # iteration; the iterations in between REUSE the cached guidance from + # the last on-cadence run (same mechanism as user_turn's cache HIT — + # the aggregator still gets advice every iteration, it's just not + # refreshed against the very latest tool results). The iteration + # counter is scoped per user turn and resets on a new user message, + # so every turn starts with fresh advice. + fanout_mode = str(preset.get("fanout") or "user_turn").strip().lower() + every_n = 0 + if fanout_mode.startswith("every_n:"): + try: + every_n = int(fanout_mode.split(":", 1)[1]) + except (TypeError, ValueError): + every_n = 0 + if every_n < 2: + # every_n:1 semantically IS per-iteration; degrade there, + # mirroring _coerce_fanout's collapse of degenerate N. + fanout_mode = "per_iteration" sig_messages = ref_messages - if fanout_mode == "user_turn": + turn_prefix = ref_messages + if fanout_mode in ("user_turn",) or every_n >= 2: # Find the last REAL user message. The advisory view appends a # synthetic user marker (_ADVISORY_INSTRUCTION) when it ends on an # assistant turn — i.e. on every tool iteration after the first — @@ -1111,19 +1751,51 @@ class MoAChatCompletions: last_user_idx = _i break if last_user_idx is not None: - sig_messages = ref_messages[: last_user_idx + 1] + turn_prefix = ref_messages[: last_user_idx + 1] + if fanout_mode == "user_turn": + sig_messages = turn_prefix + + def _hash_messages(msgs: list[dict[str, Any]]) -> str: + return hashlib.sha256( + "\u0000".join( + f"{m.get('role')}:{m.get('content')}" for m in msgs + ).encode("utf-8", "replace") + ).hexdigest() + + # every_n cadence bookkeeping: advance the per-turn iteration counter + # only when the advisory STATE actually advanced (a redundant create() + # with identical state — e.g. a streaming retry — must not consume a + # cadence slot), and reset it whenever the user-turn prefix changes. + _every_n_reuse = False + if every_n >= 2: + _turn_sig = _hash_messages(turn_prefix) + if _turn_sig != self._fanout_turn_sig: + self._fanout_turn_sig = _turn_sig + self._fanout_iteration_count = 0 + self._fanout_last_state_sig = None + _state_sig = _hash_messages(ref_messages) + if _state_sig != self._fanout_last_state_sig: + self._fanout_last_state_sig = _state_sig + self._fanout_iteration_count += 1 + # Iteration 1 is on-cadence; then every Nth iteration after it. + _on_cadence = (self._fanout_iteration_count - 1) % every_n == 0 + _every_n_reuse = not _on_cadence and bool(self._ref_cache_outputs) # Turn-scoped cache: only run + display references when the advisory # view changed (i.e. a new user turn). Within one turn the agent loop # calls create() once per tool iteration; in user_turn mode the # signature is stable across those iterations (prefix hash above), so # the fan-out runs once per user turn and iterations reuse the advice. - _sig = hashlib.sha256( - "\u0000".join( - f"{m.get('role')}:{m.get('content')}" for m in sig_messages - ).encode("utf-8", "replace") - ).hexdigest() + _sig = _hash_messages(sig_messages) _cache_key = (self.preset_name, _sig, tuple(_slot_label(s) for s in reference_models)) + if _every_n_reuse: + # Off-cadence every_n iteration: pin the key to the last + # on-cadence run so the lookup below is a HIT and its guidance is + # reused (no advisor calls, no double accounting, no re-emit) — + # exactly the user_turn cache-HIT path. When the cache is empty + # (defensive; a new turn resets the counter to on-cadence) the + # flag above stays False and the references run normally. + _cache_key = self._ref_cache_key _refs_from_cache = _cache_key == self._ref_cache_key and bool(self._ref_cache_outputs) if _refs_from_cache: @@ -1131,22 +1803,54 @@ class MoAChatCompletions: # References already ran (and were accounted) earlier this turn; # this create() is a repeat tool-iteration reusing the cached # advice. Charging their tokens/cost again here would multiply - # advisor spend by the tool-iteration count, so pending is zero. - self._pending_reference_usage = CanonicalUsage() - self._pending_reference_cost = None + # advisor spend by the tool-iteration count, so nothing new is + # deposited — but do NOT zero the pending totals: a + # late-completing interrupted reference may have deposited its + # real spend since the last consume(), and that must survive + # until the next consume_reference_usage() pick-up. # Likewise no trace on a cache HIT — the full turn was already # traced on the MISS that ran the references. A repeat iteration is # not a new MoA turn. self._pending_trace = None else: + # Per-reference progress callback: emits ``moa.progress`` so + # listeners can render ``MOA: N/M refs done`` in the status bar as + # each reference completes. The callback is bound to self so it + # goes through the same display hook as the existing + # ``moa.reference`` / ``moa.aggregating`` events. + def _progress(done: int, total: int, label: str) -> None: + self._emit( + "moa.progress", + refs_done=done, + refs_total=total, + label=label, + ) + reference_outputs = _run_references_parallel( reference_models, ref_messages, temperature=temperature, max_tokens=reference_max_tokens, + progress_callback=_progress, + reference_timeout=reference_timeout, + agent=self._agent, + late_accounting_sink=self._record_late_reference_accounting, ) - self._ref_cache_key = _cache_key - self._ref_cache_outputs = list(reference_outputs) + interrupted_any = any( + text == _INTERRUPTED_REFERENCE_NOTE + for _lbl, text, _acct in reference_outputs + ) + if interrupted_any: + # An interrupted fan-out is a partial snapshot, not real + # advice for this state. Caching it would replay the + # placeholder notes on every subsequent iteration of the + # turn (a cache HIT never re-runs the references), so leave + # the cache empty and let the next create() re-run them. + self._ref_cache_key = None + self._ref_cache_outputs = [] + else: + self._ref_cache_key = _cache_key + self._ref_cache_outputs = list(reference_outputs) # Sum the advisor fan-out's token usage AND cost so the caller can # fold advisor spend into session accounting exactly once per turn. # Only the freshly run references (cache MISS) contribute; a cache @@ -1163,16 +1867,35 @@ class MoAChatCompletions: _ref_usage = _ref_usage + _acct.usage if _acct.cost_usd is not None: _ref_cost = (_ref_cost or 0) + _acct.cost_usd - self._pending_reference_usage = _ref_usage - self._pending_reference_cost = _ref_cost + with self._accounting_lock: + # Fold (don't overwrite): a late-completing interrupted + # reference from a PREVIOUS turn may have deposited its real + # spend here between consume() calls — keep it. + self._pending_reference_usage = ( + self._pending_reference_usage or CanonicalUsage() + ) + _ref_usage + if _ref_cost is not None: + self._pending_reference_cost = ( + self._pending_reference_cost or 0 + ) + _ref_cost # Stash the full reference fan-out for trace persistence. The # aggregator input/label are filled in below once agg_messages is # built; the aggregator OUTPUT is stitched in by the caller # (consume_and_save_trace) once the response resolves — the caller # holds the live session_id and the resolved aggregator response. + # Traces are a persisted, user-readable surface, so ANY active + # privacy mode ('display' or 'full') redacts the advisor text and + # the full per-advisor input/output carried by _RefAccounting. + if privacy_mode: + _trace_refs = [ + (label, _redact_reference_text(text), _redact_trace_accounting(acct)) + for label, text, acct in reference_outputs + ] + else: + _trace_refs = list(reference_outputs) self._pending_trace = { "preset": self.preset_name, - "reference_outputs": list(reference_outputs), + "reference_outputs": _trace_refs, "aggregator_slot": aggregator, "aggregator_temperature": aggregator_temperature, } @@ -1182,17 +1905,31 @@ class MoAChatCompletions: # actually ran them). The user sees one labelled block per # reference (rendered like a thinking block) so the MoA process is # visible rather than a silent pause. Best-effort: never blocks the - # turn. + # turn. Reference blocks are a user-visible surface: both privacy + # modes redact them (the cache keeps the RAW text — redaction + # always happens at the consuming surface, so a mid-session mode + # change never leaks or double-redacts). _ref_count = len(reference_outputs) - for _idx, (_label, _text, _usage) in enumerate(reference_outputs, start=1): + for _idx, (_label, _text, _accounting) in enumerate(reference_outputs, start=1): self._emit( "moa.reference", index=_idx, count=_ref_count, label=_label, - text=_text, + text=_redact_reference_text(_text) if privacy_mode else _text, ) if _ref_count: + # Phase transition: reference fan-out is complete, the + # aggregator is about to act. Listeners that prefer a single + # event family for phase tracking can switch on ``phase`` + # instead of subscribing to ``moa.aggregating`` separately. + self._emit( + "moa.phase", + phase="aggregator", + refs_done=_ref_count, + refs_total=_ref_count, + aggregator=_slot_label(aggregator), + ) self._emit( "moa.aggregating", aggregator=_slot_label(aggregator), @@ -1201,16 +1938,58 @@ class MoAChatCompletions: guidance: str | None = None agg_messages = [dict(m) for m in messages] - if reference_outputs: + successful_outputs = _successful_references(reference_outputs) + failed_labels = _failed_reference_labels(reference_outputs) + joined = "" + _agg_refs: list = [] + if successful_outputs: + # 'full' privacy mode: redact the advisor text that reaches the + # AGGREGATOR too (issue #59959's literal ask). 'display' leaves + # the aggregator input raw so synthesis quality is unaffected. + # The redaction is applied to a per-call copy — the cache always + # holds raw advisor text (see the emit comment above). Failed + # refs are already filtered out; only successful advisor text is + # joined (and redacted when requested). + _agg_refs = ( + _redact_reference_outputs(successful_outputs) + if privacy_mode == "full" + else successful_outputs + ) joined = "\n\n".join( f"Reference {idx} — {label}:\n{text}" - for idx, (label, text, _usage) in enumerate(reference_outputs, start=1) + for idx, (label, text, _usage) in enumerate(_agg_refs, start=1) ) + degraded = _degraded_notice(failed_labels, degraded_reference_policy) + if reference_outputs and not successful_outputs: + # Every reference failed or was skipped: don't wrap a wall of + # failure sentinels in "use the reference responses below" + # guidance — the aggregator IS the acting model, so it simply + # acts alone this turn. Under the loud policy it still gets the + # sanitized unavailability notice so it can disclose degraded + # mode; under silent it gets nothing. + logger.warning( + "MoA: all %d reference(s) failed — acting aggregator-alone " + "without reference guidance", + len(reference_outputs), + ) + if degraded: + guidance = ( + "[Mixture of Agents reference context]\n" + f"Preset: {self.preset_name}\n" + f"Aggregator/acting model: {_slot_label(aggregator)}\n\n" + "All reference models failed this turn — no advisory " + "guidance is available. Act on your own judgment.\n\n" + f"{degraded}" + ) + _attach_reference_guidance(agg_messages, guidance) + elif joined or degraded: + if degraded: + joined = f"{joined}\n\n{degraded}" if joined else degraded guidance = ( "[Mixture of Agents reference context]\n" f"Preset: {self.preset_name}\n" f"Aggregator/acting model: {_slot_label(aggregator)}\n" - f"References: {', '.join(label for label, _, _ in reference_outputs)}\n\n" + f"References: {', '.join(label for label, _, _ in _agg_refs)}\n\n" "Use the reference responses below as private context. You are the aggregator and acting model: " "answer the user directly or call tools as needed.\n\n" f"{joined}" @@ -1229,9 +2008,11 @@ class MoAChatCompletions: class MoAClient: - def __init__(self, preset_name: str, reference_callback: Any = None): + def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.chat = type("_MoAChat", (), {})() - self.chat.completions = MoAChatCompletions(preset_name, reference_callback=reference_callback) + self.chat.completions = MoAChatCompletions( + preset_name, reference_callback=reference_callback, agent=agent, + ) def consume_reference_usage(self) -> Any: """Pop the pending reference-fan-out usage from the completions facade. @@ -1261,3 +2042,85 @@ class MoAClient: return self.chat.completions.consume_and_save_trace( session_id, aggregator_output_fallback=aggregator_output_fallback ) + + +def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: + """Build the MoA facade client for ``agent``, wiring the reference relay. + + Single construction point for ``MoAClient`` wherever the agent's shared + client is (re)built: initial setup (``agent_init``), turn-start fallback + restore (``restore_primary_runtime``), transient transport recovery + (``try_recover_primary_transport``), and mid-session model switches + (``switch_model``). + + Constructing a bare ``MoAClient(preset)`` at any of those sites silently + drops the ``reference_callback`` relay that ``agent_init`` wires to + ``agent.tool_progress_callback`` — after a fallback+restore cycle the + facade would still work, but every frontend (CLI spinner, TUI, desktop, + gateway) would stop receiving ``moa.reference`` / ``moa.aggregating`` + display events for the rest of the session (#53802). + + The relay reads ``agent.tool_progress_callback`` at *emit* time, so a + callback attached after client construction is picked up automatically. + Best-effort and display-only — it never raises into the model call. + """ + def _moa_reference_relay(event: str, **kwargs: Any) -> None: + cb = getattr(agent, "tool_progress_callback", None) + if cb is None: + return + try: + if event == "moa.reference": + label = str(kwargs.get("label") or "") + text = str(kwargs.get("text") or "") + idx = kwargs.get("index") + count = kwargs.get("count") + cb( + "moa.reference", + label, + text, + None, + moa_index=idx, + moa_count=count, + ) + elif event == "moa.progress": + # Per-reference completion. Frontends render this as a + # status-bar progress indicator like ``MOA: N/M refs done``. + cb( + "moa.progress", + str(kwargs.get("label") or ""), + None, + None, + moa_refs_done=kwargs.get("refs_done"), + moa_refs_total=kwargs.get("refs_total"), + ) + elif event == "moa.phase": + # Phase transition (currently only ``phase="aggregator"`` + # fires once the fan-out is done). Subscribers can switch + # on ``moa_phase`` to know which phase is active. + cb( + "moa.phase", + str(kwargs.get("aggregator") or ""), + None, + None, + moa_phase=kwargs.get("phase"), + moa_refs_done=kwargs.get("refs_done"), + moa_refs_total=kwargs.get("refs_total"), + ) + elif event == "moa.aggregating": + cb( + "moa.aggregating", + str(kwargs.get("aggregator") or ""), + None, + None, + moa_ref_count=kwargs.get("ref_count"), + ) + except Exception: + pass + + return MoAClient( + str(preset_name or getattr(agent, "model", None) or "default"), + reference_callback=_moa_reference_relay, + # Thread the agent through so the reference fan-out wait can be + # aborted on a user interrupt (see _run_references_parallel). + agent=agent, + ) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 288083628e0..296fe0aedca 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -215,6 +215,7 @@ DEFAULT_CONTEXT_LENGTHS = { # OpenRouter-prefixed models resolve via OpenRouter live API or models.dev. "claude-fable-5": 1000000, "claude-fable": 1000000, + "claude-opus-5": 1000000, "claude-sonnet-5": 1000000, "claude-opus-4-8": 1000000, "claude-opus-4.8": 1000000, diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index 415d367ca17..0234eef2ea2 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -117,7 +117,7 @@ def record_nous_rate_limit( # Atomic write: write to temp file + rename fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp") try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(state, f) atomic_replace(tmp_path, path) except Exception: diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index fc94ca2a6b2..845e4260ddb 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -192,7 +192,13 @@ SKILLS_GUIDANCE = ( "skill with skill_manage so you can reuse it next time.\n" "When using a skill and finding it outdated, incomplete, or wrong, " "patch it immediately with skill_manage(action='patch') — don't wait to be asked. " - "Skills that aren't maintained become liabilities." + "Skills that aren't maintained become liabilities.\n" + "\n" + "## Skill Safety Rule\n" + "1. **UNAVAILABLE** — If a skill placeholder contains `[SKILL_PRUNED]`, the skill content was lost in compression and is inaccessible.\n" + "2. **RELOAD** — Before performing any action that depends on a skill, re-check its content with `skill_view(name='...')` if it shows `[SKILL_PRUNED]`.\n" + "3. **WAIT** — If a skill is loading or was just pruned, wait for the reload confirmation before proceeding.\n" + "4. **DEDUP** — After reloading a pruned skill, **ignore any remaining `[SKILL_PRUNED]` markers for that same skill** — they are historical artifacts from previous compactions and do not need further action." ) KANBAN_GUIDANCE = ( diff --git a/agent/prompt_caching.py b/agent/prompt_caching.py index 9a2fdf4ccce..09b51e61d01 100644 --- a/agent/prompt_caching.py +++ b/agent/prompt_caching.py @@ -1,9 +1,11 @@ """Anthropic prompt caching strategy. -Single layout: ``system_and_3``. 4 cache_control breakpoints — system -prompt + last 3 non-system messages, all at the same TTL (5m or 1h). -Reduces input token costs by ~75% on multi-turn conversations within a -single session. +The default layout uses 4 cache_control breakpoints: the static system +prefix, the end of the system prompt, and the last 2 non-system messages. +When a static system prefix is unavailable, it falls back to one system +breakpoint plus the last 3 messages. All markers use the same TTL (5m or 1h). +This preserves intra-session caching while allowing new sessions to reuse the +stable system-prompt prefix. Pure functions -- no class state, no AIAgent dependency. """ @@ -81,15 +83,55 @@ def _build_marker(ttl: str) -> Dict[str, str]: return marker +def _apply_system_cache_markers( + message: dict, + cache_marker: dict, + static_system_prefix: str | None, + *, + native_anthropic: bool, +) -> int: + """Mark the static system prefix and full prompt when they can be split. + + The system prompt remains one stored string. Splitting it only in the + outgoing request keeps session persistence and non-Anthropic transports + unchanged while making the stable prefix independently cacheable. + """ + content = message.get("content") + if ( + isinstance(static_system_prefix, str) + and static_system_prefix + and isinstance(content, str) + and content.startswith(static_system_prefix) + ): + suffix = content[len(static_system_prefix):] + if suffix: + message["content"] = [ + { + "type": "text", + "text": static_system_prefix, + "cache_control": cache_marker, + }, + {"type": "text", "text": suffix, "cache_control": cache_marker}, + ] + return 2 + + _apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic) + return 1 + + def apply_anthropic_cache_control( api_messages: List[Dict[str, Any]], cache_ttl: str = "5m", native_anthropic: bool = False, + static_system_prefix: str | None = None, ) -> List[Dict[str, Any]]: - """Apply system_and_3 caching strategy to messages for Anthropic models. + """Apply Anthropic cache-control markers to API messages. - Places up to 4 cache_control breakpoints: system prompt + last 3 non-system - messages, all at the same TTL. + When ``static_system_prefix`` exactly matches the beginning of a string + system prompt, it receives an early marker and the full system prompt gets + a trailing marker. The remaining two markers target the latest cacheable + non-system messages. Without that prefix, the legacy system-and-3 layout + is retained. Returns: Deep copy of messages with cache_control breakpoints injected. @@ -103,8 +145,12 @@ def apply_anthropic_cache_control( breakpoints_used = 0 if messages[0].get("role") == "system": - _apply_cache_marker(messages[0], marker, native_anthropic=native_anthropic) - breakpoints_used += 1 + breakpoints_used = _apply_system_cache_markers( + messages[0], + marker, + static_system_prefix, + native_anthropic=native_anthropic, + ) remaining = 4 - breakpoints_used non_sys = [ diff --git a/agent/proxy_sources/__init__.py b/agent/proxy_sources/__init__.py new file mode 100644 index 00000000000..34af31ca6fb --- /dev/null +++ b/agent/proxy_sources/__init__.py @@ -0,0 +1,8 @@ +"""Egress proxy integrations. + +Currently ships an iron-proxy (ironsh/iron-proxy) wrapper that intercepts +outbound traffic from remote terminal sandboxes and swaps proxy tokens +for real upstream credentials at the network edge. + +Design notes live in :mod:`agent.proxy_sources.iron_proxy`. +""" diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py new file mode 100644 index 00000000000..277cd018654 --- /dev/null +++ b/agent/proxy_sources/iron_proxy.py @@ -0,0 +1,2494 @@ +"""iron-proxy (`ironsh/iron-proxy`) integration for credential-injecting egress control. + +Why +--- + +Remote terminal sandboxes (Docker, Modal, SSH) currently see real upstream +API credentials. A prompt-injected agent inside one of these sandboxes can +``cat ~/.config/openrouter/auth.json`` or ``printenv | grep -i key`` and +exfiltrate them. + +iron-proxy is a TLS-intercepting egress firewall (Apache-2.0, Go binary, by +ironsh). It sits between the sandbox and the internet, enforces a default-deny +allowlist on outbound hosts, and *swaps proxy tokens for real credentials* +on the way out. The sandbox only ever holds opaque proxy tokens — leaking +them is useless, since they only work behind the configured trusted proxy +boundary (the CA private key and proxy endpoint integrity are part of that +boundary: if traffic can be redirected to attacker-controlled proxy +infrastructure, the guarantee no longer holds). + +Design summary +-------------- + +* The ``iron-proxy`` binary is auto-installed into ``/bin/iron-proxy`` + on first use. Hermes pins one upstream version (``_IRON_PROXY_VERSION``) + and downloads the matching tar.gz from the official GitHub Releases page, + verifying the SHA-256 against the release's ``checksums.txt``. + +* A long-lived CA at ``/proxy/ca.{crt,key}`` is generated on + first ``hermes egress setup``. Sandboxes trust this CA so iron-proxy can + terminate TLS and rewrite headers. + +* The proxy config lives at ``/proxy/proxy.yaml``. It enumerates + the per-provider allowlists and the ``secrets`` transform that does the + Authorization-header swap. + +* Token mappings (proxy token -> real credential lookup) live alongside the + config. The real credential is **never** written to the config — iron-proxy + reads it from its own environment via ``{type: env, var: NAME}``. When + Bitwarden Secrets Manager is configured, the real value is pulled there + at proxy startup instead. + +* The proxy runs as a managed subprocess (``hermes egress start``), pidfile + at ``/proxy/iron-proxy.pid``. Daemon output (including + per-request records on v0.39) goes to ``/proxy/iron-proxy.log``; + ``audit.log`` is pre-created but reserved for a future pin that supports + ``log.audit_path``. + +* Failures (binary missing, port collision, bad config) emit a one-line + warning and do *not* block agent startup. The Docker backend refuses to + start a sandbox with the proxy enabled-but-down, with a clear error. + +This module is intentionally subprocess-driven rather than depending on any +iron-proxy Python bindings — a single cross-platform binary is easier to +lazy-install than a wheels-with-extension dependency, and we keep maintenance +to a "bump the pinned version" loop. +""" + +from __future__ import annotations + +import hashlib +import ipaddress +import json +import logging +import os +import platform +import shutil +import signal +import stat +import subprocess +import tarfile +import tempfile +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Configuration constants +# --------------------------------------------------------------------------- + +# Pinned upstream version. Bump in a follow-up PR — never auto-resolve "latest" +# because upstream YAML schema is allowed to change between releases and we +# want updates to be deliberate. +_IRON_PROXY_VERSION = "0.39.0" + +_IRON_PROXY_RELEASE_BASE = ( + f"https://github.com/ironsh/iron-proxy/releases/download/v{_IRON_PROXY_VERSION}" +) +_IRON_PROXY_CHECKSUM_NAME = "checksums.txt" +# Detached signature for checksums.txt + the signing public key, both shipped on +# the release. Used for optional GPG verification of the release channel +# (maxpetrusenko P1): SHA-256 only protects the archive if checksums.txt itself +# came from an uncompromised channel; verifying its signature closes that gap. +_IRON_PROXY_CHECKSUM_SIG_NAME = "checksums.txt.asc" +_IRON_PROXY_PUBKEY_NAME = "public-key.asc" + +# How long to wait for HTTP downloads and subprocess interactions, in seconds. +_DOWNLOAD_TIMEOUT = 120 # binary is ~16MB +_RUN_TIMEOUT = 30 +_STARTUP_GRACE_SECONDS = 5 + +# Management (operator) API. iron-proxy v0.39 ships an authenticated +# loopback HTTP endpoint (``management.listen`` + ``management.api_key_env``) +# whose ``POST /v1/reload`` re-reads proxy.yaml and atomically swaps the +# transform pipeline in-place — no restart, no dropped connections. We +# always enable it on generated configs: it binds loopback only and every +# request needs the bearer key below. ``hermes egress reload`` is the +# client. +# +# The key is minted at setup time, stored at +# ``/proxy/management.token`` (0600), and injected into the +# daemon's env under this name at start. v0.39 validates at startup that +# the named env var is non-empty when management.listen is set. +_MGMT_API_KEY_ENV = "HERMES_IRON_PROXY_MGMT_KEY" +# The management listener binds loopback at tunnel_port + 2 (tunnel_port +# is CONNECT/MITM, +1 is the plain-HTTP forward listener). +_MGMT_PORT_OFFSET = 2 +_MGMT_RELOAD_TIMEOUT = 15 + +# Default listen ports. HTTPS_PROXY semantics use a single CONNECT tunnel, +# so we expose only the tunnel listener for v1 — no need to put the sandbox +# DNS at the iron-proxy IP. This greatly simplifies wiring. +_DEFAULT_TUNNEL_PORT = 9090 + +# Hosts allowed by default for AI inference traffic. Anything else is 403'd. +_DEFAULT_ALLOWED_HOSTS: Tuple[str, ...] = ( + "openrouter.ai", + "*.openrouter.ai", + "api.openai.com", + "api.anthropic.com", + "generativelanguage.googleapis.com", + "api.x.ai", + "api.mistral.ai", + "api.groq.com", + "api.together.xyz", + "api.deepseek.com", + "inference.nousresearch.com", +) + +# Provider env-var name -> upstream host (or list of hosts) on which the +# Authorization Bearer token should be swapped. +_BEARER_PROVIDERS: Dict[str, Tuple[str, ...]] = { + "OPENROUTER_API_KEY": ("openrouter.ai", "*.openrouter.ai"), + "OPENAI_API_KEY": ("api.openai.com",), + "GROQ_API_KEY": ("api.groq.com",), + "TOGETHER_API_KEY": ("api.together.xyz",), + "DEEPSEEK_API_KEY": ("api.deepseek.com",), + "MISTRAL_API_KEY": ("api.mistral.ai",), + "XAI_API_KEY": ("api.x.ai",), + "NOUS_API_KEY": ("inference.nousresearch.com",), +} + + +# Providers whose API authenticates with a NON-Authorization header. +# iron-proxy v0.39's ``secrets.replace.match_headers`` targets arbitrary +# header names (case-insensitive; confirmed by the iron-proxy author on +# PR #30179 and verified in the pinned v0.39.0 source — ``swapHeaders`` +# + ``parseHeaderMatchers``), so these are first-class swapped providers, +# not "uncovered". +# +# ``aliases`` are interchangeable env-var names for the SAME upstream +# credential (Hermes' auth.py keys Google on both GEMINI_API_KEY and +# GOOGLE_API_KEY). Aliased names MUST collapse into a single mapping: +# every rule carries ``require: true``, and two require-rules on the same +# host reject each other's requests (each rule whose own token isn't +# present returns ActionReject). The sandbox receives the minted token +# under the canonical name AND every alias so SDKs reading either work. +_HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { + # Anthropic native: x-api-key. Authorization is also matched so an + # SDK sending the token as a Bearer (OAuth-style) still swaps. + "ANTHROPIC_API_KEY": { + "hosts": ("api.anthropic.com",), + "match_headers": ("x-api-key", "Authorization"), + "aliases": (), + }, + # Azure OpenAI: api-key header (AAD bearer flows use Authorization). + "AZURE_OPENAI_API_KEY": { + "hosts": ( + "*.openai.azure.com", + "*.cognitiveservices.azure.com", + "*.services.ai.azure.com", + ), + "match_headers": ("api-key", "Authorization"), + "aliases": (), + }, + # Google AI Studio (Gemini): x-goog-api-key header; the SDKs that pass + # ``?key=`` as a query param are covered by match_query, which + # scans every query parameter for the token value. + "GEMINI_API_KEY": { + "hosts": ("generativelanguage.googleapis.com",), + "match_headers": ("x-goog-api-key",), + "aliases": ("GOOGLE_API_KEY",), + }, +} + + +# Providers whose env-var names we recognize but whose auth genuinely cannot +# be swapped by a static header/query replacement (SigV4 request signing, +# OAuth tokens minted by an SDK from a service-account file). Presence is +# surfaced as a warning at setup/status time — these are generic cloud creds +# that are usually present for unrelated tooling (terraform, gcloud, aws-cli), +# so they never block the proxy from starting. +# +# NOTE: this list used to include Anthropic / Azure OpenAI / Gemini, with an +# LLM-specific fail-closed tier (``proxy.fail_on_uncovered_providers``). +# Those providers moved to ``_HEADER_AUTH_PROVIDERS`` once we wired +# ``match_headers`` (upstream confirmed support on the pinned v0.39.0), which +# emptied the fail-closed tier — the flag and its refuse-start path were +# deleted rather than kept as a dead toggle. +_NON_BEARER_PROVIDERS: Tuple[str, ...] = ( + # AWS Bedrock / SageMaker: SigV4-signed requests. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + # GCP Vertex AI: OAuth bearer minted by the SDK from a service-account + # file, not a static env key. + "GOOGLE_APPLICATION_CREDENTIALS", +) + + +# Default SSRF-protection deny list applied to the proxy's outbound traffic. +# Mirrors the public docs promise ("cloud metadata IPs are refused by default +# regardless of allowlist"). Tests / dev setups that need loopback can pass +# an explicit override (e.g. [] to disable, or a smaller subset). +_DEFAULT_UPSTREAM_DENY_CIDRS: Tuple[str, ...] = ( + "127.0.0.0/8", # IPv4 loopback + "::1/128", # IPv6 loopback + "169.254.0.0/16", # IPv4 link-local incl. AWS/GCP/Azure IMDS + "fe80::/10", # IPv6 link-local + "10.0.0.0/8", # RFC1918 + "172.16.0.0/12", # RFC1918 + "192.168.0.0/16", # RFC1918 + "fc00::/7", # IPv6 ULA + # IPv4-mapped IPv6 (``::ffff:0:0/96``) covers the dual-stack case + # where an upstream resolves to e.g. ``::ffff:169.254.169.254`` and + # the kernel hands the v4-mapped form to the socket — that would + # otherwise be a clean SSRF bypass to IMDS through the v6 path. + "::ffff:0:0/96", + # RFC6598 / CGNAT — used by AWS VPC for shared services, K8s pod + # networks, many cloud overlays. Not strictly RFC1918 but operators + # universally want it denied for the same reasons. + "100.64.0.0/10", + # RFC2544 benchmark range — rare in practice but occasionally used + # for internal services and never legitimate as an upstream. + "198.18.0.0/15", +) + + +# Min env vars the iron-proxy subprocess actually needs. Everything else +# is stripped — see ``_build_proxy_subprocess_env`` for the rationale. +_PROXY_SUBPROCESS_ENV_ALLOWLIST: Tuple[str, ...] = ( + "PATH", + "HOME", + "TMPDIR", + "TZ", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NO_COLOR", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SYSTEMROOT", # Windows + "USERPROFILE", # Windows +) + + +# Env vars that must be stripped from the subprocess env even if they're on +# the allowlist or named in mappings — these would either recurse the proxy +# back through itself or send its traffic through a corporate proxy. +_PROXY_SUBPROCESS_ENV_STRIP: Tuple[str, ...] = ( + "HTTPS_PROXY", "https_proxy", + "HTTP_PROXY", "http_proxy", + "ALL_PROXY", "all_proxy", + "NO_PROXY", "no_proxy", +) + + +# SIGKILL doesn't exist on Windows. We fall back to SIGTERM there, which the +# OS treats as a hard terminate via TerminateProcess() — equivalent semantics. +_KILL_SIGNAL = getattr(signal, "SIGKILL", signal.SIGTERM) + + +# Cached ``iron-proxy --version`` output keyed by binary path. ``get_status`` +# is invoked per Docker-container-create; the version string is constant for +# a given binary so a one-shot subprocess call is plenty. +_VERSION_CACHE: Dict[str, str] = {} + + +# --------------------------------------------------------------------------- +# Public dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class ProxyStatus: + """Snapshot of the iron-proxy installation + runtime state.""" + + enabled: bool = False + binary_path: Optional[Path] = None + binary_version: Optional[str] = None + config_path: Optional[Path] = None + ca_cert_path: Optional[Path] = None + pid: Optional[int] = None + listening: bool = False + tunnel_port: int = _DEFAULT_TUNNEL_PORT + warnings: List[str] = field(default_factory=list) + + @property + def installed(self) -> bool: + return self.binary_path is not None and self.binary_path.exists() + + @property + def configured(self) -> bool: + return ( + self.config_path is not None + and self.config_path.exists() + and self.ca_cert_path is not None + and self.ca_cert_path.exists() + ) + + +@dataclass +class TokenMapping: + """Map a sandbox-visible proxy token to a real upstream credential lookup. + + ``real_env_name`` is the env-var name iron-proxy reads at egress time. + When Bitwarden is configured as the credential source for the proxy, + iron-proxy's *own* environment is populated from bws on startup — the + sandbox still sees only ``proxy_token``. + + ``match_headers`` names the request headers iron-proxy scans for the + proxy token (default: ``Authorization`` for bearer providers; e.g. + ``("x-api-key", "Authorization")`` for Anthropic native). + + ``alias_env_names`` are additional env-var names the SANDBOX receives + the same proxy token under (e.g. ``GOOGLE_API_KEY`` for + ``GEMINI_API_KEY``). They do not appear in the iron-proxy config — + only one secrets rule is emitted per mapping, keyed on + ``real_env_name``. + """ + + proxy_token: str + real_env_name: str + upstream_hosts: Tuple[str, ...] + match_headers: Tuple[str, ...] = ("Authorization",) + alias_env_names: Tuple[str, ...] = () + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + + +def _hermes_bin_dir() -> Path: + from hermes_constants import get_hermes_home + + return get_hermes_home() / "bin" + + +def _proxy_state_dir_ro() -> Path: + """Return the proxy state dir without creating it. + + Read-only callers (status probes, pidfile reads, version queries) use + this — there's no reason to materialize ``~/.hermes/proxy/`` just to + check whether a pidfile exists. + """ + from hermes_constants import get_hermes_home + + return get_hermes_home() / "proxy" + + +def _proxy_state_dir() -> Path: + """Return the proxy state dir, creating it with 0o700 if absent. + + Writable callers (CA gen, config write, mappings write, start_proxy) + use this. We force 0o700 — the dir holds the CA signing key, audit + log, and pidfile, so traversal by other local users is undesirable. + The chmod is unconditional so a pre-existing dir with a slack umask + gets tightened on first access. + """ + d = _proxy_state_dir_ro() + d.mkdir(parents=True, exist_ok=True) + try: + d.chmod(0o700) + except OSError: + # On Windows the chmod is a no-op for POSIX modes; on shared + # filesystems we may not own the dir. Don't fail here — the + # individual files still get explicit perms. + pass + return d + + +def _platform_binary_name() -> str: + return "iron-proxy.exe" if platform.system() == "Windows" else "iron-proxy" + + +def _platform_asset_name() -> str: + """Map (uname, arch) → upstream release asset filename. + + iron-proxy ships ``iron-proxy___.tar.gz``. + Windows builds aren't published upstream as of v0.39.0; we raise a + clear error for callers on Windows. + """ + + system = platform.system() + machine = platform.machine().lower() + + if system == "Linux": + arch = "arm64" if machine in ("arm64", "aarch64") else "amd64" + return f"iron-proxy_{_IRON_PROXY_VERSION}_linux_{arch}.tar.gz" + if system == "Darwin": + arch = "arm64" if machine in ("arm64", "aarch64") else "amd64" + return f"iron-proxy_{_IRON_PROXY_VERSION}_darwin_{arch}.tar.gz" + if system == "Windows": + raise RuntimeError( + "iron-proxy does not ship native Windows binaries as of " + f"v{_IRON_PROXY_VERSION}. Run the proxy on a Linux/macOS host, " + "or inside WSL." + ) + + raise RuntimeError( + f"Unsupported platform for iron-proxy auto-install: {system} {machine}" + ) + + +# --------------------------------------------------------------------------- +# Binary discovery + lazy install +# --------------------------------------------------------------------------- + + +def find_iron_proxy(*, install_if_missing: bool = False) -> Optional[Path]: + """Return a path to a usable ``iron-proxy`` binary, or None. + + Resolution order: + 1. ``/bin/iron-proxy`` (our managed copy — preferred) + 2. ``shutil.which("iron-proxy")`` (system PATH) + + When ``install_if_missing`` is True and neither resolves, calls + :func:`install_iron_proxy` to download and verify the pinned version. + """ + + managed = _hermes_bin_dir() / _platform_binary_name() + if managed.exists() and os.access(managed, os.X_OK): + return managed + + system = shutil.which("iron-proxy") + if system: + return Path(system) + + if install_if_missing: + try: + return install_iron_proxy() + except Exception as exc: # noqa: BLE001 — never block startup + logger.warning("iron-proxy auto-install failed: %s", exc) + return None + return None + + +def install_iron_proxy(*, force: bool = False) -> Path: + """Download, verify, and install the pinned ``iron-proxy`` binary. + + Returns the path to the installed executable. Raises on any failure + (network, checksum, extraction). Callers in the auto-install path catch + these; the user-facing ``hermes proxy install`` surface lets them + propagate so the wizard can show a clear error. + """ + + bin_dir = _hermes_bin_dir() + bin_dir.mkdir(parents=True, exist_ok=True) + target = bin_dir / _platform_binary_name() + + if target.exists() and not force: + return target + + asset_name = _platform_asset_name() + asset_url = f"{_IRON_PROXY_RELEASE_BASE}/{asset_name}" + checksum_url = f"{_IRON_PROXY_RELEASE_BASE}/{_IRON_PROXY_CHECKSUM_NAME}" + + with tempfile.TemporaryDirectory(prefix="hermes-iron-proxy-") as tmpdir: + tmp = Path(tmpdir) + archive_path = tmp / asset_name + checksum_path = tmp / _IRON_PROXY_CHECKSUM_NAME + + logger.info("Downloading %s", asset_url) + _http_download(asset_url, archive_path) + _http_download(checksum_url, checksum_path) + + # Defense-in-depth (maxpetrusenko P1): verify the GPG signature of + # checksums.txt before trusting it. The archive download honors ambient + # proxy env (urllib), so a compromised channel could serve a matching + # binary + checksums pair; the detached signature + pinned public key + # close that release-channel tamper gap. Best-effort: if gpg or the + # signature assets aren't available we log and fall back to the SHA-256 + # check alone rather than hard-failing offline installs. + _verify_checksums_signature(tmp, checksum_path) + + expected = _expected_sha256(checksum_path, asset_name) + actual = _sha256_file(archive_path) + if expected.lower() != actual.lower(): + raise RuntimeError( + f"Checksum mismatch for {asset_name}: " + f"expected {expected}, got {actual}" + ) + + with tarfile.open(archive_path, "r:gz") as tf: + member = _pick_tar_member(tf, _platform_binary_name()) + # PEP 706 data filter — strips ownership/mode replay (we set + # chmod explicitly below) AND rejects symlink/hardlink members + # that escape the extraction dir. Required on 3.12+ to silence + # the deprecation warning and on 3.14+ to opt into the + # tarbomb-rejecting default. + try: + tf.extract(member, tmp, filter="data") # noqa: S202 + except TypeError: + # Python < 3.12 — filter kw didn't exist yet; the + # _pick_tar_member sanitization already rejects path + # traversal so this is acceptable. + tf.extract(member, tmp) # noqa: S202 + extracted = tmp / member.name + + # Stage into the final directory then atomically rename so the new + # binary is never visible half-written. + fd, staged = tempfile.mkstemp(dir=str(bin_dir), prefix=".iron-proxy_") + os.close(fd) + shutil.copy2(extracted, staged) + os.chmod( + staged, + stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IXOTH, + ) + os.replace(staged, target) + + # Invalidate the version cache so a freshly-installed binary + # re-probes ``--version`` on the next ``get_status()`` call instead + # of returning the pre-upgrade string. Long-lived processes that + # bump the pinned version via ``force=True`` need this. + _VERSION_CACHE.pop(str(target), None) + + logger.info("Installed iron-proxy %s at %s", _IRON_PROXY_VERSION, target) + return target + + +def _http_download(url: str, dest: Path) -> None: + req = urllib.request.Request(url, headers={"User-Agent": "hermes-agent"}) + try: + with urllib.request.urlopen(req, timeout=_DOWNLOAD_TIMEOUT) as resp: # noqa: S310 + with open(dest, "wb") as f: + shutil.copyfileobj(resp, f) + except urllib.error.URLError as exc: + raise RuntimeError(f"Failed to download {url}: {exc}") from exc + + +def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: + """Best-effort GPG verification of ``checksums.txt`` (maxpetrusenko P1). + + Downloads the detached signature (``checksums.txt.asc``) and the release + signing key (``public-key.asc``), imports the key into an ephemeral + keyring, and verifies the signature over ``checksum_path``. + + Returns True when the signature is verified. Returns False (with a warning) + when verification is unavailable — ``gpg`` not installed, or the signature / + public-key assets are missing from the release. Raises RuntimeError ONLY + when verification actively FAILS (a present-but-bad signature), which is a + tamper signal we must not ignore. + + Rationale for graceful degradation on "unavailable": the SHA-256 check + against ``checksums.txt`` remains in force regardless, and many install + hosts (CI, minimal containers) won't have gpg. We harden when we can and + never make gpg a hard dependency for a working install. + """ + gpg = shutil.which("gpg") + if not gpg: + logger.warning( + "gpg not found on PATH — skipping iron-proxy release-signature " + "verification (SHA-256 checksum check still enforced)." + ) + return False + + sig_url = f"{_IRON_PROXY_RELEASE_BASE}/{_IRON_PROXY_CHECKSUM_SIG_NAME}" + pubkey_url = f"{_IRON_PROXY_RELEASE_BASE}/{_IRON_PROXY_PUBKEY_NAME}" + sig_path = tmp / _IRON_PROXY_CHECKSUM_SIG_NAME + pubkey_path = tmp / _IRON_PROXY_PUBKEY_NAME + + try: + _http_download(sig_url, sig_path) + _http_download(pubkey_url, pubkey_path) + except RuntimeError as exc: + logger.warning( + "iron-proxy release signature assets unavailable (%s) — skipping " + "GPG verification (SHA-256 checksum check still enforced).", exc, + ) + return False + + # Ephemeral keyring so we never touch the user's real GPG home. + gnupg_home = tmp / "gnupg" + gnupg_home.mkdir(mode=0o700, exist_ok=True) + base_cmd = [gpg, "--homedir", str(gnupg_home), "--batch", "--no-tty"] + + imp = subprocess.run( # noqa: S603 — gpg path from trusted PATH lookup + [*base_cmd, "--import", str(pubkey_path)], + capture_output=True, timeout=60, + ) + if imp.returncode != 0: + logger.warning( + "Could not import iron-proxy signing key — skipping GPG " + "verification (SHA-256 still enforced): %s", + imp.stderr.decode("utf-8", "replace")[:200], + ) + return False + + verify = subprocess.run( # noqa: S603 + [*base_cmd, "--verify", str(sig_path), str(checksum_path)], + capture_output=True, timeout=60, + ) + if verify.returncode != 0: + # A present signature that does NOT verify is a tamper signal — fail hard. + raise RuntimeError( + "iron-proxy checksums.txt failed GPG signature verification — " + "refusing to install (possible release-channel tampering). " + f"gpg: {verify.stderr.decode('utf-8', 'replace')[:300]}" + ) + logger.info("Verified iron-proxy checksums.txt GPG signature.") + return True + + +def _expected_sha256(checksum_file: Path, asset_name: str) -> str: + """Parse the standard ``sha256sum`` output: `` ``.""" + + text = checksum_file.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + parts = line.strip().split() + if len(parts) >= 2 and parts[-1] == asset_name: + return parts[0] + raise RuntimeError( + f"No checksum entry for {asset_name} in {checksum_file.name}" + ) + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def _pick_tar_member(tf: tarfile.TarFile, binary_name: str) -> tarfile.TarInfo: + """Find the binary inside the upstream tar. + + iron-proxy's archive is typically flat (binary at root) but we tolerate + a top-level directory. Members must be regular files with a leaf name + matching ``binary_name``, no absolute paths, and no ``..`` traversal. + """ + + candidates: List[tarfile.TarInfo] = [] + for member in tf.getmembers(): + if not member.isfile(): + continue + if member.name.startswith("/") or ".." in Path(member.name).parts: + continue + if Path(member.name).name == binary_name: + candidates.append(member) + if not candidates: + raise RuntimeError( + f"Could not find {binary_name} inside downloaded archive " + f"(members: {[m.name for m in tf.getmembers()[:5]]}...)" + ) + candidates.sort(key=lambda m: len(m.name)) + return candidates[0] + + +def iron_proxy_version(binary: Path) -> str: + """Return ``iron-proxy --version`` output, stripped. Empty on failure. + + Cached by binary path: ``get_status`` is called per Docker container + create, but the version string is constant for a given binary. A + single subprocess invocation is plenty. + """ + + key = str(binary) + cached = _VERSION_CACHE.get(key) + if cached is not None: + return cached + + try: + # Build a minimal env: only PATH, HOME, and locale vars. + # The version probe is a one-shot subprocess — forwarding + # the full host env (OPENAI_API_KEY, ANTHROPIC_API_KEY, etc.) + # to a PATH-resolved or unverified binary is an unnecessary + # credential leak. Reuse the same allowlist the daemon + # subprocess uses (see _build_proxy_subprocess_env). + minimal_env: Dict[str, str] = {} + parent = os.environ + for name in _PROXY_SUBPROCESS_ENV_ALLOWLIST: + if name in parent: + minimal_env[name] = parent[name] + # The S603 warning is legitimate for the PATH-fallback case + # (find_iron_proxy → shutil.which), but --version with a + # scrubbed env is safe regardless of binary provenance. + res = subprocess.run( # noqa: S603 + [str(binary), "--version"], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=_RUN_TIMEOUT, + env=minimal_env, + ) + except (OSError, subprocess.TimeoutExpired): + return "" + out = (res.stdout or res.stderr or "").strip() + # Don't cache empty output — that would poison ``hermes egress + # status`` for the lifetime of the process if the first probe hit a + # corrupt binary or a flag-rename in a newer upstream. Re-probe on + # the next call instead. + if out: + _VERSION_CACHE[key] = out + return out + + +# --------------------------------------------------------------------------- +# CA cert generation +# --------------------------------------------------------------------------- + + +def ensure_ca_cert(*, force: bool = False) -> Tuple[Path, Path]: + """Generate (or return existing) iron-proxy CA cert + key. + + Uses the host's ``openssl`` binary. We don't try to bind to a Python + crypto library — openssl is universally available on the platforms we + support, and it sidesteps cryptography-package licensing/distribution + surface. + """ + + state = _proxy_state_dir() + ca_crt = state / "ca.crt" + ca_key = state / "ca.key" + + if ca_crt.exists() and ca_key.exists() and not force: + return ca_crt, ca_key + + if shutil.which("openssl") is None: + raise RuntimeError( + "openssl not found on PATH. Install OpenSSL (apt: `openssl`, " + "brew: `openssl`) to generate the iron-proxy CA cert." + ) + + # 10-year cert. iron-proxy mints short-lived leaf certs from this CA, + # so the CA itself only rotates when the user explicitly forces it. + with tempfile.TemporaryDirectory(prefix="hermes-proxy-ca-") as tmpdir: + tmp = Path(tmpdir) + tmp_key = tmp / "ca.key" + tmp_crt = tmp / "ca.crt" + + subprocess.run( # noqa: S603 — openssl path is trusted PATH lookup + ["openssl", "genrsa", "-out", str(tmp_key), "4096"], + check=True, + capture_output=True, + timeout=60, + ) + subprocess.run( # noqa: S603 + [ + "openssl", "req", "-x509", "-new", "-nodes", + "-key", str(tmp_key), + "-sha256", "-days", "3650", + "-subj", "/CN=hermes iron-proxy CA", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign", + "-out", str(tmp_crt), + ], + check=True, + capture_output=True, + timeout=60, + ) + + # Move into place with private permissions. CRITICAL: the key + # has to be created with 0o600 from the very first byte — a + # ``shutil.copy2`` followed by ``os.chmod`` leaves a TOCTOU window + # where the private key is world-readable on multi-user hosts. + key_bytes = tmp_key.read_bytes() + crt_bytes = tmp_crt.read_bytes() + + # Stage with explicit 0o600, then atomically rename into place. + # O_NOFOLLOW guards against a symlink at ca_key (defence-in-depth + # — the state dir is 0o700-owned but a malicious local user with + # the same uid could pre-create one). + key_staged = ca_key.with_suffix(ca_key.suffix + ".staged") + open_flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + # O_NOFOLLOW exists on POSIX; on Windows we just rely on the + # default semantics. + if hasattr(os, "O_NOFOLLOW"): + open_flags |= os.O_NOFOLLOW + # Best-effort: pre-unlink any existing staged file so the open + # with O_CREAT is always against a fresh inode. + try: + key_staged.unlink() + except FileNotFoundError: + pass + fd = os.open(str(key_staged), open_flags, 0o600) + try: + with os.fdopen(fd, "wb") as f: + f.write(key_bytes) + except Exception: + try: + os.close(fd) + except OSError: + pass + raise + os.replace(key_staged, ca_key) + + # Cert is public — 0o644 is fine and matches typical PEM layout. + ca_crt.write_bytes(crt_bytes) + os.chmod(ca_crt, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) + + logger.info("Generated iron-proxy CA at %s", ca_crt) + return ca_crt, ca_key + + +# --------------------------------------------------------------------------- +# Proxy config + token mapping generation +# --------------------------------------------------------------------------- + + +def mint_proxy_token(prefix: str = "hermes-proxy") -> str: + """Mint a fresh opaque token to hand to the sandbox. + + The token has no internal structure beyond a recognizable prefix — + iron-proxy matches on exact equality. We use a 128-bit random suffix + (32 hex chars from a SHA-256 of 32 bytes of os.urandom). At that + entropy the birthday-bound collision probability is below 2^-64 for + up to 2^32 tokens, which is plenty for a proxy-scoped namespace. + """ + + return f"{prefix}-{hashlib.sha256(os.urandom(32)).hexdigest()[:32]}" + + +def _management_token_path() -> Path: + return _proxy_state_dir() / "management.token" + + +def ensure_management_token(*, force: bool = False) -> str: + """Return the management-API bearer key, minting it on first call. + + Stored at ``/proxy/management.token`` with 0600 perms. + The daemon receives it via the ``HERMES_IRON_PROXY_MGMT_KEY`` env var + (named in the generated config's ``management.api_key_env``); + ``hermes egress reload`` reads the same file to authenticate. + """ + + p = _management_token_path() + if not force and p.exists(): + try: + existing = p.read_text(encoding="utf-8").strip() + if existing: + return existing + except OSError: + pass + token = mint_proxy_token(prefix="hermes-mgmt") + fd = os.open( + str(p), + os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + os.fchmod(fd, 0o600) + except (OSError, AttributeError): + pass + try: + os.write(fd, token.encode("utf-8")) + finally: + os.close(fd) + return token + + +def _read_management_token() -> Optional[str]: + p = _proxy_state_dir_ro() / "management.token" + try: + token = p.read_text(encoding="utf-8").strip() + except OSError: + return None + return token or None + + +def _read_management_listen_from_config( + config_path: Optional[Path] = None, +) -> Optional[Tuple[str, int]]: + """Return ``(host, port)`` of the management listener, if configured.""" + + cfg = config_path or (_proxy_state_dir_ro() / "proxy.yaml") + if not cfg.exists(): + return None + try: + import yaml + except ImportError: + return None + try: + data = yaml.safe_load(cfg.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + listen = ((data or {}).get("management") or {}).get("listen") or "" + if not isinstance(listen, str) or ":" not in listen: + return None + host, _, port_s = listen.rpartition(":") + try: + port = int(port_s) + except ValueError: + return None + return (host or "127.0.0.1", port) + + +def reload_proxy() -> bool: + """Hot-reload the running daemon's ruleset via the management API. + + POSTs to ``/v1/reload`` on the loopback management listener; the daemon + re-reads proxy.yaml and atomically swaps the transform pipeline — + validation failures leave the running config untouched (HTTP 422). + + Returns True on a successful reload. Raises ``RuntimeError`` with an + actionable message when the daemon isn't running, the config predates + management-API support (no ``management`` block → restart required), + or the reload is rejected. + """ + + pid = _read_pid() + if not pid or not _pid_alive(pid): + raise RuntimeError( + "iron-proxy is not running — nothing to reload. " + "Run `hermes egress start`." + ) + mgmt = _read_management_listen_from_config() + if mgmt is None: + raise RuntimeError( + "The generated proxy.yaml has no management listener (written " + "before reload support). Re-run `hermes egress setup` and use " + "`hermes egress restart` this one time." + ) + token = _read_management_token() + if not token: + raise RuntimeError( + "management.token is missing — re-run `hermes egress setup`, " + "then `hermes egress restart`." + ) + + import urllib.error + import urllib.request + + host, port = mgmt + req = urllib.request.Request( + f"http://{host}:{port}/v1/reload", + method="POST", + headers={"Authorization": f"Bearer {token}"}, + data=b"", + ) + try: + with urllib.request.urlopen(req, timeout=_MGMT_RELOAD_TIMEOUT) as resp: + if resp.status == 200: + return True + raise RuntimeError( + f"management API returned unexpected status {resp.status}" + ) + except urllib.error.HTTPError as exc: + body = "" + try: + body = exc.read().decode("utf-8", errors="replace")[:500] + except OSError: + pass + if exc.code == 422: + raise RuntimeError( + f"iron-proxy rejected the new config (validation failed; " + f"the running ruleset is unchanged): {body}" + ) from exc + if exc.code == 401: + raise RuntimeError( + "management API rejected our key (401). The running " + "daemon was started with a different management.token — " + "run `hermes egress restart`." + ) from exc + raise RuntimeError( + f"management reload failed (HTTP {exc.code}): {body}" + ) from exc + except (urllib.error.URLError, OSError) as exc: + # A daemon started from a pre-management config is alive but has + # no listener on the management port. + raise RuntimeError( + f"could not reach the management API at {host}:{port} ({exc}). " + "If the daemon was started before reload support, run " + "`hermes egress restart` once." + ) from exc + + +def _default_http_listen(tunnel_port: int) -> List[str]: + """Build the single host:port bind the proxy should listen on. + + iron-proxy v0.39 supports exactly ONE ``proxy.http_listen`` bind per + daemon process, so this returns a one-element list and the choice of + host matters: + + * **Linux:** bind the docker bridge gateway (``172.17.0.1`` by + default). Sandboxes reach the proxy via + ``host.docker.internal:host-gateway``, which Docker resolves to + exactly this bridge gateway IP on Linux — a loopback-only bind is + unreachable from inside containers there. The bridge IP is still + host-local (it's an address on the host's ``docker0`` interface), + so host-side tooling and the status probe can reach it too. When + no docker bridge is detected (docker not installed / not started), + fall back to loopback — there are no sandboxes to serve in that + state, and the operator gets a warning. + * **macOS / Windows Docker Desktop:** ``host.docker.internal`` + resolves via VPNkit to the host, so a loopback bind is reachable + from containers and is the least-exposed choice. + + We never bind ``0.0.0.0`` — that would expose the proxy (and, with a + leaked sandbox token, the user's API quota) to anyone on the local + network. The bridge-gateway bind is reachable by other containers + on the default bridge network, which is unavoidable given v0.39's + single-bind limit; requests still require a minted proxy token and + an allowlisted upstream. + """ + + if platform.system() == "Linux": + bridge_ip = _detect_docker_bridge_ip() + if bridge_ip and bridge_ip != "127.0.0.1": + return [f"{bridge_ip}:{tunnel_port}"] + logger.warning( + "No docker bridge (docker0) detected — binding iron-proxy to " + "loopback only. Docker sandboxes will NOT be able to reach " + "the proxy until it is restarted with docker running." + ) + return [f"127.0.0.1:{tunnel_port}"] + + +def _detect_docker_bridge_ip() -> Optional[str]: + """Return the docker0 bridge IPv4, if present, else None. + + Best-effort: we try ``ip -4 addr show docker0`` first. Anything that + fails, doesn't parse as a strict IPv4, or parses as an address we + must NOT bind to (unspecified, loopback, multicast, reserved, public) + returns None — callers handle that as "no bridge bind". + + SECURITY: a hostile ``ip`` shim earlier on the operator's PATH used + to be able to inject ``0.0.0.0`` here and re-open INADDR_ANY binding + that the rest of the bind-policy work explicitly closed. We + validate via :mod:`ipaddress` and reject anything that isn't + plausibly a docker bridge IP (private + non-special). + """ + + candidate: Optional[str] = None + try: + res = subprocess.run( # noqa: S603 — ip is a system binary + ["ip", "-4", "-o", "addr", "show", "docker0"], + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2, + ) + if res.returncode == 0: + for line in res.stdout.splitlines(): + parts = line.split() + # Expected: ": docker0 inet 172.17.0.1/16 ..." + for i, tok in enumerate(parts): + if tok == "inet" and i + 1 < len(parts): + candidate = parts[i + 1].split("/")[0] + break + if candidate is not None: + break + except (OSError, subprocess.TimeoutExpired): + return None + + if not candidate: + return None + + # Stdlib validation: rejects garbage strings AND special-purpose + # addresses that must not be used as a bind target. + try: + addr = ipaddress.IPv4Address(candidate) + except (ipaddress.AddressValueError, ValueError): + return None + # Reject: + # - 0.0.0.0 / INADDR_ANY (is_unspecified) + # - 127.0.0.0/8 (is_loopback — already in deny list) + # - 224.0.0.0/4 (is_multicast) + # - 240.0.0.0/4 (is_reserved) + # - 169.254.0.0/16 (is_link_local — IMDS range, never docker0) + # - global / public IPs (is_global — docker0 must be RFC1918) + if ( + addr.is_unspecified + or addr.is_loopback + or addr.is_multicast + or addr.is_reserved + or addr.is_link_local + or addr.is_global + ): + logger.warning( + "Refusing suspicious docker bridge IP %s reported by `ip`; " + "skipping bridge bind.", candidate, + ) + return None + + return str(addr) + + +def build_proxy_config( + *, + mappings: List[TokenMapping], + ca_cert: Path, + ca_key: Path, + tunnel_port: int = _DEFAULT_TUNNEL_PORT, + audit_log: Optional[Path] = None, + allowed_hosts: Optional[List[str]] = None, + upstream_deny_cidrs: Optional[List[str]] = None, + http_listen: Optional[List[str]] = None, +) -> Dict: + """Build the iron-proxy YAML config (as a dict) for a given mapping set. + + The dict is YAML-serializable via ``yaml.safe_dump``. iron-proxy reads + real secrets from its OWN environment via ``source: {type: env, var: ...}``; + the sandbox never sees them. + + Bind policy: the sandbox-facing listeners (``tunnel_listen`` on + ``tunnel_port``, plain-HTTP ``http_listen`` on ``tunnel_port + 1``) + bind the docker bridge gateway on Linux (``172.17.0.1`` or whatever + ``docker0`` resolves to — that's what ``host.docker.internal`` + resolves to inside containers there) and loopback on macOS / Windows + Docker Desktop. We do NOT bind ``0.0.0.0`` — a LAN peer with a + leaked sandbox token could otherwise spend the operator's API quota + against any allowlisted upstream. + + SSRF policy: ``upstream_deny_cidrs`` defaults to a conservative deny + list covering loopback, link-local (incl. AWS/GCP/Azure IMDS at + 169.254.169.254), and RFC1918. Pass an explicit ``[]`` to opt out of + the deny list entirely (only sensible in hermetic tests). + + Schema mirrors the official iron-proxy schema as of v0.39.0. Notable + points: + + * The ``dns`` section is required by the binary even when we only use the + CONNECT tunnel. We point it at loopback so it doesn't conflict with + anything else and disable the listener. + * The ``proxy.tunnel_listen`` is what sandboxes hit via ``HTTPS_PROXY``. + ``http_listen`` / ``https_listen`` are present (loopback only) so the + proxy boots; sandboxes never route directly to them. + * ``allowlist`` transform takes ``domains:`` and ``cidrs:``, not ``hosts:``. + * ``secrets`` transform takes ``secrets:`` (plural), each with a + ``source``, a ``replace.proxy_value`` (the sandbox-visible token), and + a list of ``rules`` saying which hosts the swap should fire on. + """ + + hosts: List[str] = list(allowed_hosts or _DEFAULT_ALLOWED_HOSTS) + for m in mappings: + for h in m.upstream_hosts: + if h not in hosts: + hosts.append(h) + + secrets_rules = [] + for m in mappings: + match_headers = list(m.match_headers or ("Authorization",)) + secrets_rules.append({ + "source": {"type": "env", "var": m.real_env_name}, + "replace": { + "proxy_value": m.proxy_token, + # Per-provider header set: bearer providers match only + # Authorization; header-auth providers (Anthropic native + # x-api-key, Azure api-key, Gemini x-goog-api-key) match + # their native header (+ Authorization where the provider + # also accepts bearer flows). v0.39 matches header names + # case-insensitively — see parseHeaderMatchers upstream. + "match_headers": match_headers, + # The token is also accepted as a query param — v0.39 scans + # every query parameter for the token value, which covers + # SDKs that pass ``?key=`` (Gemini) as well as + # bearer-in-query styles. Body matching is off — we + # don't want body inspection forced for every request. + "match_query": True, + "match_body": False, + # Fail closed (maxpetrusenko P1): when a request reaches an + # allowlisted upstream WITHOUT the proxy token present in a + # matched location, reject it instead of forwarding as-is. + # Without this, a real provider key that a sandbox process + # sent directly (not via the minted token) would still pass + # the proxy boundary to the allowed host. With require=true, + # iron-proxy returns ActionReject when no token swap fired + # (v0.39 secrets transform: replaceConfig.Require, enforced in + # TransformRequest — verified present in the pinned version). + "require": True, + }, + "rules": [{"host": h} for h in m.upstream_hosts], + }) + + # SSRF protection: default-deny cloud metadata + loopback + RFC1918. + # Callers can pass [] to opt out entirely (hermetic tests need this for + # talking to a loopback upstream). None means "use the default". + deny_cidrs: List[str] + if upstream_deny_cidrs is None: + deny_cidrs = list(_DEFAULT_UPSTREAM_DENY_CIDRS) + else: + deny_cidrs = list(upstream_deny_cidrs) + + # Listen addresses. iron-proxy v0.39 takes a single string per + # listener field — there is no plural ``http_listens`` form, despite + # earlier drafts of this module claiming v0.39 accepts both. An + # empirical strings(1) audit + a live "start the binary and observe + # the YAML unmarshal error" confirms the singular form is the only + # one the binary accepts. + # + # LISTENER ROLES (verified live against the v0.39 binary): + # * ``tunnel_listen`` is the CONNECT + MITM listener. HTTPS through + # ``HTTPS_PROXY`` issues CONNECT — this is the listener sandboxes + # must reach. A CONNECT sent to ``http_listen`` is NOT terminated: + # v0.39 forwards it upstream as a regular request and the upstream + # responds 400. + # * ``http_listen`` is the absolute-form plain-HTTP forward listener + # (``HTTP_PROXY`` for ``http://`` URLs). Transforms fire here too. + # Both get the sandbox-facing bind host: tunnel on ``tunnel_port``, + # plain HTTP on ``tunnel_port + 1``. + # + # The bind host comes from _default_http_listen: the docker bridge + # gateway on Linux (containers reach the proxy via + # host.docker.internal, which maps to the bridge gateway there — + # loopback would be unreachable from inside sandboxes) and loopback + # on macOS/Windows Docker Desktop (where host.docker.internal routes + # to the host via VPNkit). + listens = list(http_listen) if http_listen else _default_http_listen(tunnel_port) + primary_listen = listens[0] if listens else f"127.0.0.1:{tunnel_port}" + bind_host = primary_listen.rsplit(":", 1)[0] or "127.0.0.1" + plain_http_listen = f"{bind_host}:{tunnel_port + 1}" + + log_block: Dict = {"level": "info"} + # NOTE: ``log.audit_path`` is NOT a field in iron-proxy v0.39's + # ``config.Log`` struct — the binary rejects it with + # ``field audit_path not found in type config.Log``. Per-request + # audit records are written to the same log destination as + # everything else at this binary version; the operator-facing + # ``audit.log`` file we pre-create is still useful as a sentinel + # for monitoring (logrotate target, downstream tail watchers) but + # the daemon does not write to it directly. The kwarg is kept so + # we're forward-compatible with a future v0.40+ that adds the + # field; if you upgrade _IRON_PROXY_VERSION and the upstream gains + # ``log.audit_path``, re-enable the line below. + # if audit_log is not None: + # log_block["audit_path"] = str(audit_log) + _ = audit_log # consumed by ensure_audit_log() / docs only on v0.39 + + return { + # DNS section is required by the binary's config parser, but we run + # in tunnel-only mode so the DNS listener never binds an exposed port. + # Sandboxes reach the proxy via HTTPS_PROXY/CONNECT, not via DNS + # redirection. + "dns": { + "listen": "127.0.0.1:0", # ephemeral loopback — effectively disabled + "proxy_ip": "127.0.0.1", + }, + "proxy": { + # tunnel_listen is the CONNECT/MITM listener — what sandboxes + # hit via `HTTPS_PROXY=http://host:tunnel_port` for HTTPS + # upstreams (curl/requests/node issue CONNECT through it). + # http_listen handles absolute-form plain-HTTP forwards + # (`HTTP_PROXY` for http:// URLs) on tunnel_port+1. Both + # bind the docker bridge gateway on Linux / loopback on + # Docker Desktop — NEVER 0.0.0.0. LAN peers with a leaked + # sandbox token would otherwise be able to spend the + # operator's API quota against any allowlisted upstream. + "tunnel_listen": primary_listen, + "http_listen": plain_http_listen, + # The HTTPS-listener (direct TLS termination, no CONNECT) + # gets a loopback ephemeral port — we don't expose it. + "https_listen": "127.0.0.1:0", + "max_request_body_bytes": 16 * 1024 * 1024, + "max_response_body_bytes": 0, + "upstream_response_header_timeout": "120s", + # SSRF protection: deny outbound to cloud metadata + loopback by + # default. An empty list opts out entirely. + "upstream_deny_cidrs": deny_cidrs, + }, + # iron-proxy v0.39 starts a Prometheus-style metrics server by + # default on ``:9090`` — which is the SAME port as our default + # ``tunnel_port: 9090``, causing a guaranteed bind collision on + # startup. Pin the metrics listener to an ephemeral loopback + # port (``127.0.0.1:0``) so the metrics binding can't collide + # with the proxy listener regardless of what tunnel_port the + # operator chose. NOTE: ``:0`` means the kernel picks a fresh + # random port each start and nothing records it — metrics are + # effectively disabled/undiscoverable at this pin. If we want + # scrapable metrics later, allocate a fixed port and surface it + # in ``ProxyStatus`` / ``hermes egress status``. + "metrics": { + "listen": "127.0.0.1:0", + }, + # Operator-facing management API — loopback only, bearer-key + # authenticated (key read from the env var named below; injected + # by ``start_proxy`` from ``management.token``). ``POST /v1/reload`` + # re-reads THIS config file and atomically swaps the transform + # pipeline — `hermes egress reload` applies allowlist/token/mapping + # changes without a restart. Loopback deliberately: sandboxes must + # never reach the management surface, so it does NOT bind the + # docker bridge like the traffic listeners do. + "management": { + "listen": f"127.0.0.1:{tunnel_port + _MGMT_PORT_OFFSET}", + "api_key_env": _MGMT_API_KEY_ENV, + }, + "tls": { + "ca_cert": str(ca_cert), + "ca_key": str(ca_key), + "cert_cache_size": 1000, + "leaf_cert_expiry_hours": 168, + }, + "transforms": [ + { + "name": "allowlist", + "config": {"domains": hosts}, + }, + { + "name": "secrets", + "config": {"secrets": secrets_rules}, + }, + ], + "log": log_block, + } + + +def ensure_audit_log(audit_path: Path) -> None: + """Create the audit log file with private permissions (0o600). + + Called from the wizard right before ``start_proxy``. On the pinned + v0.39 the daemon never writes this file (no ``log.audit_path`` + config field), so the pre-create is purely forward-compat: when the + pin moves to a version that supports a dedicated audit stream, the + file already exists with tight permissions and the daemon inherits + them instead of creating it under the default umask. + + Raises :class:`RuntimeError` on any OSError (planted symlink, + immutable parent dir, full disk) so the caller can decide how to + surface it. The wizard treats this as a WARNING on v0.39 — the + file is non-load-bearing until the version bump — but the qualified + message keeps operators from wiring monitoring to a path that can't + exist. + """ + + try: + # Use os.open + O_CREAT to avoid races on the chmod. + open_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + if hasattr(os, "O_NOFOLLOW"): + open_flags |= os.O_NOFOLLOW + fd = os.open(str(audit_path), open_flags, 0o600) + try: + # Tighten perms even if the file already existed under a + # slacker umask. + os.fchmod(fd, 0o600) + finally: + os.close(fd) + except OSError as exc: + raise RuntimeError( + f"Refusing to start: could not pre-create audit log " + f"{audit_path} with restrictive permissions ({exc}). " + f"Move or chmod any existing file at that path and retry." + ) from exc + + +def write_proxy_config(config: Dict) -> Path: + """Serialize the config dict to ``/proxy/proxy.yaml``. + + Uses ``yaml.safe_dump`` so we never emit Python tags. + """ + + try: + import yaml # PyYAML is already a Hermes dep + except ImportError as exc: + raise RuntimeError( + "PyYAML is required to write the iron-proxy config but is not " + "installed." + ) from exc + + state = _proxy_state_dir() + out = state / "proxy.yaml" + tmp_path = state / ".proxy.yaml.tmp" + with open(tmp_path, "w", encoding="utf-8") as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) + # Tighten perms on the temp file BEFORE the atomic replace so the + # final path is never briefly world-readable under a slack umask + # (the config embeds proxy token values). chmod-after-replace would + # leave a TOCTOU window; the 0o700 state dir mitigates but same-uid + # processes could still race. + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IWUSR) + os.replace(tmp_path, out) + return out + + +def write_mappings(mappings: List[TokenMapping]) -> Path: + """Persist the sandbox-visible proxy tokens to ``mappings.json``. + + The Docker backend reads this file to inject the right tokens as env + vars when starting a sandbox. The file is NOT read by iron-proxy + itself — the mapping is already baked into ``proxy.yaml``. + """ + + state = _proxy_state_dir() + out = state / "mappings.json" + payload = { + "version": 1, + "tokens": [ + { + "proxy_token": m.proxy_token, + "env_name": m.real_env_name, + "upstream_hosts": list(m.upstream_hosts), + "match_headers": list(m.match_headers), + "alias_env_names": list(m.alias_env_names), + } + for m in mappings + ], + } + tmp_path = state / ".mappings.json.tmp" + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + # chmod before the atomic replace — see write_proxy_config. The + # mappings file holds proxy token values, so close the TOCTOU window + # rather than chmod-ing after the file is already at its final path. + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IWUSR) + os.replace(tmp_path, out) + return out + + +def load_mappings() -> List[TokenMapping]: + """Read mappings.json, if it exists. Empty list on any error.""" + + state = _proxy_state_dir() + f = state / "mappings.json" + if not f.exists(): + return [] + try: + payload = json.loads(f.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("Failed to read iron-proxy mappings.json: %s", exc) + return [] + out: List[TokenMapping] = [] + for item in payload.get("tokens", []): + try: + out.append(TokenMapping( + proxy_token=item["proxy_token"], + real_env_name=item["env_name"], + upstream_hosts=tuple(item.get("upstream_hosts") or ()), + # Pre-header-auth mappings.json files (written before the + # match_headers/alias fields existed) load with the bearer + # defaults — identical to their behavior at write time. + match_headers=tuple(item.get("match_headers") or ("Authorization",)), + alias_env_names=tuple(item.get("alias_env_names") or ()), + )) + except (KeyError, TypeError): + continue + return out + + +def discover_provider_mappings( + *, + available_env_names: Optional[List[str]] = None, +) -> List[TokenMapping]: + """Mint a TokenMapping for every known provider whose env var is set. + + Pass ``available_env_names`` to override the lookup source (used by the + Bitwarden adapter so we mint mappings for keys that *will* be in the + proxy's environment even if they aren't in the host process env right + now). + """ + + if available_env_names is not None: + names = set(available_env_names) + else: + names = {k for k, v in os.environ.items() if v} + + mappings: List[TokenMapping] = [] + for env_name, hosts in _BEARER_PROVIDERS.items(): + if env_name not in names: + continue + mappings.append(TokenMapping( + proxy_token=mint_proxy_token(prefix=env_name.lower().replace("_api_key", "")), + real_env_name=env_name, + upstream_hosts=hosts, + )) + for env_name, spec in _HEADER_AUTH_PROVIDERS.items(): + aliases = tuple(spec.get("aliases") or ()) + # A mapping is minted when the canonical name OR any alias is + # available. Aliases collapse into ONE mapping (single secrets + # rule) because two require-rules on the same host would reject + # each other's requests. The canonical env name is what + # iron-proxy reads — when only the alias is set in the host env, + # the subprocess-env builder mirrors it (see + # ``_build_proxy_subprocess_env``). + if env_name not in names and not any(a in names for a in aliases): + continue + mappings.append(TokenMapping( + proxy_token=mint_proxy_token(prefix=env_name.lower().replace("_api_key", "")), + real_env_name=env_name, + upstream_hosts=tuple(spec["hosts"]), + match_headers=tuple(spec["match_headers"]), + alias_env_names=aliases, + )) + return mappings + + +def discover_uncovered_providers( + *, + available_env_names: Optional[List[str]] = None, +) -> List[str]: + """Return env-var names for providers we recognize but can't proxy. + + AWS Bedrock (SigV4) and GCP Vertex (SDK-minted OAuth) can't be swapped + by a static header replacement. When any of these are configured, the + sandbox is holding real credentials that the proxy can't strip — the + isolation guarantee is incomplete for those providers. + + The wizard and ``hermes egress status`` use this to print a warning. + (Anthropic / Azure OpenAI / Gemini used to be here; they're now + first-class swapped providers via ``_HEADER_AUTH_PROVIDERS``.) + """ + + if available_env_names is not None: + names = set(available_env_names) + else: + names = {k for k, v in os.environ.items() if v} + + return [n for n in _NON_BEARER_PROVIDERS if n in names] + + +def merge_mappings( + *, + existing: List[TokenMapping], + discovered: List[TokenMapping], + rotate: bool = False, +) -> List[TokenMapping]: + """Combine an existing mapping set with freshly discovered providers. + + By default this PRESERVES tokens for providers already in ``existing`` — + re-running ``hermes egress setup`` should not invalidate the tokens + baked into containers that are already running. Only newly added + providers get freshly minted tokens. + + When ``rotate=True``, every token in the result is freshly minted + regardless of overlap. The wizard exposes this via ``--rotate-tokens`` + for the rare case where the operator wants to roll all tokens + deliberately (e.g. after a suspected token leak). + + Providers that are in ``existing`` but no longer in ``discovered`` + (operator removed the env var since last setup) are dropped. + """ + + by_name = {m.real_env_name: m for m in existing} + out: List[TokenMapping] = [] + for d in discovered: + prior = by_name.get(d.real_env_name) + if prior is not None and not rotate: + # Preserve the token; refresh hosts/headers/aliases in case + # the provider spec changed since last setup (new upstreams, + # a provider moving from uncovered to header-auth, etc). + out.append(TokenMapping( + proxy_token=prior.proxy_token, + real_env_name=prior.real_env_name, + upstream_hosts=d.upstream_hosts, + match_headers=d.match_headers, + alias_env_names=d.alias_env_names, + )) + else: + out.append(d) + return out + + +# --------------------------------------------------------------------------- +# Subprocess lifecycle +# --------------------------------------------------------------------------- + + +def _pidfile() -> Path: + return _proxy_state_dir() / "iron-proxy.pid" + + +def _read_pid() -> Optional[int]: + # Use the read-only path: don't create the proxy dir just to read the + # pidfile. If neither pid file nor dir exists, the daemon is plainly + # not running. + pf = _proxy_state_dir_ro() / "iron-proxy.pid" + if not pf.exists(): + return None + try: + pid = int(pf.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return None + return pid if pid > 0 else None + + +# Nonce env-var set in the iron-proxy subprocess at start_proxy time. Used +# by ``_pid_alive`` to confirm a candidate PID still refers to *our* managed +# binary even across PID recycling (a fresh process can't inherit our +# arbitrary env value). +_HERMES_IRON_PROXY_NONCE_ENV = "HERMES_IRON_PROXY_NONCE" +_proxy_nonce: Optional[str] = None + + +def _pid_proc_starttime(pid: int) -> Optional[str]: + """Return /proc//stat[21] (starttime) on Linux, else None. + + Comparing starttime is the standard cheap way to detect PID recycling + without relying on cmdline scanning. When None, callers fall back to + the cmdline + nonce check. + """ + try: + text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") + except OSError: + return None + # /proc//stat: pid (comm-with-parens) state ppid ... fields[21]=starttime + # The "comm" field can contain spaces and parens, so split from the + # right parenthesis instead of using shlex. + rparen = text.rfind(")") + if rparen < 0: + return None + fields = text[rparen + 1:].split() + # field index in the post-")" tail: original 3..n become fields[0..n-3] + # starttime is original field 22 (1-indexed) → tail index 22-3 = 19 + if len(fields) <= 19: + return None + return fields[19] + + +def _persisted_nonce_path() -> Path: + """Path to the on-disk sibling of the pidfile that stores the nonce. + + Written by ``_write_pidfile_safely`` after ``start_proxy`` plants + the nonce in the iron-proxy child env, read by ``_pid_alive`` in a + later CLI invocation (``stop`` / ``status``) so cross-process + PID-recycling defense holds. + """ + return _proxy_state_dir_ro() / "iron-proxy.nonce" + + +def _read_persisted_nonce() -> Optional[str]: + """Read the on-disk nonce written next to the pidfile. + + Returns None when the file is missing, unreadable, or empty — + callers fall back to argv0 basename matching in that case. + """ + p = _persisted_nonce_path() + try: + # O_NOFOLLOW: defence-in-depth against a planted symlink at the + # nonce path; same-uid required to plant one but worth defending + # since the nonce read here decides whether stop_proxy will + # SIGKILL a candidate PID. + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(str(p), flags) + except OSError: + return None + try: + # Ownership check — if the file isn't owned by us, ignore it. + # Same threat model as the pidfile uid check. + try: + st = os.fstat(fd) + if hasattr(os, "getuid") and st.st_uid != os.getuid(): + return None + except AttributeError: + pass + data = os.read(fd, 256).decode("utf-8", errors="ignore").strip() + return data or None + finally: + os.close(fd) + + +def _pid_alive(pid: int) -> bool: + """Return True iff ``pid`` is alive AND is an iron-proxy process. + + Defends against PID reuse via three signals (in priority order): + 1. ``/proc//environ`` contains our nonce (most reliable, Linux) + 2. ``/proc//cmdline`` basename matches the managed binary + 3. ``ps -p `` command line contains the binary path + + The legacy ``"iron-proxy" in cmdline`` match was loose enough to match + ``tail iron-proxy.log`` or an editor with that file open. We tighten + on argv[0] basename plus an in-process nonce instead. + """ + + if pid <= 0: + return False + try: + # Use psutil.pid_exists when available — it's a no-op on Windows + # whereas os.kill(pid, 0) on Windows is actually a hard kill + # (CTRL_C_EVENT to the target's console process group). See + # bpo-14484. windows-footgun: ok — we explicitly skip the + # os.kill probe on Windows below. + import psutil # type: ignore + if not psutil.pid_exists(pid): + return False + except ImportError: + if platform.system() == "Windows": + # On Windows without psutil we can't safely probe — assume + # the pidfile content is fresh and confirm via the cmdline + # path below. os.kill(pid, 0) is NOT safe here. + pass + else: + try: + os.kill(pid, 0) # windows-footgun: ok — POSIX-only branch + except (ProcessLookupError, PermissionError, OSError): + return False + + # Strong proof: nonce env var matches. /proc//environ is null- + # separated KEY=VALUE pairs; substring search is safe. + # + # The nonce can come from either: + # 1. the module-global ``_proxy_nonce`` set during this process's + # own ``start_proxy`` call (same-process case); + # 2. the on-disk ``iron-proxy.nonce`` file written by + # ``_write_pidfile_safely``, used when ``start`` and ``stop`` + # run in separate CLI invocations (cross-process case). + # Either source provides the same defeat-PID-recycling guarantee. + nonce_candidates: List[str] = [] + if _proxy_nonce: + nonce_candidates.append(_proxy_nonce) + on_disk = _read_persisted_nonce() + if on_disk and on_disk not in nonce_candidates: + nonce_candidates.append(on_disk) + if nonce_candidates: + try: + env_bytes = Path(f"/proc/{pid}/environ").read_bytes() + for nonce in nonce_candidates: + needle = f"{_HERMES_IRON_PROXY_NONCE_ENV}={nonce}".encode() + if needle in env_bytes: + return True + except OSError: + pass + + # Fallback: cmdline basename match. argv[0] is the first null- + # separated token in /proc//cmdline. + try: + cmdline_path = Path(f"/proc/{pid}/cmdline") + if cmdline_path.exists(): + tokens = cmdline_path.read_bytes().split(b"\x00") + if tokens: + argv0 = tokens[0].decode("utf-8", errors="ignore") + argv0_base = os.path.basename(argv0) + if argv0_base.startswith("iron-proxy"): + return True + return False + except OSError: + pass + + # macOS / non-Linux fallback: ``ps`` command basename. + try: + res = subprocess.run( # noqa: S603 + ["ps", "-p", str(pid), "-o", "comm="], + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2, + ) + if res.returncode == 0: + comm = (res.stdout or "").strip() + return os.path.basename(comm).startswith("iron-proxy") + except (OSError, subprocess.TimeoutExpired): + pass + + # Exotic platforms: be conservative — if the OS says alive we believe + # it. This restores the previous behaviour for non-Linux/non-macOS. + return True + + +def start_proxy( + *, + binary: Optional[Path] = None, + config_path: Optional[Path] = None, + extra_env: Optional[Dict[str, str]] = None, + install_if_missing: bool = True, + refresh_secrets_from_bitwarden: bool = False, + bitwarden_config: Optional[Dict] = None, +) -> ProxyStatus: + """Spawn iron-proxy as a managed background subprocess. + + Idempotent — if the proxy is already running with the expected PID, + just returns the live status. + + ``refresh_secrets_from_bitwarden=True`` re-fetches upstream secrets + via ``bws secret list`` at startup and injects them into the child + env. This delivers the rotation promise that distinguishes + ``credential_source: bitwarden`` from ``credential_source: env``. + Without this flag (or with ``bitwarden_config=None``) the proxy still + starts but uses whatever the host process env happens to contain. + """ + + global _proxy_nonce + + existing = _read_pid() + if existing and _pid_alive(existing): + return get_status() + + bin_path = binary or find_iron_proxy(install_if_missing=install_if_missing) + if bin_path is None: + raise RuntimeError( + "iron-proxy binary not available — run `hermes egress install`." + ) + + cfg = config_path or (_proxy_state_dir() / "proxy.yaml") + if not cfg.exists(): + raise RuntimeError( + f"iron-proxy config not found at {cfg}. " + "Run `hermes egress setup` first." + ) + + # Build a minimal subprocess env. os.environ.copy() would ship every + # secret in the operator's shell to the proxy — /proc//environ + # would then expose OPENAI_API_KEY, AWS keys, etc. to any same-uid + # local process. Defeats the threat model the proxy exists to + # mitigate. + env = _build_proxy_subprocess_env( + extra_env=extra_env, + refresh_from_bitwarden=refresh_secrets_from_bitwarden, + bitwarden_config=bitwarden_config, + ) + + # If the generated config enables the management API, the daemon + # validates at startup that the api_key_env is non-empty. Inject the + # persisted key (minting it if this is a config written by a newer + # setup but the token file was removed). + if _read_management_listen_from_config(cfg) is not None: + env[_MGMT_API_KEY_ENV] = ensure_management_token() + + # Plant a per-start nonce in the child env so ``_pid_alive`` can + # confirm a candidate PID still refers to *our* binary across PID + # recycling. Module-global is fine — only one managed proxy per + # Hermes process. + _proxy_nonce = hashlib.sha256(os.urandom(16)).hexdigest() + env[_HERMES_IRON_PROXY_NONCE_ENV] = _proxy_nonce + + log_path = _proxy_state_dir() / "iron-proxy.log" + # Keep ownership of the fd tight: open with explicit 0o600 so the + # log doesn't get world-readable under a slack umask, then close it + # immediately after Popen (the child has its own dup). Without the + # close-on-success path, every restart leaked one fd in the Hermes + # process. + # + # O_NOFOLLOW (defence-in-depth, same threat model as the pidfile + # path): a same-uid attacker who plants ``iron-proxy.log`` as a + # symlink to e.g. ``~/.ssh/authorized_keys`` would otherwise cause + # every restart to append daemon diagnostics to that file. + log_open_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + if hasattr(os, "O_NOFOLLOW"): + log_open_flags |= os.O_NOFOLLOW + try: + log_fd = os.open(str(log_path), log_open_flags, 0o600) + except OSError as exc: + # ELOOP from a planted symlink — refuse with a clear error. + raise RuntimeError( + f"Refusing to write iron-proxy log {log_path}: {exc}. " + "Remove that path manually and retry." + ) from exc + try: + os.fchmod(log_fd, 0o600) # tighten if file pre-existed + except OSError: + pass + # Verify ownership — same st_uid check the pidfile uses. + try: + st = os.fstat(log_fd) + if hasattr(os, "getuid") and st.st_uid != os.getuid(): + os.close(log_fd) + raise RuntimeError( + f"iron-proxy log {log_path} has unexpected owner " + f"uid={st.st_uid}; refusing to write." + ) + except AttributeError: + pass # Windows + + try: + # Use the fd directly via the dup mechanism; Popen will dup() it + # into the child so we can close ours unconditionally below. + # NOTE: on Windows ``start_new_session`` is invalid; we don't + # support Windows for the proxy (the binary itself doesn't ship) + # but the kwarg is POSIX-only and silently ignored on Win. + popen_kwargs: Dict = dict( + env=env, + stdin=subprocess.DEVNULL, + stdout=log_fd, + stderr=subprocess.STDOUT, + ) + if platform.system() != "Windows": + popen_kwargs["start_new_session"] = True + proc = subprocess.Popen( # noqa: S603 — binary path is trusted + [str(bin_path), "-config", str(cfg)], + **popen_kwargs, + ) + except OSError as exc: + os.close(log_fd) + raise RuntimeError(f"failed to spawn iron-proxy: {exc}") from exc + finally: + # Close our copy of the fd whether Popen raised or succeeded. + # The child has its own dup via Popen, so it's still writing. + try: + os.close(log_fd) + except OSError: + pass + + # Write the pidfile IMMEDIATELY after Popen, BEFORE the listening + # verification. If the parent dies during the poll loop (SIGINT, + # OOM, kernel pause), the pidfile is still on disk so the next + # ``hermes egress stop`` can clean up the orphan. Failure paths + # below unlink the pidfile when they kill the child. + pidfile = _pidfile() + try: + _write_pidfile_safely(pidfile, proc.pid) + except RuntimeError: + # Kill the orphan so we don't leave a daemon nobody can stop. + _kill_and_wait(proc, grace_seconds=2) + raise + + # Poll-with-timeout instead of an unconditional 5s sleep. The Go + # binary normally comes up in <200ms; falling through within 100ms + # of liveness keeps Docker container creation snappy. + # + # We scope a Ctrl-C handler around the poll loop so an operator who + # hits Ctrl-C while waiting for ``hermes egress start`` doesn't leak + # an orphan with the port bound. + # + # Probe the CONFIGURED bind host, not loopback unconditionally — on + # Linux the daemon binds the docker bridge gateway, where a loopback + # connect never succeeds and we'd kill a healthy daemon as "never + # came up". + listen_hp = _read_http_listen_from_config() + if listen_hp is not None: + probe_host, tunnel_port = listen_hp + else: + probe_host, tunnel_port = "127.0.0.1", _DEFAULT_TUNNEL_PORT + listening = False + + def _interrupt_handler(_signum, _frame): # pragma: no cover - signal path + # Kill the child and unlink the pidfile, then re-raise so the + # caller sees the interrupt. + _kill_and_wait(proc, grace_seconds=2) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise KeyboardInterrupt() + + prev_sigint = None + prev_sigterm = None + install_handlers = ( + platform.system() != "Windows" + and threading.current_thread() is threading.main_thread() + ) + if install_handlers: + prev_sigint = signal.signal(signal.SIGINT, _interrupt_handler) + prev_sigterm = signal.signal(signal.SIGTERM, _interrupt_handler) + try: + deadline = time.time() + _STARTUP_GRACE_SECONDS + # Do-while shape: check listening at least once even when the + # grace window is 0 (test harness / synchronous fast-path). + while True: + if proc.poll() is not None: + tail = _tail_log(log_path, lines=20) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise RuntimeError( + f"iron-proxy exited immediately (code {proc.returncode}). " + f"Last log lines:\n{tail}" + ) + if _port_listening(probe_host, tunnel_port): + listening = True + break + if time.time() >= deadline: + break + time.sleep(0.1) + finally: + if install_handlers: + signal.signal(signal.SIGINT, prev_sigint) + signal.signal(signal.SIGTERM, prev_sigterm) + + # Final exit check — process may have died right at deadline. + if proc.poll() is not None: + tail = _tail_log(log_path, lines=20) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise RuntimeError( + f"iron-proxy exited immediately (code {proc.returncode}). " + f"Last log lines:\n{tail}" + ) + + # The previous version of this code treated "process still alive at + # deadline" as success. That left iron-proxy running but + # non-listening on the port, with a pidfile pointing at it — + # subsequent restarts would fail with "address in use" because the + # orphan still held the port. Require port-listening for success. + if not listening: + tail = _tail_log(log_path, lines=20) + _kill_and_wait(proc, grace_seconds=2) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise RuntimeError( + f"iron-proxy did not bind {probe_host}:{tunnel_port} within " + f"{_STARTUP_GRACE_SECONDS}s. Process was killed. " + f"Last log lines:\n{tail}" + ) + + logger.info("Started iron-proxy pid=%s config=%s", proc.pid, cfg) + return get_status() + + +def _write_pidfile_safely(pidfile: Path, pid: int) -> None: + """Write ``pid`` to ``pidfile`` with O_EXCL + O_NOFOLLOW + ownership check. + + O_EXCL means "another start is in progress" if the file already + exists with a live owner — we cleanly fail rather than racing. When + the existing pidfile points at a dead pid (stale crash), we + explicitly unlink it before retrying once. + + Side effect: also persists the in-process nonce to disk so + cross-CLI-invocation ``_pid_alive`` checks (start in one process, + stop in another) can still defeat PID recycling. + """ + open_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + open_flags |= os.O_NOFOLLOW + try: + fd = os.open(str(pidfile), open_flags, 0o600) + except FileExistsError: + # Pidfile already exists. If it points at a live iron-proxy, + # caller's _read_pid + _pid_alive at the top of start_proxy + # should already have returned. Reaching here means EITHER + # the previous _pid_alive check raced (rare; another start in + # flight), OR a stale pidfile survived a crash. Discriminate + # and retry once with O_TRUNC if stale. + existing_pid = _read_pid() + if existing_pid and _pid_alive(existing_pid): + raise RuntimeError( + f"Another iron-proxy start appears to be in progress " + f"(pidfile {pidfile} -> pid {existing_pid}). " + f"Run `hermes egress stop` if that proxy is stuck." + ) + # Stale — unlink and retry. + try: + pidfile.unlink() + except FileNotFoundError: + pass + fd = os.open(str(pidfile), open_flags, 0o600) + except OSError as exc: + # ELOOP from a planted symlink at the pidfile path. + raise RuntimeError( + f"Refusing to write pidfile {pidfile}: {exc}. " + "Remove that path manually and retry." + ) from exc + + try: + # Ownership check — same st_uid pattern the log file uses. + try: + st = os.fstat(fd) + if hasattr(os, "getuid") and st.st_uid != os.getuid(): + raise RuntimeError( + f"pidfile {pidfile} has unexpected owner uid={st.st_uid}" + ) + except AttributeError: + pass # Windows + os.write(fd, str(pid).encode("utf-8")) + finally: + os.close(fd) + + # Persist the nonce next to the pidfile (sibling, 0o600). + # ``stop_proxy`` in a separate CLI invocation can read this and use + # it to confirm the pid still refers to our binary even though the + # module-global ``_proxy_nonce`` is fresh in the new process. + if _proxy_nonce: + noncefile = pidfile.with_suffix(".nonce") + nfd = -1 + try: + nopen = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + nopen |= os.O_NOFOLLOW + nfd = os.open(str(noncefile), nopen, 0o600) + os.write(nfd, _proxy_nonce.encode("utf-8")) + except OSError: + # Best-effort. Without the nonce file we fall back to + # argv0-basename matching, which is what we did before. + pass + finally: + if nfd >= 0: + try: + os.close(nfd) + except OSError: + pass + + +def _kill_and_wait(proc: "subprocess.Popen", *, grace_seconds: int = 2) -> None: + """Best-effort SIGTERM → wait → SIGKILL for a child we own.""" + try: + proc.terminate() + except OSError: + return + try: + proc.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + try: + proc.kill() + except OSError: + pass + try: + proc.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + pass + + +def _build_proxy_subprocess_env( + *, + extra_env: Optional[Dict[str, str]] = None, + refresh_from_bitwarden: bool = False, + bitwarden_config: Optional[Dict] = None, +) -> Dict[str, str]: + """Construct the minimal env for the iron-proxy subprocess. + + Allowlists infrastructure vars (PATH, HOME, locale) plus the env vars + named in ``load_mappings()`` (the real upstream secrets the proxy + needs to do the swap). Everything else is stripped — see + ``_PROXY_SUBPROCESS_ENV_STRIP`` for proxy chain protection. + + When ``refresh_from_bitwarden=True`` AND ``bitwarden_config`` is + populated, fetches upstream secrets via the BSM SDK at startup and + merges them in. This is what delivers the rotation guarantee + promised by ``credential_source: bitwarden`` — without it, rotating + a key in the Bitwarden web app doesn't reach the proxy. + """ + + env: Dict[str, str] = {} + parent = os.environ + for name in _PROXY_SUBPROCESS_ENV_ALLOWLIST: + if name in parent: + env[name] = parent[name] + + # The proxy reads the real upstream secrets from its OWN env, indexed + # by ``m.real_env_name`` in the YAML config's ``secrets.source.var`` + # field. Forward those — but only those. For alias providers + # (GEMINI_API_KEY / GOOGLE_API_KEY), the rule is keyed on the canonical + # name; when only the alias is set in the host env, mirror its value + # into the canonical name so the swap still has a real secret. + alias_sources: Dict[str, Tuple[str, ...]] = {} + needed = set() + for m in load_mappings(): + needed.add(m.real_env_name) + if m.alias_env_names: + alias_sources[m.real_env_name] = tuple(m.alias_env_names) + for name in needed: + if name in parent: + env[name] = parent[name] + else: + for alias in alias_sources.get(name, ()): + if parent.get(alias): + env[name] = parent[alias] + break + + # Optional Bitwarden refresh path. Pulled lazily so the proxy module + # doesn't hard-depend on the bitwarden module being importable in + # every install. + if refresh_from_bitwarden and bitwarden_config: + try: + from agent.secret_sources import bitwarden as bw + access_token_name = bitwarden_config.get( + "access_token_env", "BWS_ACCESS_TOKEN" + ) + access_token = parent.get(access_token_name, "").strip() + project_id = bitwarden_config.get("project_id", "") + if access_token and project_id: + secrets, warnings = bw.fetch_bitwarden_secrets( + access_token=access_token, + project_id=project_id, + cache_ttl_seconds=0, + use_cache=False, + ) + # Only inject env names we have a mapping for — extra + # secrets in the BW project shouldn't leak into the proxy + # process unless they're going to be used by the swap. + missing = sorted(needed - set(secrets)) + for n in needed: + if n in secrets: + env[n] = secrets[n] + if missing: + # stephenschoettler #1: don't silently keep stale + # host-env values when BWS mode was explicitly + # selected. An operator on credential_source=bitwarden + # picked it specifically to get rotation; falling back + # to parent env reintroduces the bug class the mode + # is supposed to defeat. ``allow_env_fallback`` is the + # documented, deliberate opt-out — honor it here exactly + # as the empty-token branch below does (the error + # message tells operators to set it, so it must work). + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + f"Bitwarden refresh did not return secrets for " + f"{missing}. Either add the secrets to your BWS " + f"project, switch to credential_source: env via " + f"`hermes egress setup --no-bitwarden`, or set " + f"`proxy.allow_env_fallback: true` in config.yaml " + f"to opt into the legacy host-env fallback." + ) + logger.warning( + "Bitwarden refresh did not return secrets for %s — " + "falling back to host env for those names " + "(allow_env_fallback=true).", + missing, + ) + # bws warnings are non-secret status messages (e.g. "no + # project found", "rate limited"), but the taint analyzer + # can't tell that — log the count and let the operator + # rerun under verbose if they need detail. + if warnings: + logger.warning( + "Bitwarden refresh produced %d warning(s); " + "run `hermes secrets bitwarden status` for detail.", + len(warnings), + ) + else: + # NOTE: deliberately do not interpolate access_token_name + # in the log message — CodeQL's taint analyzer treats + # bitwarden_config values as secret-tainted (it can't + # distinguish the env-var NAME from the env-var VALUE). + # The name is non-secret but logging it just trips the + # check for no real benefit. + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + "credential_source=bitwarden but the access-token " + "env or project_id is empty. Either set both, " + "switch to credential_source: env, or set " + "`proxy.allow_env_fallback: true` to opt into " + "the legacy fallback behaviour." + ) + logger.warning( + "credential_source=bitwarden but access-token env or " + "project_id is empty — proxy will fall back to parent env " + "(allow_env_fallback=true).", + ) + except (ImportError,) as exc: + # The BWS module or one of its runtime deps isn't importable. + # Mirror the sibling branches: if allow_env_fallback isn't + # explicitly enabled, fail closed — credential_source=bitwarden + # with a unavailable module should not silently degrade to host + # env. A wizard-time check can't catch a dependency that goes + # missing between setup and a later restart. + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + "Bitwarden refresh module unavailable at proxy start " + "(credential_source=bitwarden with " + "proxy.allow_env_fallback: false). Either fix the " + "import, switch to credential_source: env, or set " + "`proxy.allow_env_fallback: true` to opt into the " + "legacy fallback behaviour." + ) from exc + logger.warning( + "Bitwarden refresh module unavailable at proxy start, " + "falling back to parent env (allow_env_fallback=true): %s", + exc, + ) + + # Caller-supplied overrides win. This is intentionally last so the + # wizard can inject ad-hoc test secrets without recomputing the BW + # path. + if extra_env: + env.update(extra_env) + + # Strip proxy-recursion-risk vars regardless of how they got in. + for name in _PROXY_SUBPROCESS_ENV_STRIP: + env.pop(name, None) + + env.setdefault("NO_COLOR", "1") + return env + + +def stop_proxy() -> bool: + """Stop the managed iron-proxy. Returns True if it was running.""" + + global _proxy_nonce + + def _cleanup_state_files() -> None: + """Best-effort cleanup of pidfile + persisted nonce.""" + _pidfile().unlink(missing_ok=True) + try: + _persisted_nonce_path().unlink() + except FileNotFoundError: + pass + except OSError: + pass + + pid = _read_pid() + if not pid or not _pid_alive(pid): + _cleanup_state_files() + _proxy_nonce = None + return False + + # Capture starttime BEFORE signalling so we can compare after the + # grace window — if the pid got recycled mid-wait, the starttime + # changes and we abort the SIGKILL. + starttime_before = _pid_proc_starttime(pid) + + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + _cleanup_state_files() + _proxy_nonce = None + return False + + # Wait up to 5s for graceful exit, then SIGKILL. + deadline = time.time() + 5.0 + while time.time() < deadline: + if not _pid_alive(pid): + break + time.sleep(0.1) + else: + # Verify the pid hasn't been recycled before delivering SIGKILL. + # Two checks: + # 1. /proc//stat starttime is unchanged (Linux) + # 2. _pid_alive() still says it's an iron-proxy process + starttime_after = _pid_proc_starttime(pid) + recycled = ( + starttime_before is not None + and starttime_after is not None + and starttime_before != starttime_after + ) or not _pid_alive(pid) + if recycled: + logger.warning( + "iron-proxy pid=%s appears recycled before SIGKILL; " + "not killing.", pid, + ) + else: + try: + os.kill(pid, _KILL_SIGNAL) + except ProcessLookupError: + pass + + _cleanup_state_files() + _proxy_nonce = None + logger.info("Stopped iron-proxy pid=%s", pid) + return True + + +def get_status() -> ProxyStatus: + """Snapshot the current proxy state — does NOT start anything. + + Crucially, this is called per Docker-container-create when egress + enforcement is on. It must not have side-effects (no mkdir, no + binary version subprocess that takes 30s on a hung binary). The + state dir is read-only here. + """ + + status = ProxyStatus() + listen_hp = _read_http_listen_from_config() + if listen_hp is not None: + probe_host, status.tunnel_port = listen_hp + else: + probe_host = "127.0.0.1" + status.tunnel_port = _DEFAULT_TUNNEL_PORT + + binary = find_iron_proxy(install_if_missing=False) + if binary: + status.binary_path = binary + # Cached — see iron_proxy_version(). First call still costs one + # subprocess; subsequent calls in the same process are dict + # lookups. + status.binary_version = iron_proxy_version(binary) + + state = _proxy_state_dir_ro() + cfg = state / "proxy.yaml" + ca = state / "ca.crt" + if cfg.exists(): + status.config_path = cfg + if ca.exists(): + status.ca_cert_path = ca + + pid = _read_pid() + if pid and _pid_alive(pid): + status.pid = pid + # Probe the configured bind host — on Linux that's the docker + # bridge gateway, where a loopback connect would report a healthy + # daemon as "not listening". + status.listening = _port_listening(probe_host, status.tunnel_port) + + return status + + +def _read_tunnel_port_from_config() -> Optional[int]: + listen = _read_http_listen_from_config() + if listen is None: + return None + return listen[1] + + +def _read_http_listen_from_config() -> Optional[Tuple[str, int]]: + """Return ``(host, port)`` of the configured sandbox-facing listener. + + Reads ``proxy.tunnel_listen`` — the CONNECT/MITM listener sandboxes + hit via ``HTTPS_PROXY`` — falling back to ``proxy.http_listen`` for + configs written before the tunnel/http listener-role split. + + The bind host matters for liveness probes: on Linux the daemon binds + the docker bridge gateway (e.g. ``172.17.0.1``), where a loopback + connect would report "not listening" for a perfectly healthy daemon. + """ + + cfg = _proxy_state_dir_ro() / "proxy.yaml" + if not cfg.exists(): + return None + try: + import yaml + except ImportError: + return None + try: + data = yaml.safe_load(cfg.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + proxy_block = (data or {}).get("proxy") or {} + # The CLI/Docker side calls this "the tunnel port" because that's how + # sandboxes use it (HTTPS_PROXY) — on the iron-proxy side it's the + # tunnel_listen (CONNECT + MITM). http_listen is the plain-HTTP + # forward listener on tunnel_port+1. + listen = proxy_block.get("tunnel_listen") or proxy_block.get("http_listen") or "" + if not isinstance(listen, str) or ":" not in listen: + return None + host, _, port_s = listen.rpartition(":") + try: + port = int(port_s) + except ValueError: + return None + return (host or "127.0.0.1", port) + + +def _port_listening(host: str, port: int) -> bool: + """Cheap TCP connect probe — True iff something accepts on host:port.""" + + import socket + + try: + with socket.create_connection((host, port), timeout=0.5): + return True + except OSError: + return False + + +def _tail_log(path: Path, *, lines: int = 20) -> str: + if not path.exists(): + return "(no log file)" + try: + data = path.read_bytes()[-8192:] + return "\n".join(data.decode("utf-8", errors="replace").splitlines()[-lines:]) + except OSError as exc: + return f"(could not read log: {exc})" + + +# --------------------------------------------------------------------------- +# Test hook +# --------------------------------------------------------------------------- + + +def _reset_for_tests() -> None: + """Clear module-level caches so tests get a fresh start. + + This module owns two mutable globals that need reset between tests: + - ``_VERSION_CACHE`` — subprocess output cache keyed by binary path. + - ``_proxy_nonce`` — the strong-proof token written by ``start_proxy`` + and read by ``_pid_alive`` to defeat PID recycling. + + Today the repo's tests run each file in its own subprocess (per + AGENTS.md) so leakage is bounded, but any in-process caller + (notebooks, ad-hoc scripts, ``pytest -p no:xdist``) would otherwise + see whichever values were probed first regardless of subsequent + ``install_iron_proxy(force=True)`` or ``start_proxy`` calls. + """ + + global _proxy_nonce + _VERSION_CACHE.clear() + _proxy_nonce = None + + +# Make a small set of symbols available without underscored access. +__all__ = [ + "ProxyStatus", + "TokenMapping", + "build_proxy_config", + "discover_provider_mappings", + "discover_uncovered_providers", + "ensure_audit_log", + "ensure_ca_cert", + "ensure_management_token", + "find_iron_proxy", + "get_status", + "install_iron_proxy", + "iron_proxy_version", + "load_mappings", + "merge_mappings", + "mint_proxy_token", + "reload_proxy", + "start_proxy", + "stop_proxy", + "write_mappings", + "write_proxy_config", +] diff --git a/agent/reasoning_timeouts.py b/agent/reasoning_timeouts.py index 13df836b110..8af5ab799f4 100644 --- a/agent/reasoning_timeouts.py +++ b/agent/reasoning_timeouts.py @@ -102,6 +102,7 @@ _REASONING_STALE_TIMEOUT_FLOORS: tuple[tuple[str, int], ...] = ( # ``claude-opus-4`` so non-thinking Claude 3.x or future # non-reasoning Claude variants don't match. ("claude-opus-4", 240), + ("claude-opus-5", 240), ("claude-sonnet-5", 180), ("claude-sonnet-4.5", 180), ("claude-sonnet-4.6", 180), diff --git a/agent/secret_sources/base.py b/agent/secret_sources/base.py index d4ead7d3f26..070051e046b 100644 --- a/agent/secret_sources/base.py +++ b/agent/secret_sources/base.py @@ -295,7 +295,7 @@ def run_secret_cli( list(argv), env=env, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=timeout, stdin=subprocess.DEVNULL, ) diff --git a/agent/secret_sources/bitwarden.py b/agent/secret_sources/bitwarden.py index 8b047880be1..2d993f396c2 100644 --- a/agent/secret_sources/bitwarden.py +++ b/agent/secret_sources/bitwarden.py @@ -200,7 +200,7 @@ def _platform_asset_name() -> str: res = subprocess.run( ["ldd", "--version"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=2, stdin=subprocess.DEVNULL, ) @@ -684,7 +684,7 @@ def _run_bws_list( cmd, env=env, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=_BWS_RUN_TIMEOUT, stdin=subprocess.DEVNULL, ) diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index c1cec81df33..80453bc3713 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -464,7 +464,7 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]: input=stdin_json, capture_output=True, timeout=spec.timeout, - text=True, + text=True, encoding='utf-8', errors='replace', shell=False, **_popen_kwargs, ) @@ -632,7 +632,7 @@ def allowlist_path() -> Path: def load_allowlist() -> Dict[str, Any]: """Return the parsed allowlist, or an empty skeleton if absent.""" try: - raw = json.loads(allowlist_path().read_text()) + raw = json.loads(allowlist_path().read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError): return {"approvals": []} if not isinstance(raw, dict): diff --git a/agent/skill_commands.py b/agent/skill_commands.py index 11d425b91a5..294ca2b1754 100644 --- a/agent/skill_commands.py +++ b/agent/skill_commands.py @@ -54,6 +54,21 @@ _BUNDLE_MARKER = " skill bundle," _BUNDLE_USER_INSTRUCTION = "\nUser instruction: " _BUNDLE_FIRST_SKILL_BLOCK = "\n\n[Loaded as part of the " +# The skill name sits in the first quoted span of the activation note, for both +# the single-skill and the bundle header ("work" / "/clean /work"). +_SKILL_NAME_RE = re.compile(re.escape(_SKILL_INVOCATION_PREFIX) + r'"([^"]*)"') + +# SQL LIKE pattern matching a skill-expanded turn, for listing queries that +# have to recognize scaffolding before the row reaches Python. The prefix +# contains no LIKE wildcards (`%`, `_`), so it needs no ESCAPE clause. +SKILL_SCAFFOLD_SQL_LIKE = _SKILL_INVOCATION_PREFIX + "%" + +# Marks where a preview query joined the head and tail of a long scaffolded +# message. ``describe_skill_invocation`` may hand back a span that runs across +# the joint (a bundle instruction cut off by the head window); callers cut the +# description there rather than show the skill body on the far side. +SKILL_EXCERPT_JOINT = "\x1e" + def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]: """Recover the user's instruction from a slash-skill-expanded turn. @@ -82,6 +97,41 @@ def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]: return None +def describe_skill_invocation(content: Any) -> Optional[str]: + """Render a slash-skill-expanded turn the way the user typed it. + + The expanded message embeds the whole skill body, so any surface that + summarizes a user turn from its raw content — session titles, sidebar + previews, the ``/rewind`` picker — otherwise shows the skill's own prose + as if the user had written it. That is how a skill's opening line ends up + as a session title. + + Returns ``"/work — fix the title leak"``, or ``"/work"`` for a bare + invocation, or ``None`` when *content* is not skill scaffolding (the + caller should then summarize it as an ordinary message). + """ + if not isinstance(content, str) or not content.startswith(_SKILL_INVOCATION_PREFIX): + return None + + match = _SKILL_NAME_RE.match(content) + name = (match.group(1) if match else "").strip() + # Bundle headers already carry their typed "/a /b" keys; a single skill is + # a bare name. + label = name if name.startswith("/") else f"/{name}" + + instruction = extract_user_instruction_from_skill_message(content) + if instruction and instruction is not content: + # An excerpted message (head + tail, joined by SKILL_EXCERPT_JOINT) can + # put the joint inside the matched span — keep only the side the + # instruction marker was found on. + instruction = instruction.split(SKILL_EXCERPT_JOINT)[0] + instruction = " ".join(instruction.split()) + if instruction: + return f"{label} — {instruction}" if name else instruction + + return label if name else None + + def _extract_single_skill_user_instruction(message: str) -> Optional[str]: # Single-skill format appends the user instruction after the skill body, so # the last occurrence is the user-provided one; the body may quote this text. @@ -453,8 +503,8 @@ def reload_skills() -> Dict[str, Any]: } ``description`` is the skill's full SKILL.md frontmatter - ``description:`` field — the same string the system prompt renders - as `` - name: description`` for pre-existing skills. + ``description:`` field. Note: the system prompt skill index + truncates this to the first 57 chars; see ``extract_skill_description``. """ # Snapshot pre-reload state (name -> description) from the current # slash-command cache. Using dicts lets the post-rescan diff carry diff --git a/agent/skill_preprocessing.py b/agent/skill_preprocessing.py index bd0386d5805..19c6eeb80fb 100644 --- a/agent/skill_preprocessing.py +++ b/agent/skill_preprocessing.py @@ -74,7 +74,7 @@ def run_inline_shell(command: str, cwd: Path | None, timeout: int) -> str: ["bash", "-c", command], cwd=str(cwd) if cwd else None, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=max(1, int(timeout)), check=False, stdin=subprocess.DEVNULL, diff --git a/agent/skill_utils.py b/agent/skill_utils.py index f96238b9bd9..df0f933317f 100644 --- a/agent/skill_utils.py +++ b/agent/skill_utils.py @@ -779,18 +779,31 @@ def resolve_skill_config_values( # ── Description extraction ──────────────────────────────────────────────── +SKILL_PROMPT_DESC_LIMIT = 60 + + +def _normalize_skill_description(frontmatter: Dict[str, Any]) -> str: + """Normalize a skill's description field for comparison/truncation.""" + raw_desc = frontmatter.get("description", "") + return str(raw_desc).strip().strip("'\"") if raw_desc else "" + def extract_skill_description(frontmatter: Dict[str, Any]) -> str: - """Extract a truncated description from parsed frontmatter.""" - raw_desc = frontmatter.get("description", "") - if not raw_desc: + """Extract a system-prompt-length description from parsed frontmatter.""" + desc = _normalize_skill_description(frontmatter) + if not desc: return "" - desc = str(raw_desc).strip().strip("'\"") - if len(desc) > 60: - return desc[:57] + "..." + if len(desc) > SKILL_PROMPT_DESC_LIMIT: + return desc[:SKILL_PROMPT_DESC_LIMIT - 3] + "..." return desc +def is_skill_description_truncated_for_prompt(frontmatter: Dict[str, Any]) -> bool: + """True when the description will be truncated in the system prompt skill index.""" + desc = _normalize_skill_description(frontmatter) + return len(desc) > SKILL_PROMPT_DESC_LIMIT + + # ── File iteration ──────────────────────────────────────────────────────── diff --git a/agent/ssl_guard.py b/agent/ssl_guard.py index 557f8566c32..ac1b7841b2d 100644 --- a/agent/ssl_guard.py +++ b/agent/ssl_guard.py @@ -31,7 +31,8 @@ def _skip_ssl_guard_enabled() -> bool: def _repair_hint() -> str: return ( - "Repair: python -m pip install --force-reinstall certifi openai httpx\n" + "Repair: run `hermes doctor --fix` (auto-reinstalls certifi), or " + "manually: python -m pip install --force-reinstall certifi openai httpx\n" "If you configured a custom corporate CA bundle, fix or unset the " "broken CA bundle environment variable." ) diff --git a/agent/system_prompt.py b/agent/system_prompt.py index aab34cb799e..d92072d1ac1 100644 --- a/agent/system_prompt.py +++ b/agent/system_prompt.py @@ -12,9 +12,11 @@ Three tiers are joined with ``\\n\\n``: * ``stable`` — identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool guidance, computer-use guidance, nous subscription block, tool-use enforcement guidance + per-model operational guidance, skills prompt, - alibaba model-name workaround, environment hints, platform hints. + alibaba model-name workaround, environment hints, coding guidance, + platform hints. * ``context`` — caller-supplied ``system_message`` plus context files - (AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``. + (AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``, + plus the session's coding-workspace snapshot. * ``volatile`` — memory snapshot, USER.md profile, external memory provider block, timestamp/session/model/provider line. @@ -145,14 +147,14 @@ def _tui_embedded_pane_clarifier(hint: str) -> str: def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]: - """Assemble the system prompt as three ordered parts. + """Assemble the system prompt as three ordered cache tiers. Returns a dict with three keys: - * ``stable`` — identity, tool guidance, skills prompt, - environment hints, platform hints, model-family operational - guidance. - * ``context`` — context files (AGENTS.md, .cursorrules, etc.) - and caller-supplied system_message. + * ``stable`` — the cross-session-stable prefix, through the coding + operating brief when a workspace snapshot follows. + * ``context`` — the workspace snapshot followed by the remaining + session-stable guidance, context files, and caller-supplied + system_message. * ``volatile`` — memory snapshot, user profile, external memory provider block, timestamp line. @@ -345,25 +347,35 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) stable_parts.append(_env_hints) # Coding posture (base Hermes, any interactive coding surface in a code - # workspace — see agent/coding_context.py). The operating brief + the live - # git/workspace snapshot are built once here and cached for the session; - # the snapshot is never re-probed per turn (that would break the prompt - # cache), so the brief tells the model to re-check git before relying on it. + # workspace — see agent/coding_context.py). Keep the operating brief in + # the cross-session-stable prefix, while placing the live git/workspace + # snapshot behind its own cache boundary. The post-snapshot blocks must + # stay in their historical position after the workspace snapshot. + coding_workspace_parts: List[str] = [] + coding_trailing_parts: List[str] = [] if agent.valid_tool_names: try: - from agent.coding_context import coding_system_blocks + from agent.coding_context import coding_system_prompt_parts - stable_parts.extend( - coding_system_blocks( - platform=agent.platform, - cwd=resolve_context_cwd(), - model=agent.model, - ) + coding_prefix_parts, coding_workspace_parts, coding_trailing_parts = coding_system_prompt_parts( + platform=agent.platform, + cwd=resolve_context_cwd(), + model=agent.model, ) + stable_parts.extend(coding_prefix_parts) except Exception: # Coding-context probing must never block prompt build. pass + # Guidance assembled after the coding posture historically followed the + # workspace snapshot. With no snapshot, the coding tail instead remains + # directly after the coding prefix in the cacheable prefix. + if coding_workspace_parts: + post_workspace_parts: List[str] = [] + else: + stable_parts.extend(coding_trailing_parts) + post_workspace_parts = stable_parts + # Local Python toolchain probe — names python/pip/uv/PEP-668 state when # something is non-default so the model can pick the right install # strategy without discovering by failure. Emits a single line; emits @@ -376,7 +388,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) from tools.env_probe import get_environment_probe_line _probe_line = get_environment_probe_line() if _probe_line: - stable_parts.append(_probe_line) + post_workspace_parts.append(_probe_line) except Exception: # Probe failure must never block prompt build. pass @@ -394,7 +406,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) except Exception: active_profile = "default" if active_profile == "default": - stable_parts.append( + post_workspace_parts.append( "Active Hermes profile: default. Other profiles (if any) live " "under " + str(get_hermes_home()) + "/profiles//. Each profile has its own " "skills/, plugins/, cron/, and memories/ that affect a different " @@ -403,7 +415,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) "you to." ) else: - stable_parts.append( + post_workspace_parts.append( f"Active Hermes profile: {active_profile}. This session reads " f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default " f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, " @@ -449,11 +461,16 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) if platform_key == "tui" and _effective_hint: _effective_hint = _tui_embedded_pane_clarifier(_effective_hint) if _effective_hint: - stable_parts.append(_effective_hint) + post_workspace_parts.append(_effective_hint) # ── Context tier (cwd-dependent, may change between sessions) ─ context_parts: List[str] = [] + if coding_workspace_parts: + context_parts.extend(coding_workspace_parts) + context_parts.extend(coding_trailing_parts) + context_parts.extend(post_workspace_parts) + # Note: ephemeral_system_prompt is NOT included here. It's injected at # API-call time only so it stays out of the cached/stored system prompt. if system_message is not None: @@ -515,6 +532,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) timestamp_line += f"\nModel: {agent.model}" if agent.provider: timestamp_line += f"\nProvider: {agent.provider}" + if agent.platform: + timestamp_line += f"\nPlatform: {agent.platform}" volatile_parts.append(timestamp_line) return { @@ -541,6 +560,7 @@ def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str """ parts = build_system_prompt_parts(agent, system_message=system_message) joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p) + agent._cached_system_prompt_static = parts["stable"] # Surface context-file truncation warnings through the normal agent status # channel so gateway/CLI users see them in chat instead of only in logs. @@ -557,6 +577,7 @@ def invalidate_system_prompt(agent: Any) -> None: so the rebuilt prompt captures any writes from this session. """ agent._cached_system_prompt = None + agent._cached_system_prompt_static = None if agent._memory_store: agent._memory_store.load_from_disk() diff --git a/agent/title_generator.py b/agent/title_generator.py index 7469a665bfc..bb44da276ec 100644 --- a/agent/title_generator.py +++ b/agent/title_generator.py @@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool: return True +def _summarize_user_message(user_message: str) -> str: + """Collapse a slash-skill-expanded turn back to what the user typed. + + A ``/skill`` invocation expands into a message that embeds the whole skill + body, so feeding it to the titler verbatim titles the session after the + *skill's* prose — "Kick off a task in a fresh isolated git worktree" — not + after the user's request. Reuse the canonical scaffolding parser so the + model sees ``/work — fix the title leak`` instead. + """ + if not user_message: + return "" + try: + from agent.skill_commands import describe_skill_invocation + + described = describe_skill_invocation(user_message) + except Exception: + logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True) + return user_message + return described if described is not None else user_message + + def generate_title( user_message: str, assistant_response: str, @@ -110,7 +131,7 @@ def generate_title( logger.debug("Title runtime validator raised; proceeding", exc_info=True) # Truncate long messages to keep the request small - user_snippet = user_message[:500] if user_message else "" + user_snippet = _summarize_user_message(user_message)[:500] assistant_snippet = assistant_response[:500] if assistant_response else "" language = _title_language() @@ -143,6 +164,11 @@ def generate_title( title = title.strip('"\'') if title.lower().startswith("title:"): title = title[6:].strip() + # A title is one line. A model that ignores "return ONLY the title" and + # answers the prompt instead (a shell transcript, a bulleted plan) would + # otherwise be stored verbatim and truncated mid-command. Keep the first + # non-empty line — the closest thing to a title in that response. + title = next((line.strip() for line in title.splitlines() if line.strip()), "") # Enforce reasonable length if len(title) > 80: title = title[:77] + "..." diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 16b9f0f4dd5..1db855573f0 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -1204,7 +1204,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") agent._current_tool = None - agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)") + _status_suffix = " (error)" if is_error else "" + agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}") if not blocked and agent.tool_complete_callback: try: @@ -1826,7 +1827,8 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe logging.debug(f"Tool progress callback error: {cb_err}") agent._current_tool = None - agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)") + _status_suffix = " (error)" if _is_error_result else "" + agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}") if agent.verbose_logging: logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") diff --git a/agent/trace_upload.py b/agent/trace_upload.py index f65547440c7..404d9be70b1 100644 --- a/agent/trace_upload.py +++ b/agent/trace_upload.py @@ -162,7 +162,7 @@ def build_trace_jsonl( if cwd: r = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, text=True, timeout=3, cwd=cwd, + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3, cwd=cwd, ) if r.returncode == 0: git_branch = r.stdout.strip() diff --git a/agent/transports/codex.py b/agent/transports/codex.py index 5855fcfe9c5..15dd3409e3f 100644 --- a/agent/transports/codex.py +++ b/agent/transports/codex.py @@ -7,6 +7,7 @@ streaming, or the _run_codex_stream() call path. import hashlib import json +import re from typing import Any, Dict, List, Optional from agent.transports.base import ProviderTransport @@ -27,6 +28,49 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]: return f"pck_{digest}" +_EXTENDED_PROMPT_CACHE_MODELS = ( + "gpt-5.5-pro", + "gpt-5.5", + "gpt-5.4", + "gpt-5.2", + "gpt-5.1-codex-max", + "gpt-5.1-codex-mini", + "gpt-5.1-chat-latest", + "gpt-5.1-codex", + "gpt-5.1", + "gpt-5-codex", + "gpt-5", + "gpt-4.1", +) +_EXTENDED_PROMPT_CACHE_MODEL_RE = re.compile( + rf"(?:^|[./:])(?:{'|'.join(re.escape(name) for name in _EXTENDED_PROMPT_CACHE_MODELS)})" + r"(?:-\d{4}-\d{2}-\d{2})?$" +) + + +def _default_prompt_cache_retention_for_request( + model: str, + base_url: Any, +) -> Optional[str]: + """Return ``24h`` for supported models on Amazon Bedrock Mantle.""" + from utils import base_url_hostname + + hostname_parts = base_url_hostname(str(base_url or "")).split(".") + is_bedrock_mantle = ( + len(hostname_parts) == 4 + and hostname_parts[0] == "bedrock-mantle" + and bool(hostname_parts[1]) + and hostname_parts[2:] == ["api", "aws"] + ) + if not is_bedrock_mantle: + return None + + normalized = str(model or "").strip().lower().replace("_", "-") + if _EXTENDED_PROMPT_CACHE_MODEL_RE.search(normalized): + return "24h" + return None + + def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]: """Content-address the prompt cache key from the static request prefix. @@ -284,6 +328,13 @@ class ResponsesApiTransport(ProviderTransport): if not is_github_responses and not is_xai_responses and cache_key: kwargs["prompt_cache_key"] = cache_key + cache_retention = _default_prompt_cache_retention_for_request( + model, + params.get("base_url"), + ) + if cache_retention: + kwargs.setdefault("prompt_cache_retention", cache_retention) + if reasoning_enabled and is_xai_responses: from agent.model_metadata import grok_supports_reasoning_effort diff --git a/agent/transports/codex_app_server.py b/agent/transports/codex_app_server.py index 7f5831f2a3e..c23ff836ed8 100644 --- a/agent/transports/codex_app_server.py +++ b/agent/transports/codex_app_server.py @@ -394,7 +394,7 @@ def check_codex_binary( proc = subprocess.run( [codex_bin, "--version"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=10, stdin=subprocess.DEVNULL, ) diff --git a/agent/turn_context.py b/agent/turn_context.py index 498f68771c1..5a006894887 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -34,7 +34,9 @@ from typing import Any, Dict, List, Mapping, Optional from agent.conversation_compression import ( IDLE_COMPACTION_STATUS_TEMPLATE, PREFLIGHT_COMPRESSION_STATUS_TEMPLATE, + compression_skipped_due_to_lock, conversation_history_after_compression, + recover_rotated_compression_session, ) from agent.context_engine import automatic_compaction_status_message from agent.iteration_budget import IterationBudget @@ -352,6 +354,13 @@ def build_turn_context( # Guard stdio against OSError from broken pipes (systemd/headless/daemon). install_safe_stdio() + # Recover a session rotated by another path before binding log/turn ids or + # copying client-supplied history. Everything in this turn must consistently + # belong to the canonical child, including observability metadata. + recovered_history = recover_rotated_compression_session(agent) + if recovered_history is not None: + conversation_history = recovered_history + # NOTE: the DB session row is created later, AFTER the system prompt is # restored/built (see _ensure_db_session() below the system-prompt block). # Creating it here — before _cached_system_prompt is populated — inserts a @@ -710,6 +719,8 @@ def build_turn_context( # issue #27405 (a few very large messages slipping past the count gate). _preflight_compressed = False _preflight_compression_blocked = False + agent._turn_received_provider_response = False + agent._turn_preflight_display_snapshot = None if agent.compression_enabled and _should_run_preflight_estimate( messages, agent.context_compressor.protect_first_n, @@ -722,6 +733,21 @@ def build_turn_context( tools=agent.tools or None, ) _compressor = agent.context_compressor + # getattr guard: minimal compressor doubles (SimpleNamespace in the + # engine-preflight tests) and plugin context engines lack this + # ContextCompressor-only method — absence means no snapshot, and the + # finalizer's rollback stays disarmed for the turn (display-only). + _snapshot_fn = getattr( + _compressor, "snapshot_preflight_display_tokens", None + ) + if callable(_snapshot_fn): + _snapshot_val = _snapshot_fn() + # Type pin: MagicMock compressors return truthy Mock objects — + # only a real int snapshot may arm the interrupted-turn rollback. + if isinstance(_snapshot_val, int) and not isinstance( + _snapshot_val, bool + ): + agent._turn_preflight_display_snapshot = _snapshot_val _defer_preflight = getattr( _compressor, "should_defer_preflight_to_real_usage", @@ -838,10 +864,29 @@ def build_turn_context( for _pass in range(_max_preflight_passes): _orig_len = len(messages) _orig_tokens = _preflight_tokens + _preflight_input = messages messages, active_system_prompt = agent._compress_context( messages, system_message, approx_tokens=_preflight_tokens, task_id=effective_task_id, ) + if ( + messages is _preflight_input + and compression_skipped_due_to_lock(agent) + ): + # #69870 lock-skip: another path holds this session's + # compression lock, so the pass no-oped. That is a + # temporary DEFER, not proof the transcript cannot + # compress — do NOT arm the insufficient-progress + # blocker (the loop's error handlers must keep their + # provider-proven retry budget) and stop preflight + # passes for this turn; the lock winner is shrinking + # the same session concurrently. + logger.info( + "Preflight compression deferred: compression lock " + "held by another path (session %s)", + agent.session_id or "none", + ) + break # Re-estimate now so size-only compression (same row count, # lower token count — e.g. summarising tool outputs) is # recognised as progress instead of being misread as diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 7bf4ee4bbe5..0f636c1dd24 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -201,6 +201,36 @@ def finalize_turn( ) ) + # Preflight can seed the display count before the provider receives the + # request. Roll that estimate back only when an interrupt wins the race + # before any successful provider response. Compaction state remains owned + # by the real-usage/post-compaction path, including its ``-1`` sentinel. + # Guard rules (test-double density on this path is high): + # - snapshot is type-pinned to a real int — MagicMock agents auto-create + # truthy Mock attributes that must never arm the rollback; + # - the received-response flag is pinned to ``is not True`` — its real + # domain is True/False, and only a literal True means a provider + # response completed; + # - the compressor method gets a getattr+callable guard — SimpleNamespace + # compressor doubles and plugin context engines lack it. + _preflight_snapshot = getattr( + agent, "_turn_preflight_display_snapshot", None + ) + if ( + interrupted is True + and isinstance(_preflight_snapshot, int) + and not isinstance(_preflight_snapshot, bool) + and getattr(agent, "_turn_received_provider_response", False) is not True + and getattr(agent, "context_compressor", None) is not None + ): + _rollback_fn = getattr( + agent.context_compressor, + "rollback_interrupted_preflight_display_tokens", + None, + ) + if callable(_rollback_fn): + _rollback_fn(_preflight_snapshot) + # Post-loop cleanup must never lose the response. Trajectory save, # resource teardown, and session persistence all touch fallible # surfaces — file I/O / JSON serialization (_save_trajectory), remote @@ -495,6 +525,34 @@ def finalize_turn( except Exception as exc: logger.warning("post_llm_call hook failed: %s", exc) + # Context engine observation hook: notify the active engine that this + # turn has finished, with the finalized transcript. Complements the + # per-request select_context() hook (selection before the request; + # observation after the turn). No-op default, fail-open. + try: + from agent.conversation_loop import _notify_context_engine_turn_complete + # Forward the turn's canonical usage when the host has it. The loop + # stashes the most recent API response's usage dict (the same + # canonical buckets fed to ``update_from_response``) on the agent as + # ``_last_turn_usage``. It is ``None`` on turns that never reached a + # provider response (early failure / interrupt), which is exactly the + # contract: real usage when available, ``None`` otherwise. + _turn_usage = getattr(agent, "_last_turn_usage", None) + _notify_context_engine_turn_complete( + agent, + messages, + usage=_turn_usage, + logger=logger, + turn_id=turn_id, + task_id=effective_task_id, + api_call_count=api_call_count, + interrupted=interrupted, + failed=failed, + turn_exit_reason=_turn_exit_reason, + ) + except Exception as exc: + logger.warning("on_turn_complete notification failed: %s", exc) + # Extract reasoning from the CURRENT turn only. Walk backwards # but stop at the user message that started this turn — anything # earlier is from a prior turn and must not leak into the reasoning @@ -627,4 +685,7 @@ def finalize_turn( except Exception as exc: logger.warning("on_session_end hook failed: %s", exc) + agent._turn_preflight_display_snapshot = None + agent._turn_received_provider_response = False + return result diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py index c3154378f5e..2c8d1f85efa 100644 --- a/agent/verification_evidence.py +++ b/agent/verification_evidence.py @@ -13,10 +13,11 @@ import shlex import sqlite3 import tempfile import threading +from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Optional +from typing import Any, Iterator, Optional from hermes_constants import get_hermes_home @@ -65,13 +66,38 @@ def _connect() -> sqlite3.Connection: path = _db_path() path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path) - apply_wal_with_fallback(conn, db_label="verification_evidence.db") - conn.execute("PRAGMA busy_timeout=5000") conn.row_factory = sqlite3.Row - _ensure_schema(conn) + try: + apply_wal_with_fallback(conn, db_label="verification_evidence.db") + conn.execute("PRAGMA busy_timeout=5000") + _ensure_schema(conn) + except Exception: + # A PRAGMA/DDL failure after a successful connect() must not leak the + # just-opened connection back to the caller. + conn.close() + raise return conn +@contextmanager +def _transaction() -> Iterator[sqlite3.Connection]: + """Open a connection, commit/rollback on exit, and ALWAYS close it. + + ``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the + transaction; they do not close the connection. Using ``with _connect()`` + alone therefore leaks a connection — and its WAL/SHM file descriptors — on + every call, deferring the close to the garbage collector, which over a + long-running process can exhaust ``RLIMIT_NOFILE`` (the cron-ledger sibling + of this bug was #69567 / PR #69594). + """ + conn = _connect() + try: + with conn: + yield conn + finally: + conn.close() + + def _ensure_schema(conn: sqlite3.Connection) -> None: conn.execute( """ @@ -454,7 +480,7 @@ def record_terminal_result( created_at = _utc_now() with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: cur = conn.execute( """ INSERT INTO verification_events( @@ -520,7 +546,7 @@ def mark_workspace_edited( edited_at = _utc_now() with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: row = conn.execute( """ SELECT changed_paths_json FROM verification_state @@ -570,7 +596,7 @@ def verification_status( sid = str(session_id or "default") root = str(facts.get("root") or Path(cwd or ".").resolve()) with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: state = conn.execute( """ SELECT last_event_id, last_edit_at, changed_paths_json diff --git a/apps/desktop/README.md b/apps/desktop/README.md index a1da176726d..706611acb1a 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -30,7 +30,7 @@ Already have the Hermes CLI? Just run: hermes desktop ``` -It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. On first launch Hermes walks you through picking a provider and model; nothing else to configure. +It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. If Desktop cannot find a usable runtime or saved remote connection, first launch lets you connect to an existing Hermes gateway or install Hermes locally. Local onboarding then walks you through choosing a provider and model. ### Prebuilt installers @@ -134,6 +134,19 @@ Desktop supports a managed local backend, explicit remote gateways, and Hermes Cloud connections. Remote and cloud modes use the same remote-capability path; authentication and discovery differ, not the renderer feature model. +When no usable local runtime or saved remote connection exists, the first-run +screen offers **Connect to existing Hermes** before starting the local installer. +Desktop probes the gateway to discover token or OAuth authentication, requires a +successful HTTP and WebSocket connection test, and saves the connection using +the same encrypted Desktop configuration used by Settings. A saved remote +connection bypasses this choice on later launches. The regular Desktop build +still includes the local-install option; this is a remote operating mode, not a +separate client-only application. + +In remote mode the gateway host is the execution boundary: agent tools, +terminal commands, and file operations run against the remote Hermes host, not +the computer displaying the Desktop UI. + Projects are the workspace abstraction. A project may own multiple folders, repositories, worktrees, and sessions; a bare new chat remains detached unless the user enters a project or configures a default project directory. Use the diff --git a/apps/desktop/e2e/correction-session-switch.spec.ts b/apps/desktop/e2e/correction-session-switch.spec.ts index dd32caa61e7..dc435b1d538 100644 --- a/apps/desktop/e2e/correction-session-switch.spec.ts +++ b/apps/desktop/e2e/correction-session-switch.spec.ts @@ -21,8 +21,16 @@ const INFERENCE_SWITCH_TRIGGER = 'E2E_INFERENCE_SWITCH_TRIGGER' const INFERENCE_PROMPT = `${INFERENCE_SWITCH_TRIGGER}: original inference prompt must remain singular.` const INFERENCE_CORRECTION = `${INFERENCE_SWITCH_TRIGGER}: correction sent while inference is live.` +// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the +// renderer's keep-alive visibility policy instead of relying on DOM order. +const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])' + +function activeSurface(page: Page) { + return page.locator(SURFACE).last() +} + async function send(page: Page, text: string): Promise { - const composer = page.locator('[contenteditable="true"]').first() + const composer = activeSurface(page).locator('[contenteditable="true"]').first() await composer.waitFor({ state: 'visible', timeout: 15_000 }) await composer.click() await composer.type(text, { delay: 5 }) @@ -30,8 +38,9 @@ async function send(page: Page, text: string): Promise { } async function steer(page: Page, text: string): Promise { - const composer = page.locator('[contenteditable="true"]').first() - const primary = page.locator('[data-slot="composer-root"] button[type="submit"]') + const surface = activeSurface(page) + const composer = surface.locator('[contenteditable="true"]').first() + const primary = surface.locator('[data-slot="composer-root"] button[type="submit"]') await composer.waitFor({ state: 'visible', timeout: 15_000 }) await composer.click() @@ -42,55 +51,80 @@ async function steer(page: Page, text: string): Promise { async function waitForTranscriptText(page: Page, text: string): Promise { await page.waitForFunction( - (expected: string) => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected), - text, + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const active = surfaces[surfaces.length - 1] + + return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected) + }, + [text, SURFACE] as [string, string], { timeout: 30_000 }, ) } async function textNodeOccurrences(page: Page, text: string): Promise { - return page.evaluate((expected: string) => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') - if (!viewport) return 0 + return page.evaluate( + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]') + if (!viewport) return 0 - const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT) - let count = 0 - while (walker.nextNode()) { - if (walker.currentNode.textContent?.includes(expected)) { - count += 1 + const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT) + let count = 0 + while (walker.nextNode()) { + if (walker.currentNode.textContent?.includes(expected)) { + count += 1 + } } - } - return count - }, text) + return count + }, + [text, SURFACE] as [string, string], + ) } async function transcriptTextOrder(page: Page): Promise { - return page.evaluate(() => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') + return page.evaluate((surfaceSelector: string) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]') if (!viewport) return [] return Array.from(viewport.querySelectorAll('[data-role="message"], [data-message-id]')) .map(message => message.textContent?.trim() ?? '') .filter(Boolean) - }) + }, SURFACE) } async function transcriptMessageOrder(page: Page): Promise { - return page.evaluate(() => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') + return page.evaluate((surfaceSelector: string) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]') if (!viewport) return [] - return Array.from(viewport.querySelectorAll('[data-role="user"], [data-role="assistant"]')) + return Array.from( + viewport.querySelectorAll('[data-role="user"], [data-role="assistant"], [data-role="system"]'), + ) .map(message => message.textContent?.trim() ?? '') .filter(Boolean) - }) + }, SURFACE) } +/** + * The sidebar "+" opens a NEW TAB beside the current chat rather than + * replacing it, so the prior session stays mounted in its own surface. Wait + * for the newly-mounted surface to show an empty transcript instead of waiting + * for the old text to disappear from the page (it never will). + */ async function openFreshDraft(page: Page, priorSessionText: string): Promise { await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click() await page.waitForFunction( - (priorText: string) => !(document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(priorText), - priorSessionText, + ([priorText, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const active = surfaces[surfaces.length - 1] + const transcript = active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '' + + return surfaces.length > 0 && !transcript.includes(priorText) + }, + [priorSessionText, SURFACE] as [string, string], { timeout: 15_000 }, ) } @@ -116,7 +150,12 @@ async function reopenInferenceSession(page: Page): Promise { } function relevantOrder(messages: string[]): string[] { - return messages.filter(message => message.includes(ORIGINAL_PROMPT) || message.includes(CORRECTION)) + return messages.flatMap(message => { + if (message.includes(ORIGINAL_PROMPT)) return [ORIGINAL_PROMPT] + if (message.includes(CORRECTION)) return [CORRECTION] + + return [] + }) } function steerTurnOrder(messages: string[]): string[] { diff --git a/apps/desktop/e2e/image-attachment-resume.spec.ts b/apps/desktop/e2e/image-attachment-resume.spec.ts new file mode 100644 index 00000000000..a4f8da68e39 --- /dev/null +++ b/apps/desktop/e2e/image-attachment-resume.spec.ts @@ -0,0 +1,197 @@ +/** + * Regression coverage for an attached image in a durable session. The gateway + * persists the turn, the builder exits, and desktop renders it from SessionDB + * for the first time — the "quit and relaunch" case, where the transcript used + * to come back as vision-enrichment prose instead of a thumbnail. + * + * The fixture pins `image_input_mode: native` because that is the majority + * routing path (any vision-capable model) and the one where a text-only + * persist override is silently dropped. The image also sits behind directory + * and file names containing spaces, mirroring the macOS composer's + * `~/Library/Application Support/...` staging path. + */ + +import * as fs from 'node:fs' +import * as path from 'node:path' + +import { + buildAppEnv, + createSandbox, + launchDesktop, + type Sandbox, + waitForAppReady, + writeEnvFile, + writeMockProviderConfig, +} from './fixtures' +import { type MockServer, startMockServer } from './mock-server' +import { RealSessionBuilder } from './real-session-builder' +import { type ElectronApplication, expect, type Page, test } from './test' + +// A seeded session has no generated title, so every label falls back to the +// session preview — the first 60 characters of the first user message. +const SESSION_TITLE = 'E2E attached image session' +const CAPTION = 'E2E attached image must survive a relaunch' +const IMAGE_DIR = 'Application Support/e2e shots' +const IMAGE_NAME = 'e2e capture.png' +const NATIVE_IMAGE_CONFIG = 'agent:\n image_input_mode: native' + +/** A 160x100 framed magenta block — small, but visible in the screenshots. */ +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAKAAAABkCAIAAACO1KzYAAAA30lEQVR42u3dwQ2AIBAAQTAWAx1iBXYI7diCuWhEMvP2dZsj+CL30hLr2oxAYARGYARGYARGYIERGIH53n7nozpOk5rQqIcNdkQjMAIjMNPeomP3N54V+5exwY5oBEZgBEZgBEZggREYgREYgREYgQVGYARGYARGYARGYIERGIERGIERGIEFRmAERmAERmAEFhiBERiBERiBERiBBUZgBEZgBEZgBBYYgREYgREYgRFYYCMQGIERmPSj94Njb9ligxEYgRFYYJaQe2mmYIMRGIERGIERGIEFRmAERmDedAFtjAtAGWDnoAAAAABJRU5ErkJggg==' + +interface SeededFixture { + app: ElectronApplication + mock: MockServer + page: Page + sandbox: Sandbox + cleanup: () => Promise +} + +function writeImage(sandbox: Sandbox): string { + const dir = path.join(sandbox.root, IMAGE_DIR) + fs.mkdirSync(dir, { recursive: true }) + + const imagePath = path.join(dir, IMAGE_NAME) + fs.writeFileSync(imagePath, Buffer.from(PNG_BASE64, 'base64')) + + return imagePath +} + +async function setupSeededDesktop(): Promise { + const mock = await startMockServer() + const sandbox = createSandbox('image-attachment') + writeMockProviderConfig(sandbox.hermesHome, mock.url, undefined, NATIVE_IMAGE_CONFIG) + writeEnvFile(sandbox.hermesHome) + + const builder = await RealSessionBuilder.start(sandbox.hermesHome) + + try { + await builder.createSession({ + title: SESSION_TITLE, + turns: [{ images: [writeImage(sandbox)], text: CAPTION }], + }) + } finally { + await builder.close() + } + + const { app, page } = await launchDesktop(buildAppEnv(sandbox)) + + return { + app, + mock, + page, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + sandbox.cleanup() + }, + } +} + +function sessionRow(page: Page) { + return page.locator('[data-slot="sidebar"] button').filter({ hasText: CAPTION }).first() +} + +// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the +// renderer's keep-alive visibility policy instead of relying on DOM order. +const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])' + +function activeViewportText(surfaceSelector: string): string { + const surfaces = document.querySelectorAll(surfaceSelector) + + return surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '' +} + +async function openSeededSession(page: Page): Promise { + const row = sessionRow(page) + await row.waitFor({ state: 'visible', timeout: 60_000 }) + await row.click() + await page.waitForFunction( + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '' + + return text.includes(expected) + }, + [CAPTION, SURFACE] as [string, string], + { timeout: 30_000 }, + ) +} + +/** + * The sidebar "+" opens a NEW TAB beside the current chat instead of replacing + * it, so the seeded session stays mounted in its own surface. Assert the new + * surface is empty rather than waiting for the old caption to leave the page. + */ +async function openNewSession(page: Page): Promise { + await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click() + await page.waitForFunction( + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '' + + return surfaces.length > 0 && !text.includes(expected) + }, + [CAPTION, SURFACE] as [string, string], + { timeout: 15_000 }, + ) +} + +async function transcriptText(page: Page): Promise { + return page.evaluate(activeViewportText, SURFACE) +} + +async function assertRendersThumbnail(page: Page, label: string): Promise { + const thumbnail = page.locator('[data-slot="aui_directive-image"] img') + await expect(thumbnail, `${label}: the attachment should render as an image`).toHaveCount(1) + await expect(thumbnail, `${label}: the thumbnail should resolve off disk`).toHaveAttribute('src', /^data:image\//) + + const text = await transcriptText(page) + expect(text, `${label}: the caption should survive alongside the image`).toContain(CAPTION) + // A broken ref falls back to a chip whose label leaks the path, and a + // flattened multimodal turn leaves the agent's placeholder behind. + expect(text, `${label}: the raw image path should not leak into the transcript`).not.toContain(IMAGE_NAME) + expect(text, `${label}: the image directive should not render literally`).not.toContain('@image:') + expect(text, `${label}: the flattening placeholder should not render`).not.toContain('[screenshot]') +} + +test.describe('attached image resume', () => { + let fixture: SeededFixture | null = null + + test.afterEach(async () => { + await fixture?.cleanup() + fixture = null + }) + + test('renders a persisted attachment as a thumbnail on first open and after a cold reload', async ({}, testInfo) => { + // Seeding through the real gateway plus two full app boots does not fit the + // default per-test budget on a cold runner. + test.slow() + + fixture = await setupSeededDesktop() + await waitForAppReady(fixture, 120_000) + + // The sidebar labels a session by its preview, so the caption has to lead + // the persisted turn — a leading directive reads as a truncated file path. + const row = sessionRow(fixture.page) + await row.waitFor({ state: 'visible', timeout: 60_000 }) + + const label = (await row.textContent())?.trim() ?? '' + expect(label.startsWith(CAPTION), `sidebar label should open with the caption: ${label}`).toBe(true) + + await openSeededSession(fixture.page) + await assertRendersThumbnail(fixture.page, 'first open') + await fixture.page.screenshot({ path: testInfo.outputPath('attachment-first-open.png') }) + + // A reload drops every cached attachment ref, so the transcript has to come + // back from the persisted turn alone. + await fixture.page.reload() + await waitForAppReady(fixture, 120_000) + await openNewSession(fixture.page) + + await openSeededSession(fixture.page) + await assertRendersThumbnail(fixture.page, 'cold reload') + await fixture.page.screenshot({ path: testInfo.outputPath('attachment-cold-reload.png') }) + }) +}) diff --git a/apps/desktop/e2e/mock-server.ts b/apps/desktop/e2e/mock-server.ts index a198ff8989c..ce4665d1775 100644 --- a/apps/desktop/e2e/mock-server.ts +++ b/apps/desktop/e2e/mock-server.ts @@ -14,8 +14,11 @@ * prove the full boot → gateway → inference → renderer chain works. */ +import fs from 'node:fs' import http from 'node:http' import type { ServerResponse } from 'node:http' +import os from 'node:os' +import nodePath from 'node:path' /** A canned assistant reply used for every chat completion request. */ export const MOCK_REPLY = 'Hello from the mock inference server! The full boot chain is working.' @@ -27,6 +30,14 @@ export interface MockServerOptions { holdFirstCompletionContaining?: string /** Absolute sandbox path written by the verify-on-stop scripted tool call. */ verificationWritePath?: string +/** + * Sentinel path that ends the E2E_SIDEBAR_CROSS background process. + * + * Without it that process is a bare `sleep 5`, which races the agent turn and + * the 4s auto-dismiss linger — see `createBackgroundReleaseHandle`. Pass a + * handle's `path` to let the test decide when the process exits. + */ +backgroundReleasePath?: string } export interface MockServer { @@ -167,37 +178,68 @@ const SIDEBAR_SCRIPT: ScriptedTurn[] = [ // ─── Sidebar cross-session script ────────────────────────────────────── // -// E2E_SIDEBAR_CROSS trigger uses a longer background process (sleep 5) so -// the "background running" dot is visible long enough for the test to: +// E2E_SIDEBAR_CROSS starts a long background process plus a subagent so the +// tests can: // 1. See the background dot while the subagent runs. // 2. Open a different session and see session A's dot transition to // "finished unread" when the background process completes. +// +// The background process must outlive the agent turn — the whole point is a +// dot that is still "running" after the final answer lands. A fixed `sleep` +// cannot guarantee that: on a loaded CI runner the turn (two model round +// trips + a real subagent delegation) can take longer than the sleep, the +// process exits early, the 4s success linger elapses, and the dot is gone +// before the test looks. That is a wall-clock race between three independent +// timers, and it made this the flakiest spec in the suite. +// +// When `backgroundReleasePath` is set the process instead blocks until the +// test creates that sentinel file, so the test — not the clock — decides when +// the dot clears. `sleep 5` remains the fallback for callers that don't pass +// a handle. +function sidebarCrossBgCommand(releasePath?: string): string { + if (!releasePath) { + return 'echo "long bg output" && sleep 5 && echo "finished"' + } + // Bounded wait (60s): if a test forgets to release (or crashes mid-way), + // the process still exits instead of hanging the worker until the suite + // times out. + const quoted = JSON.stringify(releasePath) + return [ + 'echo "long bg output"', + `for _ in $(seq 1 600); do [ -e ${quoted} ] && break; sleep 0.1; done`, + 'echo "finished"', + ].join(' && ') +} -const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = [ - { - text: 'Starting a long background task and delegating work.', - toolCalls: [ - { - name: 'terminal', - args: { - command: 'echo "long bg output" && sleep 5 && echo "finished"', - background: true, - notify_on_complete: true, +function sidebarCrossScript(releasePath?: string): ScriptedTurn[] { + return [ + { + text: 'Starting a long background task and delegating work.', + toolCalls: [ + { + name: 'terminal', + args: { + command: sidebarCrossBgCommand(releasePath), + background: true, + notify_on_complete: true, + }, }, - }, - { - name: 'delegate_task', - args: { - goal: 'Analyze cross-session state', - context: 'Testing that the background dot updates across sessions.', + { + name: 'delegate_task', + args: { + goal: 'Analyze cross-session state', + context: 'Testing that the background dot updates across sessions.', + }, }, - }, - ], - }, - { - text: 'Both tasks are running in the background now.', - }, -] + ], + }, + { + text: 'Both tasks are running in the background now.', + }, + ] +} + +const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = sidebarCrossScript() const QUEUE_STOP_SCRIPT: ScriptedTurn[] = [ { @@ -423,7 +465,8 @@ export function startMockServer(options: MockServerOptions = {}): Promise void + /** Remove the sentinel if it still exists. Safe to call twice. */ + cleanup: () => void +} + +/** + * Create a sentinel that keeps the E2E_SIDEBAR_CROSS background process alive + * until the test explicitly releases it. + * + * The cross-session sidebar tests need a background process that is still + * RUNNING after the agent turn finishes — that is the state under test (a + * session whose turn is done but whose background work is not). With a fixed + * `sleep`, three independent clocks race: the sleep, the agent turn (two model + * round trips plus a real subagent delegation), and the 4s success linger + * before a finished task auto-dismisses. When a loaded CI runner makes the + * turn slower than the sleep, the process is already gone and the assertion + * samples an empty sidebar. Observed on CI 2026-07-26 across two unrelated + * PRs: the "should appear" poll needed 7.5s to see the dot, by which point + * `sleep 5` had exited. + * + * With a sentinel there is one clock and the test owns it: + * + * ```ts + * const release = createBackgroundReleaseHandle() + * const mock = await startMockServer({ backgroundReleasePath: release.path }) + * // ... assert the dot is visible; it cannot vanish on its own ... + * release.release() // now, and only now, the process exits + * ``` + */ +export function createBackgroundReleaseHandle(): BackgroundReleaseHandle { + const path = nodePath.join( + os.tmpdir(), + `hermes-e2e-bg-release-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ) + return { + path, + release: () => { + try { + fs.writeFileSync(path, 'release') + } catch { + // The process also has a bounded fallback wait; a failed write must + // not crash the test before its real assertions run. + } + }, + cleanup: () => { + try { + fs.rmSync(path, { force: true }) + } catch { + // Best-effort — the sentinel lives in the OS temp dir. + } + }, + } +} + /** * The interim script's text constants, exported for test assertions. * Each entry is the visible text of one turn. Turns with empty text @@ -756,8 +858,12 @@ export const SIDEBAR_CROSS_TEXTS = { interimText: SIDEBAR_CROSS_SCRIPT[0].text, /** The final answer text. */ finalText: SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1].text, - /** The longer background process command (sleep 5). */ - bgCommand: 'echo "long bg output" && sleep 5 && echo "finished"', + /** + * The default (unheld) background process command. Tests that pass a + * `backgroundReleasePath` get a sentinel-waiting command instead — see + * `createBackgroundReleaseHandle`. + */ + bgCommand: sidebarCrossBgCommand(), /** The subagent's goal. */ subagentGoal: 'Analyze cross-session state', } as const diff --git a/apps/desktop/e2e/real-session-builder.ts b/apps/desktop/e2e/real-session-builder.ts index 488291695b5..0f670e9698b 100644 --- a/apps/desktop/e2e/real-session-builder.ts +++ b/apps/desktop/e2e/real-session-builder.ts @@ -28,11 +28,18 @@ interface CreatedSession { stored_session_id: string } +export interface RealSessionTurn { + /** Local image paths attached before the prompt, as the composer would. */ + images?: readonly string[] + text: string +} + export interface RealSessionSpec { - /** Human-visible sidebar title, persisted by the first completed turn. */ + /** Session label. The durable row stores no title, so clients fall back to + * the preview (the first 60 characters of the first user message). */ title: string /** Each item becomes one real user prompt followed by the mock provider's reply. */ - turns: readonly string[] + turns: readonly (RealSessionTurn | string)[] } export interface RealSession { @@ -107,7 +114,13 @@ export class RealSessionBuilder { const runtimeId = requireString(created, 'session_id') const sessionId = requireString(created, 'stored_session_id') - for (const text of spec.turns) { + for (const turn of spec.turns) { + const { images = [], text } = typeof turn === 'string' ? { text: turn } : turn + + for (const image of images) { + await this.request('image.attach', { session_id: runtimeId, path: image }) + } + const completion = this.waitForEvent( frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId, ) diff --git a/apps/desktop/e2e/session-compression-and-queue-stop.spec.ts b/apps/desktop/e2e/session-compression-and-queue-stop.spec.ts index f15b8d0befc..911f15c7840 100644 --- a/apps/desktop/e2e/session-compression-and-queue-stop.spec.ts +++ b/apps/desktop/e2e/session-compression-and-queue-stop.spec.ts @@ -122,11 +122,16 @@ auxiliary: // A normal message crosses the tiny configured context budget. The mock // blocks only the resulting summary request, so these assertions run // during automatic compaction rather than a slash-command path. + // The payload must cross threshold_tokens (22k) on its OWN weight + // (~12k tokens) on top of the system prompt. Do not shrink it: at + // repeat(500) the trigger only worked because the ambient system prompt + // (skills index + tool schemas) happened to carry it over the line, and + // a 160-token skills-index cleanup on main broke the test for a day. await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_ONE '.repeat(5)) await waitForTranscript(page, MOCK_REPLY) await pasteAndSend(page, 'E2E_COMPACTION_HISTORY_TWO '.repeat(5)) await waitForTranscript(page, MOCK_REPLY) - await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(500)) + await pasteAndSend(page, 'E2E_TRIGGER_AUTOMATIC_COMPACTION '.repeat(1500)) await fixture.mock.waitForHeldCompletion() await expect(page.getByRole('status', { name: 'Summarizing thread' }).last()).toBeVisible() diff --git a/apps/desktop/e2e/sidebar-states.spec.ts b/apps/desktop/e2e/sidebar-states.spec.ts index 051efda3543..6d8c0c2c9c9 100644 --- a/apps/desktop/e2e/sidebar-states.spec.ts +++ b/apps/desktop/e2e/sidebar-states.spec.ts @@ -16,7 +16,12 @@ import { setupMockBackend, waitForAppReady, } from './fixtures' -import { SIDEBAR_CROSS_TEXTS, SIDEBAR_TEXTS, restartMockServer } from './mock-server' +import { + createBackgroundReleaseHandle, + restartMockServer, + SIDEBAR_CROSS_TEXTS, + SIDEBAR_TEXTS, +} from './mock-server' /** Background-running dot aria-label (from i18n en.ts). */ const BG_DOT_LABEL = 'Background task running' @@ -176,21 +181,30 @@ test.describe('sidebar states — cross-session dot transition', () => { test.describe.configure({ mode: 'serial' }) let fixture: MockBackendFixture + // Keeps the background process alive until this test releases it, so the + // "still running after the turn finished" state can't expire on its own. + const bgRelease = createBackgroundReleaseHandle() test.beforeAll(async () => { restartMockServer() - fixture = await setupMockBackend() + fixture = await setupMockBackend({ + mockServer: { backgroundReleasePath: bgRelease.path }, + }) await waitForAppReady(fixture, 120_000) }) test.afterAll(async () => { + // Release first so the process exits even if the test failed early, + // then drop the sentinel file. + bgRelease.release() await fixture?.cleanup() + bgRelease.cleanup() }) test('background dot transitions to finished when viewing another session', async () => { const page = fixture.page - // Start a turn with a long background process (sleep 5). + // Start a turn whose background process runs until we release it. const composer = page.locator('[contenteditable="true"]').first() await composer.waitFor({ state: 'visible', timeout: 10_000 }) await composer.click() @@ -212,8 +226,9 @@ test.describe('sidebar states — cross-session dot transition', () => { { timeout: 90_000 }, ) - // The background dot should still be visible (sleep 5 hasn't finished yet, - // or auto-dismiss hasn't fired). + // The background dot must still be visible: the turn is done but the + // process is held open by the sentinel, so this is a stable state rather + // than a window we have to catch in time. const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count() expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0) @@ -225,8 +240,9 @@ test.describe('sidebar states — cross-session dot transition', () => { await page.locator('button:has-text("New session")').first().click() await page.waitForTimeout(2000) - // Now wait for the background process to finish (sleep 5 + auto-dismiss). - // The session A dot should transition away from "background running". + // Now let the background process finish. The session A dot should + // transition away from "background running". + bgRelease.release() await expect .poll( () => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(), diff --git a/apps/desktop/e2e/tile-unread-bug.spec.ts b/apps/desktop/e2e/tile-unread-bug.spec.ts index cd87a2eb681..fa614a21d2f 100644 --- a/apps/desktop/e2e/tile-unread-bug.spec.ts +++ b/apps/desktop/e2e/tile-unread-bug.spec.ts @@ -22,7 +22,12 @@ import { setupMockBackend, waitForAppReady, } from './fixtures' -import { SIDEBAR_CROSS_TEXTS, restartMockServer } from './mock-server' +import { + type BackgroundReleaseHandle, + createBackgroundReleaseHandle, + restartMockServer, + SIDEBAR_CROSS_TEXTS, +} from './mock-server' /** Finished-unread dot aria-label. */ const UNREAD_DOT_LABEL = 'Finished — unread' @@ -34,7 +39,7 @@ function sessionRow(page: import('@playwright/test').Page, text: string) { return page.locator('[data-slot="sidebar"] button').filter({ hasText: text }).first() } -/** Common setup: start a turn with a sleep 5 bg process + subagent, wait for +/** Common setup: start a turn with a held bg process + subagent, wait for * the turn to complete, then switch to a new session so the first session is * no longer $selectedStoredSessionId (required before opening a tile). */ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) { @@ -67,7 +72,9 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) { { timeout: 90_000 }, ) - // The background dot should still be visible (sleep 5 hasn't finished). + // The background dot must still be visible: the turn is done but the + // process is held open by the sentinel, so this is a stable state rather + // than a window we have to catch in time. const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count() expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0) @@ -77,8 +84,12 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) { await page.waitForTimeout(2000) } -/** Wait for the background process to finish (sleep 5 + auto-dismiss). */ -async function waitForBgProcessToFinish(page: import('@playwright/test').Page) { +/** Release the held background process, then wait for its dot to clear. */ +async function waitForBgProcessToFinish( + page: import('@playwright/test').Page, + release?: BackgroundReleaseHandle, +) { + release?.release() await expect .poll( () => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(), @@ -95,15 +106,20 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => { test.describe.configure({ mode: 'serial' }) let fixture: MockBackendFixture + const bgRelease = createBackgroundReleaseHandle() test.beforeAll(async () => { restartMockServer() - fixture = await setupMockBackend() + fixture = await setupMockBackend({ + mockServer: { backgroundReleasePath: bgRelease.path }, + }) await waitForAppReady(fixture, 120_000) }) test.afterAll(async () => { + bgRelease.release() await fixture?.cleanup() + bgRelease.cleanup() }) test('session opened as a tab (not visible) correctly gets unread dot', async () => { @@ -123,12 +139,21 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => { // Evidence: the tab is open but the session is not visible on screen. await page.screenshot({ path: 'test-results/tile-bug-tab-opened.png' }) - await waitForBgProcessToFinish(page) + await waitForBgProcessToFinish(page, bgRelease) // A tab that's not the active tab IS hidden — the unread dot is correct. // The user is NOT looking at it, so marking it "unread" is right. - const unreadCount = await page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count() - expect(unreadCount, 'hidden tab should be marked unread').toBeGreaterThan(0) + // + // Poll rather than sampling once: "finished-unread" is an event-driven + // transition that lands slightly after the running dot clears, and with a + // released (rather than slowly-expiring) process there is no incidental + // slack between the two. Same reasoning as the cross-session spec. + await expect + .poll( + () => page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count(), + { timeout: 30_000, message: 'hidden tab should be marked unread' }, + ) + .toBeGreaterThan(0) await page.screenshot({ path: 'test-results/tile-bug-tab-unread-correct.png' }) }) @@ -142,15 +167,20 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () => test.describe.configure({ mode: 'serial' }) let fixture: MockBackendFixture + const bgRelease = createBackgroundReleaseHandle() test.beforeAll(async () => { restartMockServer() - fixture = await setupMockBackend() + fixture = await setupMockBackend({ + mockServer: { backgroundReleasePath: bgRelease.path }, + }) await waitForAppReady(fixture, 120_000) }) test.afterAll(async () => { + bgRelease.release() await fixture?.cleanup() + bgRelease.cleanup() }) test('session visible in a split tile does NOT get unread dot when it finishes', async () => { @@ -196,7 +226,7 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () => // Evidence: the split tile is now open side-by-side — both sessions visible. await page.screenshot({ path: 'test-results/tile-bug-split-opened.png' }) - await waitForBgProcessToFinish(page) + await waitForBgProcessToFinish(page, bgRelease) // THE BUG: the session visible in the split tile should NOT have the green // "finished unread" dot — the user is looking right at it. This assertion diff --git a/apps/desktop/e2e/warm-resume-jitter.spec.ts b/apps/desktop/e2e/warm-resume-jitter.spec.ts index 72e71087d69..dbc0fe5dd11 100644 --- a/apps/desktop/e2e/warm-resume-jitter.spec.ts +++ b/apps/desktop/e2e/warm-resume-jitter.spec.ts @@ -24,6 +24,8 @@ * MutationObserver burst), but `$messages` was still set twice. * * The test passes when bursts === 1 AND reconciles === 0. + * The sidebar "+" keeps the session warm in another tab. Its reactivation + * follows the same contract: one additive paint and zero reconciles. * * Prerequisite: `npm run build` must have been run so dist/ exists. */ @@ -43,6 +45,11 @@ import { startMockServer } from './mock-server' import { RealSessionBuilder } from './real-session-builder' const SESSION_TITLE = 'E2E Warm Resume Jitter Test' + +// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the +// renderer's keep-alive visibility policy instead of relying on DOM order. +const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])' +const ALL_SURFACES = '[data-composer-target]' /** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */ const MESSAGE_COUNT = 32 /** Seeded PRNG so the generated content is deterministic across runs. */ @@ -154,15 +161,29 @@ test.afterAll(async () => { * after the initial paint, catching key-based reconciles that don't * add/remove nodes. */ -async function installRenderCounter(page: import('@playwright/test').Page): Promise { - await page.evaluate(() => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') +async function installRenderCounter( + page: import('@playwright/test').Page, + transcriptText?: string, +): Promise { + await page.evaluate(([visibleSelector, allSelector, expected]: [string, string, string | undefined]) => { + const surfaces = [...document.querySelectorAll(expected ? allSelector : visibleSelector)] + const surface = expected + ? surfaces.find(candidate => + (candidate.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected), + ) + : surfaces.at(-1) + const viewport = surface?.querySelector('[data-slot="aui_thread-viewport"]') if (!viewport) { throw new Error('Thread viewport not found before warm resume') } const state = { bursts: 0, mutations: 0, timeline: [] as number[], stopped: false, reconciles: 0 } - ;(window as unknown as { __RENDER_COUNT__: typeof state }).__RENDER_COUNT__ = state + const debugWindow = window as unknown as { + __RENDER_COUNT__: typeof state + __RENDER_VIEWPORT__: Element + } + debugWindow.__RENDER_COUNT__ = state + debugWindow.__RENDER_VIEWPORT__ = viewport let currentBatch = 0 let flushTimer: ReturnType | null = null @@ -224,7 +245,53 @@ async function installRenderCounter(page: import('@playwright/test').Page): Prom hasMessages = true } }, 2) - }) + }, [SURFACE, ALL_SURFACES, transcriptText] as [string, string, string | undefined]) +} + +/** Wait until the ACTIVE chat surface's transcript contains `text`. */ +async function waitForActiveTranscriptText( + page: import('@playwright/test').Page, + text: string, + timeout = 30_000, +): Promise { + await page.waitForFunction( + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const active = surfaces[surfaces.length - 1] + + return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected) + }, + [text, SURFACE] as [string, string], + { timeout }, + ) +} + +async function waitForActiveTranscriptWithoutText( + page: import('@playwright/test').Page, + text: string, +): Promise { + await page.waitForFunction( + ([expected, surfaceSelector]: [string, string]) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const active = surfaces[surfaces.length - 1] + + return !(active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected) + }, + [text, SURFACE] as [string, string], + { timeout: 15_000 }, + ) +} + +/** Replace the primary surface with a draft while retaining its warm cache. */ +async function openFreshDraft(page: import('@playwright/test').Page, priorText: string): Promise { + await page.keyboard.press(process.platform === 'darwin' ? 'Meta+N' : 'Control+N') + await waitForActiveTranscriptWithoutText(page, priorText) +} + +/** Stack an empty tab while leaving the current transcript mounted and warm. */ +async function openNewSessionTab(page: import('@playwright/test').Page, priorText: string): Promise { + await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click() + await waitForActiveTranscriptWithoutText(page, priorText) } /** Stop the render counter and return the recorded burst/reconcile counts. */ @@ -245,6 +312,30 @@ async function readRenderCount(page: import('@playwright/test').Page): Promise<{ }) } +async function observedViewportIsActive(page: import('@playwright/test').Page): Promise { + return page.evaluate((surfaceSelector: string) => { + const surfaces = document.querySelectorAll(surfaceSelector) + const activeViewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]') + const observedViewport = (window as unknown as { __RENDER_VIEWPORT__?: Element }).__RENDER_VIEWPORT__ + + return activeViewport === observedViewport + }, SURFACE) +} + +/** A kept-alive tab must become visible without rebuilding its transcript. */ +function assertNoRepaint(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void { + expect(result, 'MutationObserver should have recorded render data').toBeTruthy() + expect( + result!.bursts, + `Expected no additive render bursts for a kept-alive tab, but got ${result!.bursts}. ` + + `Mutation timeline: ${JSON.stringify(result!.timeline)}.`, + ).toBe(0) + expect( + result!.reconciles, + `Expected no transcript reconciles for a kept-alive tab, but got ${result!.reconciles}.`, + ).toBe(0) +} + /** Assert the render counter shows exactly one paint with no re-renders. */ function assertNoJitter(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void { expect(result, 'MutationObserver should have recorded render data').toBeTruthy() @@ -261,7 +352,7 @@ function assertNoJitter(result: { bursts: number; mutations: number; timeline: n ).toBe(0) } -test('warm-route resume paints transcript exactly once (no jitter)', async ({}, testInfo) => { +test('tab reactivation preserves the mounted transcript without repainting', async ({}, testInfo) => { const page = fixture!.page // Wait for the sidebar to populate with our seeded session. @@ -277,63 +368,29 @@ test('warm-route resume paints transcript exactly once (no jitter)', async ({}, // Wait for the transcript to appear — the first user message text confirms // the cold-path prefetch painted. - await page.waitForFunction( - (text: string) => - document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ?? - false, - FIRST_USER_MSG, - { timeout: 30_000 }, - ) + await waitForActiveTranscriptText(page, FIRST_USER_MSG) // Wait for the session to fully settle (cold-path RPC + reconciliation). await page.waitForTimeout(2_000) - // Step 2: Navigate away to a new chat — this does NOT evict the warm cache. - const newSessionButton = page - .locator('[data-slot="sidebar"] button[aria-label="New session"]') - .first() - await newSessionButton.click() - - // Wait for the new-chat empty state. - await page.waitForFunction( - (firstMsg: string) => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') - if (!viewport) return false - const text = viewport.textContent ?? '' - return !text.includes(firstMsg) - }, - FIRST_USER_MSG, - { timeout: 15_000 }, - ) - + // Stack a new tab, then observe the seeded transcript while it is hidden. + // Installing after the switch isolates reactivation from mutations caused + // while the new tab was being created. + await openNewSessionTab(page, FIRST_USER_MSG) await page.waitForTimeout(500) + await installRenderCounter(page, FIRST_USER_MSG) - // Step 3: Install render counter, click back (warm resume), wait, assert. - await installRenderCounter(page) + // Step 3: Click back and verify the same kept-alive viewport becomes active + // without rebuilding or reconciling its transcript. await sessionRow.click() - await page.waitForFunction( - (text: string) => - document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ?? - false, - FIRST_USER_MSG, - { timeout: 30_000 }, - ) - - // Wait for at least 1 burst, then settle. - await page.waitForFunction( - () => { - const w = window as unknown as { __RENDER_COUNT__?: { bursts: number } } - return Boolean(w.__RENDER_COUNT__ && w.__RENDER_COUNT__.bursts > 0) - }, - undefined, - { timeout: 10_000 }, - ) + await waitForActiveTranscriptText(page, FIRST_USER_MSG) await page.waitForTimeout(2_000) + expect(await observedViewportIsActive(page), 'Reactivation should reveal the observed kept-alive viewport').toBe(true) const result = await readRenderCount(page) await page.screenshot({ path: testInfo.outputPath('warm-resume-idle.png') }) - assertNoJitter(result) + assertNoRepaint(result) }) test('warm-route resume after background inference completes (no jitter)', async ({}, testInfo) => { @@ -354,13 +411,7 @@ test('warm-route resume after background inference completes (no jitter)', async // Step 1: Cold resume — populate the warm cache. await sessionRow.click() - await page.waitForFunction( - (text: string) => - document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ?? - false, - FIRST_USER_MSG, - { timeout: 30_000 }, - ) + await waitForActiveTranscriptText(page, FIRST_USER_MSG) await page.waitForTimeout(2_000) // Step 2: Send a message — triggers inference via the mock server. @@ -373,34 +424,15 @@ test('warm-route resume after background inference completes (no jitter)', async // Wait for the mock response to appear in the transcript, confirming // the turn completed and message.complete fired (which updates the warm // cache via updateSessionState). - await page.waitForFunction( - () => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') - return viewport?.textContent?.includes('mock inference server') ?? false - }, - undefined, - { timeout: 60_000 }, - ) + await waitForActiveTranscriptText(page, 'mock inference server', 60_000) // Extra settle for message.complete → updateSessionState → cache write. await page.waitForTimeout(2_000) // Verify the prompt was received by the mock server. expect(mock.receivedPrompts).toContain(PROMPT) - // Step 3: Navigate away — the warm cache retains the updated messages. - const newSessionButton = page - .locator('[data-slot="sidebar"] button[aria-label="New session"]') - .first() - await newSessionButton.click() - await page.waitForFunction( - (prompt: string) => { - const viewport = document.querySelector('[data-slot="aui_thread-viewport"]') - if (!viewport) return false - return !(viewport.textContent ?? '').includes(prompt) - }, - PROMPT, - { timeout: 15_000 }, - ) + // Step 3: Replace the primary chat; the warm cache retains the updated messages. + await openFreshDraft(page, PROMPT) await page.waitForTimeout(500) // Step 4: Install render counter, click back (warm resume), wait, assert. @@ -409,13 +441,7 @@ test('warm-route resume after background inference completes (no jitter)', async // Wait for the transcript to reappear — the warm cache should already // have the completed turn (updated by message.complete events). - await page.waitForFunction( - (text: string) => - document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ?? - false, - FIRST_USER_MSG, - { timeout: 30_000 }, - ) + await waitForActiveTranscriptText(page, FIRST_USER_MSG) // Wait for at least 1 burst, then settle. await page.waitForFunction( diff --git a/apps/desktop/electron/backend-env.test.ts b/apps/desktop/electron/backend-env.test.ts index e24a66ee396..a92ce6e062e 100644 --- a/apps/desktop/electron/backend-env.test.ts +++ b/apps/desktop/electron/backend-env.test.ts @@ -68,6 +68,26 @@ test('buildDesktopBackendEnv extends PYTHONPATH and backend PATH together', () = assert.ok(env.PATH.includes('/opt/homebrew/bin')) }) +test('buildDesktopBackendEnv forces PYTHONUTF8 unless the user set it explicitly', () => { + const defaulted = buildDesktopBackendEnv({ + hermesHome: '/Users/test/.hermes', + currentEnv: { PATH: '/usr/bin' }, + platform: 'darwin', + pathModule: path.posix + }) + + assert.equal(defaulted.PYTHONUTF8, '1') + + const optedOut = buildDesktopBackendEnv({ + hermesHome: '/Users/test/.hermes', + currentEnv: { PATH: '/usr/bin', PYTHONUTF8: '0' }, + platform: 'darwin', + pathModule: path.posix + }) + + assert.equal(optedOut.PYTHONUTF8, '0') +}) + test('normalizeHermesHomeRoot maps profile homes back to the global Hermes root', () => { assert.equal( normalizeHermesHomeRoot('/Users/test/.hermes/profiles/oracle', { pathModule: path.posix }), diff --git a/apps/desktop/electron/backend-env.ts b/apps/desktop/electron/backend-env.ts index a225e238177..3db4a19d034 100644 --- a/apps/desktop/electron/backend-env.ts +++ b/apps/desktop/electron/backend-env.ts @@ -104,6 +104,13 @@ function buildDesktopBackendEnv({ return { PYTHONPATH: appendUniquePathEntries([...pythonPathEntries, currentPythonPath], { delimiter }), + // Force PEP 540 UTF-8 mode in the spawned Python backend so its stdio and + // subprocess defaults are UTF-8 even on non-UTF-8 Windows locales (GBK, + // cp1252, ...). hermes_bootstrap sets this inside the child too, but only + // after import — anything emitted earlier (interpreter startup errors, + // pre-bootstrap tracebacks) still decodes with the locale default without + // this. User's explicit setting wins. Re-port of PR #56499 (echoriver89). + PYTHONUTF8: currentEnv?.PYTHONUTF8 ?? '1', [key]: buildDesktopBackendPath({ hermesHome, venvRoot, diff --git a/apps/desktop/electron/backend-health.test.ts b/apps/desktop/electron/backend-health.test.ts new file mode 100644 index 00000000000..8d29ea1988d --- /dev/null +++ b/apps/desktop/electron/backend-health.test.ts @@ -0,0 +1,340 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { + DEFAULT_HEALTH_PROBE_TIMEOUT_MS, + isAuthRejectionError, + isGatedMissingHealthError, + isMissingHealthEndpointError, + isReauthRequiredError, + waitForHermesReady +} from './backend-health' + +const GATE_401 = '401: {"error":"unauthenticated","detail":"Unauthorized","reason":"no_cookie","login_url":"/login"}' + +test('uses lightweight /api/health for current backends', async () => { + const calls: string[][] = [] + + await waitForHermesReady('http://127.0.0.1:9000/', { + token: 'secret-token', + fetchPublicJson: async url => { + calls.push(['public', url]) + + return { ok: true } + }, + fetchJson: async url => { + calls.push(['token', url]) + throw new Error('status should not be called') + }, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }) + + assert.deepEqual(calls, [['public', 'http://127.0.0.1:9000/api/health']]) +}) + +test('falls back to /api/status only for old backends without /api/health', async () => { + const calls: string[][] = [] + + await waitForHermesReady('http://127.0.0.1:9000', { + token: 'secret-token', + fetchPublicJson: async url => { + calls.push(['public', url]) + + throw new Error('404: {"detail":"Not Found"}') + }, + fetchJson: async (url, token) => { + calls.push(['token', url, token ?? '']) + + return { version: 'old' } + }, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }) + + assert.deepEqual(calls, [ + ['public', 'http://127.0.0.1:9000/api/health'], + ['token', 'http://127.0.0.1:9000/api/status', 'secret-token'] + ]) +}) + +test('does not fall back to heavyweight /api/status for transient health failures', async () => { + const calls: string[][] = [] + let currentTime = 0 + + await assert.rejects( + waitForHermesReady('http://127.0.0.1:9000', { + fetchPublicJson: async url => { + calls.push(['public', url]) + throw new Error('Timed out connecting to Hermes backend after 15000ms') + }, + fetchJson: async url => { + calls.push(['token', url]) + }, + sleep: async () => {}, + now: () => { + currentTime += 20 + + return currentTime + }, + timeoutMs: 50, + pollMs: 1 + }), + /Timed out connecting/ + ) + + assert.ok(calls.length > 0) + assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health'))) +}) + +test('probes health on a short timeout but leaves the legacy fallback its own', async () => { + const timeouts: (number | undefined)[] = [] + + await waitForHermesReady('http://127.0.0.1:9000', { + fetchPublicJson: async (_url, options) => { + timeouts.push(options?.timeoutMs) + + throw new Error('404: {"detail":"Not Found"}') + }, + fetchJson: async (_url, _token, options) => { + timeouts.push(options?.timeoutMs) + + return { version: 'old' } + }, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }) + + assert.deepEqual(timeouts, [DEFAULT_HEALTH_PROBE_TIMEOUT_MS, undefined]) +}) + +test('aborts as superseded when the bootstrap signal fires', async () => { + const controller = new AbortController() + controller.abort() + + await assert.rejects( + waitForHermesReady('http://127.0.0.1:9000', { + signal: controller.signal, + fetchPublicJson: async () => { + throw new Error('should not probe after abort') + }, + fetchJson: async () => { + throw new Error('should not probe after abort') + }, + timeoutMs: 100, + pollMs: 1 + }), + (error: any) => error.kind === 'superseded' + ) +}) + +test('recognizes missing-route shapes only', () => { + assert.equal(isMissingHealthEndpointError(new Error('404: {"detail":"Not Found"}')), true) + assert.equal( + isMissingHealthEndpointError( + new Error('Expected JSON from /api/health but got HTML. The endpoint is likely missing on the Hermes backend.') + ), + true + ) + assert.equal(isMissingHealthEndpointError(new Error('Timed out connecting to Hermes backend after 15000ms')), false) + assert.equal(isMissingHealthEndpointError(new Error('500: boom')), false) +}) + +// --- Gated backends that predate /api/health (release 0.19.0 and earlier) --- +// +// The dashboard auth gate runs ahead of the SPA catch-all, so on a backend +// without the route an ANONYMOUS probe is rejected as unauthenticated rather +// than 404 — verified against a simulated 0.19.0 backend: +// credential-free: /api/health -> 401 no_cookie, /api/status -> 200 +// credentialed: /api/health -> 404, /api/sessions -> 200 + +test('anonymous gate-shaped 401 falls back to /api/status (backend predates /api/health)', async () => { + const calls: string[][] = [] + + await waitForHermesReady('http://192.168.1.132:9119', { + token: null, + fetchPublicJson: async url => { + calls.push(['public', url]) + throw new Error(GATE_401) + }, + fetchJson: async (url, token) => { + calls.push(['token', url, token == null ? 'null' : token]) + + return { version: '0.19.0', auth_required: true } + }, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }) + + assert.deepEqual(calls, [ + ['public', 'http://192.168.1.132:9119/api/health'], + ['token', 'http://192.168.1.132:9119/api/status', 'null'] + ]) +}) + +test('a credentialed 401 fails fast for reauth instead of reporting a dead session ready', async () => { + // The regression a blanket 401->fallback introduces: /api/status is public, + // so an expired session would answer 200 and boot would report "ready", + // deferring the no_cookie to the first real API call. + const calls: string[][] = [] + + await assert.rejects( + waitForHermesReady('https://gateway.example', { + token: 'session-token', + fetchPublicJson: async () => { + throw new Error('public probe must not be used when credentialed') + }, + fetchJson: async url => { + calls.push(['status', url]) + + return { version: '0.19.0' } + }, + probeHealth: async url => { + calls.push(['probe', url]) + throw new Error(GATE_401) + }, + probeIsCredentialed: true, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }), + (error: any) => { + assert.equal(isReauthRequiredError(error), true) + assert.equal(error.needsOauthLogin, true) + assert.match(error.message, /remote gateway session has expired/i) + + return true + } + ) + + // Fail fast: never reached the public /api/status leg. + assert.deepEqual(calls, [['probe', 'https://gateway.example/api/health']]) +}) + +test('a credentialed 403 is also a terminal reauth failure', async () => { + await assert.rejects( + waitForHermesReady('https://gateway.example', { + fetchPublicJson: async () => ({}), + fetchJson: async () => ({}), + probeHealth: async () => { + throw new Error('403: {"detail":"Forbidden"}') + }, + probeIsCredentialed: true, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }), + (error: any) => isReauthRequiredError(error) + ) +}) + +test('a credentialed probe still uses the 404 fallback for a genuinely missing route', async () => { + // With credentials the gate lets the request through to the SPA catch-all, + // so an old backend answers a real 404 — that must still fall back, not be + // mistaken for a rejected session. + const calls: string[][] = [] + + await waitForHermesReady('https://gateway.example', { + token: 'session-token', + fetchPublicJson: async () => { + throw new Error('public probe must not be used when credentialed') + }, + fetchJson: async url => { + calls.push(['status', url]) + + return { version: '0.19.0' } + }, + probeHealth: async url => { + calls.push(['probe', url]) + throw new Error('404: {"detail":"Not Found"}') + }, + probeIsCredentialed: true, + sleep: async () => {}, + timeoutMs: 100, + pollMs: 1 + }) + + assert.deepEqual(calls, [ + ['probe', 'https://gateway.example/api/health'], + ['status', 'https://gateway.example/api/status'] + ]) +}) + +test('a non-gate 401 keeps polling rather than skipping a misconfigured health route', async () => { + const calls: string[][] = [] + let currentTime = 0 + + await assert.rejects( + waitForHermesReady('http://127.0.0.1:9000', { + fetchPublicJson: async url => { + calls.push(['public', url]) + throw new Error('401: {"detail":"Unauthorized"}') + }, + fetchJson: async url => { + calls.push(['token', url]) + }, + sleep: async () => {}, + now: () => { + currentTime += 20 + + return currentTime + }, + timeoutMs: 50, + pollMs: 1 + }), + /401: \{"detail":"Unauthorized"\}/ + ) + + assert.ok(calls.length > 0) + assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health'))) +}) + +test('credentialed 5xx and 429 keep polling — only 401/403 are terminal', async () => { + for (const transient of ['500: boom', '429: {"detail":"Too Many Requests"}']) { + let attempts = 0 + let currentTime = 0 + + await assert.rejects( + waitForHermesReady('https://gateway.example', { + fetchPublicJson: async () => ({}), + fetchJson: async () => ({}), + probeHealth: async () => { + attempts += 1 + throw new Error(transient) + }, + probeIsCredentialed: true, + sleep: async () => {}, + now: () => { + currentTime += 20 + + return currentTime + }, + timeoutMs: 100, + pollMs: 1 + }), + (error: any) => isReauthRequiredError(error) === false + ) + + assert.ok(attempts > 1, `${transient} should have retried, got ${attempts} attempt(s)`) + } +}) + +test('error-shape predicates', () => { + assert.equal(isGatedMissingHealthError(new Error(GATE_401)), true) + assert.equal(isGatedMissingHealthError(new Error('401: {"detail":"Unauthorized"}')), false) + assert.equal(isGatedMissingHealthError(new Error('404: {"detail":"Not Found"}')), false) + + assert.equal(isAuthRejectionError(new Error(GATE_401)), true) + assert.equal(isAuthRejectionError(new Error('403: {"detail":"Forbidden"}')), true) + assert.equal(isAuthRejectionError(new Error('404: {"detail":"Not Found"}')), false) + assert.equal(isAuthRejectionError(new Error('429: slow down')), false) + assert.equal(isAuthRejectionError(new Error('500: boom')), false) + + // A gated 401 must NOT be conflated with a missing route by the 404 predicate. + assert.equal(isMissingHealthEndpointError(new Error(GATE_401)), false) +}) diff --git a/apps/desktop/electron/backend-health.ts b/apps/desktop/electron/backend-health.ts new file mode 100644 index 00000000000..3da6c7a80fa --- /dev/null +++ b/apps/desktop/electron/backend-health.ts @@ -0,0 +1,169 @@ +export const DEFAULT_BACKEND_READY_TIMEOUT_MS = 45_000 +export const DEFAULT_BACKEND_READY_POLL_MS = 500 +// A cold backend can stall its event loop for tens of seconds while Windows +// scans and byte-compiles the gateway import tree. At the default 15s socket +// timeout only three probes fit in the budget; a short one keeps retrying +// across the stall. Health only — the legacy /api/status fallback is genuinely +// slow to answer and keeps the caller's default timeout. +export const DEFAULT_HEALTH_PROBE_TIMEOUT_MS = 5_000 + +type FetchPublicJson = (url: string, options?: { timeoutMs?: number }) => Promise +type FetchJson = (url: string, token?: string | null, options?: { timeoutMs?: number }) => Promise + +export interface HermesReadyOptions { + fetchPublicJson: FetchPublicJson + fetchJson: FetchJson + token?: string | null + signal?: AbortSignal + timeoutMs?: number + pollMs?: number + healthProbeTimeoutMs?: number + sleep?: (ms: number) => Promise + now?: () => number + /** + * Credentialed health probe. When supplied, readiness is probed with the + * connection's own credentials instead of anonymously — which is what lets + * a gated backend answer 404 for a genuinely missing /api/health, and what + * makes a 401 from this probe mean "session rejected" rather than "route + * behind a gate". Defaults to the credential-free `fetchPublicJson`. + */ + probeHealth?: (url: string, options?: { timeoutMs?: number }) => Promise + /** + * Whether `probeHealth` actually presents credentials. Distinguishes the + * two very different meanings of a 401 (see `waitForHermesReady`). + */ + probeIsCredentialed?: boolean +} + +export const REMOTE_SESSION_EXPIRED_MESSAGE = + 'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.' + +export function isMissingHealthEndpointError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error ?? '') + + return /^404:/.test(message) || message.includes('endpoint is likely missing') +} + +/** + * True for a hard auth rejection (401/403) as opposed to a transient failure. + * Deliberately shape-based: 429 is a throttle and 5xx is a server fault, and + * both must keep polling. + */ +export function isAuthRejectionError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error ?? '') + + return /^40[13]:/.test(message) +} + +/** + * True for an auth rejection carrying the dashboard gate's "no session at all" + * shape. On a backend that predates `/api/health`, the gate runs ahead of the + * SPA catch-all, so an unknown `/api/*` path is rejected as unauthenticated + * instead of 404 — this is the signal that an ANONYMOUS probe cannot reach the + * route, and the reason a credential-free 401 must fall back to `/api/status` + * rather than be reported as a boot failure. + */ +export function isGatedMissingHealthError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error ?? '') + + return isAuthRejectionError(error) && message.includes('no_cookie') +} + +/** Tag a terminal reauth failure the main process latches and the overlay keys on. */ +export function makeReauthRequiredError(detail?: string): Error { + const error = new Error(REMOTE_SESSION_EXPIRED_MESSAGE) as any + error.needsOauthLogin = true + error.isReauthRequired = true + + if (detail) { + error.detail = detail + } + + return error +} + +export function isReauthRequiredError(error: unknown): boolean { + return Boolean((error as any)?.isReauthRequired) +} + +function supersededError() { + const error: any = new Error('SSH bootstrap was superseded by newer connection settings.') + error.kind = 'superseded' + + return error +} + +export async function waitForHermesReady(baseUrl: string, options: HermesReadyOptions): Promise { + const timeoutMs = options.timeoutMs ?? DEFAULT_BACKEND_READY_TIMEOUT_MS + const pollMs = options.pollMs ?? DEFAULT_BACKEND_READY_POLL_MS + const healthProbeTimeoutMs = options.healthProbeTimeoutMs ?? DEFAULT_HEALTH_PROBE_TIMEOUT_MS + const now = options.now ?? Date.now + const signal = options.signal + + const sleep = + options.sleep ?? + (ms => + new Promise((resolve, reject) => { + const timer = setTimeout(resolve, ms) + signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer) + reject(supersededError()) + }, + { once: true } + ) + })) + + const base = baseUrl.replace(/\/+$/, '') + const deadline = now() + timeoutMs + const probeHealth = options.probeHealth ?? options.fetchPublicJson + const probeIsCredentialed = Boolean(options.probeIsCredentialed) + let lastError: unknown = null + let useStatusFallback = false + + while (now() < deadline) { + if (signal?.aborted) { + throw supersededError() + } + + try { + if (useStatusFallback) { + await options.fetchJson(`${base}/api/status`, options.token) + } else { + await probeHealth(`${base}/api/health`, { timeoutMs: healthProbeTimeoutMs }) + } + + return + } catch (error) { + lastError = error + + // A confirmed 401/403 from a CREDENTIALED probe means the session was + // rejected, not that the route is missing. Fail fast into a reauth + // state: falling back to the public /api/status would answer 200 and + // report a dead session as "ready", deferring the failure to the first + // real API call. Applies to the /api/status leg too — it is routed + // through the same credentials. + if (probeIsCredentialed && isAuthRejectionError(error)) { + throw makeReauthRequiredError(error instanceof Error ? error.message : String(error)) + } + + // An explicitly missing route means the backend predates /api/health. + // So does a gate-shaped 401 on an ANONYMOUS probe: the dashboard auth + // gate runs ahead of the SPA catch-all, so a pre-/api/health backend + // rejects the unknown path as unauthenticated instead of 404 and a + // credential-free probe can never observe the 404. Timeouts, 5xx, 429, + // and non-gate 401s keep polling health. + if (!useStatusFallback && (isMissingHealthEndpointError(error) || isGatedMissingHealthError(error))) { + useStatusFallback = true + + continue + } + + await sleep(pollMs) + } + } + + const detail = lastError instanceof Error ? lastError.message : 'timeout' + throw new Error(`Hermes backend did not become ready: ${detail}`) +} diff --git a/apps/desktop/electron/backend-start-failure.test.ts b/apps/desktop/electron/backend-start-failure.test.ts index 0888d65fbc4..36d352d66cc 100644 --- a/apps/desktop/electron/backend-start-failure.test.ts +++ b/apps/desktop/electron/backend-start-failure.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { test } from 'vitest' -import { shouldLatchBackendStartFailure } from './backend-start-failure' +import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure' test('latches a LOCAL backend failure so the install-retry loop is broken', () => { assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false }), true) @@ -21,3 +21,32 @@ test('the two branches are mutually exclusive (a failure either latches or stays assert.equal(latched, !attemptedRemote) } }) + +test('latches a CONFIRMED remote reauth failure so the overlay stays clickable', () => { + // Without this the non-latching remote path re-runs startHermes on every + // getConnection/api call, re-emits running:true, and the overlay hides + // itself — the "Sign in" button flickers away before it can be clicked. + assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: true }), true) +}) + +test('does not latch a transient remote failure as reauth', () => { + // A mint timeout or a host unreachable across sleep must still self-heal. + assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: false }), false) +}) + +test('never latches a LOCAL failure as reauth (that is backendStartFailure job)', () => { + assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: true }), false) + assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: false }), false) +}) + +test('the two latches never fire for the same failure', () => { + // They are complementary, not overlapping: local failures latch via + // backendStartFailure, confirmed remote reauth latches via its own flag. + for (const attemptedRemote of [true, false]) { + for (const isReauth of [true, false]) { + const start = shouldLatchBackendStartFailure({ attemptedRemote }) + const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth }) + assert.ok(!(start && reauth), `both latched for remote=${attemptedRemote} reauth=${isReauth}`) + } + } +}) diff --git a/apps/desktop/electron/backend-start-failure.ts b/apps/desktop/electron/backend-start-failure.ts index 4998b0164a7..3c5ffdbcad9 100644 --- a/apps/desktop/electron/backend-start-failure.ts +++ b/apps/desktop/electron/backend-start-failure.ts @@ -39,3 +39,34 @@ export interface BackendStartFailureContext { export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean { return !context.attemptedRemote } + +export interface RemoteReauthFailureContext { + /** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */ + attemptedRemote: boolean + /** + * True when the failure was a CONFIRMED auth rejection (a credentialed + * probe got 401/403), not a transient connectivity fault. + */ + isReauth: boolean +} + +/** + * Whether a failed remote boot should latch as a reauth failure. + * + * This is the deliberate counterpart to `shouldLatchBackendStartFailure`, + * which never latches a remote failure because remote faults are usually + * transient and must stay retryable. A *confirmed* reauth rejection is the + * exception: it cannot self-heal, because nothing will change until the user + * signs in again. + * + * Without a latch, the non-latching remote path actively prevents recovery. + * Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits + * `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error) + * && !boot.running`) hides itself — so the "Sign in" button flickers out from + * under the user before they can click it. Latching holds the overlay still + * and clickable. Cleared on every recovery path (reset, repair, apply-config, + * and a confirmed sign-in) so a fresh session boots normally. + */ +export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean { + return context.attemptedRemote && context.isReauth +} diff --git a/apps/desktop/electron/connection-apply.ts b/apps/desktop/electron/connection-apply.ts index 5764b9091e7..579af1a79e9 100644 --- a/apps/desktop/electron/connection-apply.ts +++ b/apps/desktop/electron/connection-apply.ts @@ -1,6 +1,7 @@ async function applyConnectionChange({ cancelAndWait, isPrimary, + rehomePrimary = null, scope, sendApplied, stopPool, @@ -16,6 +17,12 @@ async function applyConnectionChange({ return } + if (rehomePrimary) { + await rehomePrimary() + + return + } + await teardownPrimary() sendApplied() } diff --git a/apps/desktop/electron/find-git-bash.test.ts b/apps/desktop/electron/find-git-bash.test.ts new file mode 100644 index 00000000000..63e00b3c3d6 --- /dev/null +++ b/apps/desktop/electron/find-git-bash.test.ts @@ -0,0 +1,54 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { findGitBash } from './find-git-bash' + +const yes = () => true +const no = () => false + +test('HERMES_GIT_BASH_PATH override takes precedence', () => { + const result = findGitBash({ + isWindows: true, + env: { HERMES_GIT_BASH_PATH: 'D:\\CustomGit\\bin\\bash.exe' }, + fileExists: yes, + findOnPath: () => null + }) + + assert.equal(result, 'D:\\CustomGit\\bin\\bash.exe') +}) + +test('HERMES_GIT_BASH_PATH invalid path falls through to candidates', () => { + const env = { + HERMES_GIT_BASH_PATH: 'X:\\Missing\\bash.exe', + LOCALAPPDATA: 'C:\\Users\\test\\AppData\\Local', + ProgramFiles: 'C:\\Program Files', + 'ProgramFiles(x86)': 'C:\\Program Files (x86)' + } + + const fileExists = (p: string) => p !== 'X:\\Missing\\bash.exe' && p.includes('Program Files\\Git\\bin\\bash.exe') + const result = findGitBash({ isWindows: true, env, fileExists, findOnPath: () => null }) + assert.equal(result, 'C:\\Program Files\\Git\\bin\\bash.exe') +}) + +test('HERMES_GIT_BASH_PATH empty string is ignored', () => { + const result = findGitBash({ + isWindows: true, + env: { HERMES_GIT_BASH_PATH: '', LOCALAPPDATA: '' }, + fileExists: no, + findOnPath: () => 'C:\\msys64\\usr\\bin\\bash.exe' + }) + + assert.equal(result, 'C:\\msys64\\usr\\bin\\bash.exe') +}) + +test('non-Windows uses findOnPath', () => { + const result = findGitBash({ + isWindows: false, + env: {}, + fileExists: no, + findOnPath: () => '/usr/bin/bash' + }) + + assert.equal(result, '/usr/bin/bash') +}) diff --git a/apps/desktop/electron/find-git-bash.ts b/apps/desktop/electron/find-git-bash.ts new file mode 100644 index 00000000000..62369e9bad6 --- /dev/null +++ b/apps/desktop/electron/find-git-bash.ts @@ -0,0 +1,67 @@ +import path from 'node:path' + +export interface GitBashOptions { + isWindows: boolean + env: Record + fileExists: (filePath: string) => boolean + findOnPath?: (command: string) => string | null +} + +/** + * Locate bash.exe on Windows. + * Resolution order (first match wins): + * 1. HERMES_GIT_BASH_PATH env var override + * 2. PortableGit under %LOCALAPPDATA%\hermes\git\ (install.ps1) + * 3. Standard Git for Windows install locations + * 4. %LOCALAPPDATA%\Programs\Git\ (user-scoped) + * 5. bash on PATH + */ +export function findGitBash(opts: GitBashOptions): string | null { + const { isWindows, env, fileExists, findOnPath } = opts + + if (!isWindows) { + return findOnPath ? findOnPath('bash') : null + } + + // Respect HERMES_GIT_BASH_PATH if set (mirrors tools/environments/local.py:_find_bash). + const gitBashPath = env.HERMES_GIT_BASH_PATH + + if (gitBashPath && fileExists(gitBashPath)) { + return gitBashPath + } + + const localAppData = env.LOCALAPPDATA || '' + const candidates: string[] = [] + + // Candidate paths are Windows paths regardless of host platform (tests run + // on POSIX CI hosts too), so join with win32 semantics explicitly. + const joinWin = path.win32.join + + if (localAppData) { + candidates.push(joinWin(localAppData, 'hermes', 'git', 'bin', 'bash.exe')) + candidates.push(joinWin(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe')) + } + + candidates.push(joinWin(env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe')) + candidates.push(joinWin(env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe')) + + if (localAppData) { + candidates.push(joinWin(localAppData, 'Programs', 'Git', 'bin', 'bash.exe')) + } + + for (const candidate of candidates) { + if (fileExists(candidate)) { + return candidate + } + } + + if (findOnPath) { + const onPath = findOnPath('bash') + + if (onPath) { + return onPath + } + } + + return null +} diff --git a/apps/desktop/electron/first-run-setup-gate.test.ts b/apps/desktop/electron/first-run-setup-gate.test.ts new file mode 100644 index 00000000000..f8b53525f0f --- /dev/null +++ b/apps/desktop/electron/first-run-setup-gate.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { createFirstRunSetupGate } from './first-run-setup-gate' + +const bootstrapBackend = { + activeRoot: '/tmp/hermes-home/hermes-agent', + kind: 'bootstrap-needed', + platform: 'linux' +} + +function delay(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +async function settledState(promise: Promise) { + return Promise.race([promise.then(() => 'resolved'), delay(10).then(() => 'pending')]) +} + +test('first-run setup gate skips non-bootstrap backends', async () => { + const prompts = [] + const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 }) + + await gate.wait({ kind: 'remote' }) + await gate.wait(null) + + assert.deepEqual(prompts, []) + assert.equal(gate.hasWaiter(), false) +}) + +test('first-run setup gate prompts once for concurrent waits', async () => { + const prompts = [] + const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 }) + + const first = gate.wait(bootstrapBackend) + const second = gate.wait(bootstrapBackend) + + assert.equal(gate.hasWaiter(), true) + assert.equal(prompts.length, 1) + assert.equal(await settledState(first), 'pending') + + gate.continueLocal() + + assert.deepEqual(await Promise.all([first, second]), ['continue-local', 'continue-local']) + assert.equal(gate.hasWaiter(), false) + assert.equal(gate.isLocalBootstrapConfirmed(), true) +}) + +test('continueLocal keeps the setup choice visible until bootstrap owns the overlay', async () => { + let hidden = 0 + const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 }) + const pending = gate.wait(bootstrapBackend) + + gate.continueLocal() + + assert.equal(await pending, 'continue-local') + assert.equal(hidden, 0) + assert.equal(gate.isLocalBootstrapConfirmed(), true) +}) + +test('retry reset preserves the local install confirmation', async () => { + const prompts = [] + const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 }) + + const pending = gate.wait(bootstrapBackend) + gate.continueLocal() + await pending + + gate.resetForRetry() + await gate.wait(bootstrapBackend) + + assert.equal(gate.isLocalBootstrapConfirmed(), true) + assert.equal(prompts.length, 1) + assert.equal(gate.hasWaiter(), false) +}) + +test('retry reset explicitly settles an active waiter without allowing local bootstrap', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + const pending = gate.wait(bootstrapBackend) + + gate.resetForRetry() + + assert.equal(await pending, 'reset') + assert.equal(gate.hasWaiter(), false) + assert.equal(gate.isLocalBootstrapConfirmed(), false) +}) + +test('repair reset clears the local install confirmation and shows the gate again', async () => { + const prompts = [] + const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 }) + + const pending = gate.wait(bootstrapBackend) + gate.continueLocal() + await pending + + gate.resetForRepair() + const next = gate.wait(bootstrapBackend) + + assert.equal(gate.isLocalBootstrapConfirmed(), false) + assert.equal(prompts.length, 2) + assert.equal(gate.hasWaiter(), true) + + gate.continueLocal() + await next +}) + +test('remote apply settles the gated boot for remote re-resolution and hides the choice', async () => { + let hidden = 0 + const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 }) + const pending = gate.wait(bootstrapBackend) + + const resumedWaiter = gate.abandonForRemoteApply() + + assert.equal(resumedWaiter, true) + assert.equal(hidden, 1) + assert.equal(gate.hasWaiter(), false) + assert.equal(gate.isLocalBootstrapConfirmed(), false) + assert.equal(await pending, 'remote-applied') +}) + +test('remote apply without a waiter has no first-run side effects', async () => { + let hidden = 0 + const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 }) + const pending = gate.wait(bootstrapBackend) + + gate.continueLocal() + await pending + + assert.equal(gate.abandonForRemoteApply(), false) + assert.equal(hidden, 0) + assert.equal(gate.isLocalBootstrapConfirmed(), true) +}) diff --git a/apps/desktop/electron/first-run-setup-gate.ts b/apps/desktop/electron/first-run-setup-gate.ts new file mode 100644 index 00000000000..2d83cbeafe6 --- /dev/null +++ b/apps/desktop/electron/first-run-setup-gate.ts @@ -0,0 +1,146 @@ +interface FirstRunSetupBackend { + activeRoot?: string + kind?: string + platform?: string +} + +interface FirstRunSetupGateOptions { + hideChoice?: () => void + log?: (message: string) => void + onStuck?: (backend: FirstRunSetupBackend, stuckAfterMs: number) => void + promptChoice?: (backend: FirstRunSetupBackend) => void + stuckAfterMs?: number +} + +export type FirstRunSetupDecision = 'continue-local' | 'remote-applied' | 'reset' + +export function createFirstRunSetupGate({ + hideChoice, + log, + onStuck, + promptChoice, + stuckAfterMs = 120000 +}: FirstRunSetupGateOptions = {}) { + let localBootstrapConfirmed = false + + let waiter: { + promise: Promise + resolve: (decision: FirstRunSetupDecision) => void + } | null = null + + let stuckTimer: ReturnType | null = null + + const clearStuckTimer = () => { + if (stuckTimer) { + clearTimeout(stuckTimer) + stuckTimer = null + } + } + + const armStuckTimer = (backend: FirstRunSetupBackend) => { + clearStuckTimer() + + if (!Number.isFinite(stuckAfterMs) || stuckAfterMs <= 0 || typeof log !== 'function') { + return + } + + stuckTimer = setTimeout(() => { + onStuck?.(backend, stuckAfterMs) + log( + `[bootstrap] still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)}s ` + + `(platform=${backend?.platform || 'unknown'})` + ) + }, stuckAfterMs) + + if (typeof stuckTimer.unref === 'function') { + stuckTimer.unref() + } + } + + const shouldGate = (backend?: FirstRunSetupBackend | null) => + Boolean(backend && backend.kind === 'bootstrap-needed' && !localBootstrapConfirmed) + + const wait = async (backend?: FirstRunSetupBackend | null) => { + if (!shouldGate(backend)) { + return 'continue-local' as const + } + + if (waiter) { + return waiter.promise + } + + promptChoice?.(backend) + armStuckTimer(backend) + + let resolveWaiter: (decision: FirstRunSetupDecision) => void = () => {} + + const promise = new Promise(resolve => { + resolveWaiter = resolve + }) + + waiter = { promise, resolve: resolveWaiter } + + return promise + } + + const settleWaiter = (decision: FirstRunSetupDecision) => { + clearStuckTimer() + + if (!waiter) { + return false + } + + const activeWaiter = waiter + waiter = null + activeWaiter.resolve(decision) + + return true + } + + const continueLocal = () => { + localBootstrapConfirmed = true + settleWaiter('continue-local') + } + + const resetForRetry = () => { + // Reset paths are followed by a renderer reload / fresh startHermes() call. + // Settle the old boot explicitly so it cannot fall through into local + // bootstrap and cannot leak a forever-pending connection promise. + settleWaiter('reset') + } + + const resetForRepair = () => { + resetForRetry() + localBootstrapConfirmed = false + } + + const abandonForRemoteApply = () => { + // Resume the gated startHermes() with an explicit remote decision. The + // caller re-resolves the newly-persisted remote config instead of falling + // through into local bootstrap or leaking the original connection promise. + const resumedWaiter = settleWaiter('remote-applied') + + if (!resumedWaiter) { + return false + } + + localBootstrapConfirmed = false + hideChoice?.() + + return true + } + + const isLocalBootstrapConfirmed = () => localBootstrapConfirmed + const hasWaiter = () => Boolean(waiter) + + return { + abandonForRemoteApply, + continueLocal, + hasWaiter, + isLocalBootstrapConfirmed, + resetForRepair, + resetForRetry, + shouldGate, + wait + } +} diff --git a/apps/desktop/electron/first-run-setup-main-process.test.ts b/apps/desktop/electron/first-run-setup-main-process.test.ts new file mode 100644 index 00000000000..1dc1b920ef6 --- /dev/null +++ b/apps/desktop/electron/first-run-setup-main-process.test.ts @@ -0,0 +1,119 @@ +import assert from 'node:assert/strict' + +import { test, vi } from 'vitest' + +import { applyConnectionChange } from './connection-apply' +import { createFirstRunSetupGate } from './first-run-setup-gate' +import { runPrimaryBackendStartup } from './primary-backend-startup' +import { rehomePrimaryConnection } from './primary-connection-rehome' + +test('a first-run bootstrap-needed remote apply connects without ensuring or bootstrapping locally', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + + const bootstrapBackend = { + activeRoot: '/tmp/hermes-home/hermes-agent', + kind: 'bootstrap-needed', + platform: 'linux' + } + + const candidateRemote = { + authMode: 'token', + baseUrl: 'https://gateway.example.com/hermes', + source: 'settings', + token: 'secret', + wsUrl: 'wss://gateway.example.com/hermes/api/ws?token=secret' + } + + let savedRemote: typeof candidateRemote | null = null + + const resolveRemote = vi.fn(async () => savedRemote) + const connectRemote = vi.fn(async remote => ({ ...remote, mode: 'remote' as const })) + const runBootstrap = vi.fn() + + const ensureLocalRuntime = vi.fn(async backend => { + await runBootstrap() + + return { ...backend, command: 'hermes' } + }) + + const teardownPrimaryBackend = vi.fn(async () => {}) + const cancelSshBootstrap = vi.fn(async () => {}) + const teardownSsh = vi.fn(async () => {}) + const clearLocalBootstrapFailure = vi.fn() + const notifyConnectionApplied = vi.fn() + const waitForLocalStart = vi.fn(async () => {}) + const prepareLocalBackend = vi.fn(async () => bootstrapBackend) + + const pendingConnection = runPrimaryBackendStartup({ + connectRemote, + ensureLocalRuntime, + prepareLocalBackend, + resolveRemote, + waitForDecision: gate.wait, + waitForLocalStart + }) + + await vi.waitFor(() => assert.equal(gate.hasWaiter(), true)) + + // Mirrors the IPC handler's production ordering: persist the tested config, + // then re-home. The pending start must re-resolve this saved value. + savedRemote = candidateRemote + + await applyConnectionChange({ + cancelAndWait: cancelSshBootstrap, + isPrimary: true, + rehomePrimary: () => + rehomePrimaryConnection({ + clearLocalBootstrapFailure, + mode: 'remote', + notifyConnectionApplied, + resumeFirstRunRemote: gate.abandonForRemoteApply, + teardownPrimaryBackend + }), + scope: '', + sendApplied: notifyConnectionApplied, + stopPool: vi.fn(), + teardownPrimary: teardownPrimaryBackend, + teardownSsh + }) + + assert.deepEqual(await pendingConnection, { + kind: 'remote', + connection: { ...candidateRemote, mode: 'remote' } + }) + assert.deepEqual(resolveRemote.mock.calls, [[], []]) + assert.deepEqual(connectRemote.mock.calls, [[candidateRemote]]) + assert.deepEqual(waitForLocalStart.mock.calls, [[]]) + assert.deepEqual(prepareLocalBackend.mock.calls, [[]]) + assert.equal(ensureLocalRuntime.mock.calls.length, 0) + assert.equal(runBootstrap.mock.calls.length, 0) + assert.deepEqual(cancelSshBootstrap.mock.calls, [['']]) + assert.deepEqual(teardownSsh.mock.calls, [['']]) + assert.equal(teardownPrimaryBackend.mock.calls.length, 0) + assert.equal(clearLocalBootstrapFailure.mock.calls.length, 1) + assert.equal(notifyConnectionApplied.mock.calls.length, 0) +}) + +test('a primary apply without an active first-run gate tears down before reconnect notification', async () => { + const order: string[] = [] + const clearLocalBootstrapFailure = vi.fn(() => order.push('clear-failure')) + + const teardownPrimaryBackend = vi.fn(async () => { + order.push('teardown') + }) + + const notifyConnectionApplied = vi.fn(() => order.push('notify')) + + assert.deepEqual( + await rehomePrimaryConnection({ + clearLocalBootstrapFailure, + mode: 'remote', + notifyConnectionApplied, + resumeFirstRunRemote: () => false, + teardownPrimaryBackend + }), + { resumedFirstRunRemote: false } + ) + assert.deepEqual(teardownPrimaryBackend.mock.calls, [[{ soft: true }]]) + assert.deepEqual(order, ['clear-failure', 'teardown', 'notify']) +}) diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 3856cd2abc2..f967aa24c3b 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -34,9 +34,10 @@ import { stopBackendChild as stopBackendChildImpl } from './backend-child' import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command' import { createBackendConnectionState } from './backend-connection-state' import { buildDesktopBackendEnv, normalizeHermesHomeRoot } from './backend-env' +import { isReauthRequiredError, waitForHermesReady } from './backend-health' import { canImportHermesCli, shouldTrustHermesOverride, verifyHermesCli } from './backend-probes' import { waitForDashboardPortAnnouncement } from './backend-ready' -import { shouldLatchBackendStartFailure } from './backend-start-failure' +import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure' import { detectRemoteDisplay, isWindowsBinaryPathInWsl, isWslEnvironment } from './bootstrap-platform' import { runBootstrap } from './bootstrap-runner' import { applyConnectionChange, resolveTerminalConnection } from './connection-apply' @@ -78,6 +79,8 @@ import { } from './desktop-uninstall' import { installEmbedReferer } from './embed-referer' import { createEventDeduper } from './event-dedupe' +import { findGitBash as _findGitBash } from './find-git-bash' +import { createFirstRunSetupGate } from './first-run-setup-gate' import { readDirForIpc } from './fs-read-dir' import { probeGatewayWebSocket } from './gateway-ws-probe' import { scanGitRepos } from './git-repo-scan' @@ -116,7 +119,13 @@ import { } from './hardening' import { createLinkTitleWindow, guardLinkTitleSession, readLinkTitleWindowTitle } from './link-title-window' import { ensureMainWindow } from './main-window-lifecycle' -import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions' +import { + oauthGuardMayHardFail, + oauthSessionIsLive, + resolveJsonBody, + resolveOauthRestAuth, + resolveReadinessProbeAuth +} from './native-auth-decisions' import { nativeRefreshUrl, type NativeTokenSet, @@ -127,6 +136,8 @@ import { import { runNativeLogin } from './native-oauth-login' import { serializeJsonBody, setJsonRequestHeaders } from './oauth-net-request' import { createKeepAwake } from './power-save' +import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup' +import { rehomePrimaryConnection } from './primary-connection-rehome' import { decideProfileDeleteAction, profileNameFromDeleteRequest, resolveRouteProfile } from './profile-delete-routing' import * as remoteLifecycle from './remote-lifecycle' import { RemoteLivenessTracker, RemoteRevalidationCoordinator, revalidateRemoteConnection } from './remote-liveness' @@ -1011,6 +1022,13 @@ let bootstrapFailure = null // Latched non-bootstrap backend spawn failure — stops getConnection() from // respawning hermes serve backend children in a tight loop while boot is broken. let backendStartFailure = null +// Latched CONFIRMED remote reauth failure. Remote failures deliberately do not +// latch via backendStartFailure (they're usually transient and must stay +// retryable), but a rejected session cannot self-heal — and the non-latching +// path actively breaks recovery: each retry re-emits running:true and hides +// the boot-failure overlay, so the "Sign in" button flickers away before it +// can be clicked. Cleared on every recovery path and on a confirmed sign-in. +let remoteReauthFailure = null // Active first-launch install, so the renderer's Cancel button (and app quit) // can abort the in-flight install.sh/ps1 instead of leaving it running. let bootstrapAbortController = null @@ -1398,13 +1416,17 @@ let bootstrapState = { log: [], startedAt: null, completedAt: null, + setupChoice: null, unsupportedPlatform: null } +let firstRunSetupGate = null + function broadcastBootstrapEvent(ev) { if (ev.type === 'manifest') { bootstrapState.manifest = ev bootstrapState.active = true + bootstrapState.setupChoice = null bootstrapState.startedAt = bootstrapState.startedAt || Date.now() bootstrapState.stages = {} @@ -1432,14 +1454,30 @@ function broadcastBootstrapEvent(ev) { } else if (ev.type === 'failed') { bootstrapState.active = false bootstrapState.error = ev.error || 'unknown error' + bootstrapState.setupChoice = null } else if (ev.type === 'unsupported-platform') { bootstrapState.active = false + bootstrapState.setupChoice = null bootstrapState.unsupportedPlatform = { platform: ev.platform, activeRoot: ev.activeRoot, installCommand: ev.installCommand, docsUrl: ev.docsUrl } + } else if (ev.type === 'setup-choice') { + bootstrapState.active = false + bootstrapState.error = null + bootstrapState.manifest = null + bootstrapState.stages = {} + bootstrapState.setupChoice = ev.active + ? { + platform: ev.platform, + activeRoot: ev.activeRoot + } + : null + bootstrapState.unsupportedPlatform = null + } else if (ev.type === 'dismissed') { + resetBootstrapSnapshot() } if (!mainWindow || mainWindow.isDestroyed()) { @@ -1459,6 +1497,100 @@ function getBootstrapState() { return bootstrapState } +function resetBootstrapSnapshot() { + bootstrapState = { + active: false, + manifest: null, + stages: {}, + error: null, + log: [], + startedAt: null, + completedAt: null, + setupChoice: null, + unsupportedPlatform: null + } +} + +function promptFirstRunSetupChoice(backend) { + broadcastBootstrapEvent({ + type: 'setup-choice', + active: true, + platform: backend.platform || process.platform, + activeRoot: backend.activeRoot || ACTIVE_HERMES_ROOT + }) +} + +function hideFirstRunSetupChoice() { + if (bootstrapState.setupChoice) { + broadcastBootstrapEvent({ type: 'setup-choice', active: false }) + } +} + +function getFirstRunSetupGate() { + if (!firstRunSetupGate) { + firstRunSetupGate = createFirstRunSetupGate({ + hideChoice: hideFirstRunSetupChoice, + log: rememberLog, + onStuck: (_backend, stuckAfterMs) => { + updateBootProgress( + { + error: null, + message: `Still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)} seconds`, + phase: 'bootstrap.choice', + progress: 12, + running: true + }, + { allowDecrease: true } + ) + }, + promptChoice: promptFirstRunSetupChoice + }) + } + + return firstRunSetupGate +} + +async function waitForFirstRunSetupChoice(backend) { + const gate = getFirstRunSetupGate() + + if (!gate.shouldGate(backend)) { + return 'continue-local' + } + + updateBootProgress( + { + error: null, + message: 'Waiting for first-run setup choice', + phase: 'bootstrap.choice', + progress: 12, + running: true + }, + { allowDecrease: true } + ) + + return gate.wait(backend) +} + +function continueFirstRunLocalBootstrap() { + getFirstRunSetupGate().continueLocal() +} + +function abandonFirstRunSetupChoiceForRemoteApply() { + const gate = getFirstRunSetupGate() + + if (!gate.hasWaiter()) { + return false + } + + const resumedGatedConnection = gate.abandonForRemoteApply() + + if (resumedGatedConnection) { + broadcastBootstrapEvent({ type: 'dismissed' }) + } + + return resumedGatedConnection +} + function updateBootProgress(update, options: { allowDecrease?: boolean } = {}) { const nextProgressRaw = typeof update.progress === 'number' ? clampBootProgress(update.progress) : bootProgressState.progress @@ -1915,48 +2047,16 @@ function findSystemPython() { return null } -// findGitBash — locate bash.exe on Windows. Hermes' terminal tool requires -// bash (POSIX shell), and on Windows that's almost always Git for Windows' -// bundled Git Bash. We check the same set of locations tools/environments/ -// local.py:_find_bash() checks at runtime, so a positive result here means -// the agent will be able to start a terminal too. -// -// On non-Windows hosts bash is part of the OS and this just returns the -// first bash on PATH. +// findGitBash — locate bash.exe on Windows. Resolves HERMES_GIT_BASH_PATH +// first (mirrors tools/environments/local.py:_find_bash), then PortableGit, +// standard install locations, and finally PATH. function findGitBash() { - if (!IS_WINDOWS) { - return findOnPath('bash') - } - - // install.ps1 drops PortableGit at %LOCALAPPDATA%\hermes\git\... — checked - // first so users who installed via install.ps1 are detected before we - // start probing system-wide locations. - const localAppData = process.env.LOCALAPPDATA || '' - const candidates = [] - - if (localAppData) { - candidates.push(path.join(localAppData, 'hermes', 'git', 'bin', 'bash.exe')) - candidates.push(path.join(localAppData, 'hermes', 'git', 'usr', 'bin', 'bash.exe')) - } - - // Standard Git for Windows install locations. - candidates.push(path.join(process.env['ProgramFiles'] || 'C:\\Program Files', 'Git', 'bin', 'bash.exe')) - candidates.push(path.join(process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Git', 'bin', 'bash.exe')) - - if (localAppData) { - candidates.push(path.join(localAppData, 'Programs', 'Git', 'bin', 'bash.exe')) - } - - for (const candidate of candidates) { - if (fileExists(candidate)) { - return candidate - } - } - - // Last resort — bash on PATH (covers WSL bash, MSYS2, custom installs). - // On WSL hosts findOnPath itself filters out Windows-binary paths via - // isWindowsBinaryPathInWsl, so we won't hand back a wsl.exe shim either. - return findOnPath('bash') + return _findGitBash({ + isWindows: IS_WINDOWS, + env: process.env, + fileExists, + findOnPath + }) } function getVenvPython(venvRoot) { @@ -2686,6 +2786,11 @@ async function applyUpdates(opts = {}) { const venvBin = path.join(updateRoot, 'venv', IS_WINDOWS ? 'Scripts' : 'bin') + // ── Pre-flight state.db integrity guard (#68474) ───────────────── + // Emergency backup and header verification before the update touches + // anything. Runs while the backend is still alive. + preflightStateDb(HERMES_HOME, rememberLog) + // Stop our own backend(s) and wait for the venv shim to unlock BEFORE we // spawn the updater. Without this the updater races a still-locked // hermes.exe (held by the backend child / its grandchildren) and the update @@ -2883,6 +2988,92 @@ function runningAppBundle() { return dir.endsWith('.app') ? dir : null } +// ── Pre-flight state.db integrity guard (#68474) ───────────────────── +// Take an emergency snapshot of state.db and verify the live copy is +// intact before any update process mutates the install. Runs in the +// desktop Electron process itself, before the backend is killed and +// before the updater is spawned — a separate safety net from the +// Python-level pre-update snapshot inside `hermes update`. +function preflightStateDb(hermesHome, rememberLog) { + const stateDbPath = path.join(hermesHome, 'state.db') + + if (!fileExists(stateDbPath)) { + rememberLog('[updates] state.db pre-flight: not found (fresh install?)') + + return + } + + try { + const stat = fs.statSync(stateDbPath) + + if (stat.size > 100) { + const fd = fs.openSync(stateDbPath, 'r') + const header = Buffer.alloc(16) + + fs.readSync(fd, header, 0, 16, 0) + fs.closeSync(fd) + + const expectedHeader = Buffer.from('SQLite format 3\0') + const headerOk = header.equals(expectedHeader) + + rememberLog( + `[updates] state.db pre-flight: size=${stat.size}, ` + + `headerOk=${headerOk}, headerHex=${header.toString('hex')}` + ) + + if (!headerOk) { + rememberLog( + '[updates] state.db header is INVALID before update — ' + + 'this indicates pre-existing corruption or a concurrent write issue' + ) + } + + // Emergency timestamped backup, separate from the Python-level snapshot. + const ts = new Date().toISOString().replace(/[:.]/g, '-') + + const emergencyPath = path.join(hermesHome, `state.db.pre-update-emergency-${ts}.bak`) + + try { + fs.copyFileSync(stateDbPath, emergencyPath) + const emergStat = fs.statSync(emergencyPath) + + rememberLog(`[updates] emergency state.db backup: ${emergencyPath} ` + `(${emergStat.size} bytes)`) + + // Prune to the 2 most recent emergency backups. + try { + const homeDir = fs.readdirSync(hermesHome) + + const backups = homeDir + .filter( + f => + f.startsWith('state.db.pre-update-emergency-') && + f.endsWith('.bak') && + f !== path.basename(emergencyPath) + ) + .sort() + .reverse() + + for (const old of backups.slice(2)) { + try { + fs.unlinkSync(path.join(hermesHome, old)) + } catch { + void 0 + } + } + } catch { + void 0 + } + } catch (copyErr) { + rememberLog(`[updates] emergency state.db backup failed: ${copyErr.message}`) + } + } else { + rememberLog(`[updates] state.db too small (${stat.size} bytes) for a valid SQLite database`) + } + } catch (statErr) { + rememberLog(`[updates] could not stat state.db before update: ${statErr.message}`) + } +} + function shellQuote(value) { return `'${String(value).replace(/'/g, `'\\''`)}'` } @@ -2901,9 +3092,12 @@ async function applyUpdatesPosixInApp(opts: any) { return { ok: true, manual: true, command: 'hermes update', hermesRoot: updateRoot } } + // ── Pre-flight state.db integrity guard (#68474) ── + preflightStateDb(HERMES_HOME, rememberLog) + // Put the Hermes-managed Node and the venv on PATH so `hermes desktop`'s // npm build can find them on a machine with no system Node. Windows portable - // Node lives directly under %LOCALAPPDATA%\hermes\node, not node\bin. + // Node lives directly under %LOCALAPPDATA%\\hermes\\node, not node\\bin. // PYTHONUNBUFFERED: `hermes update` writes to a pipe here, so CPython // block-buffers stdout and long quiet steps (the pre-update backup can zip // multi-GB archives for minutes) stream nothing to the progress UI — users @@ -4634,40 +4828,88 @@ function closePreviewWatchers() { } } -async function waitForHermes(baseUrl, token, signal?) { - const deadline = Date.now() + 45_000 - let lastError = null +// Best-effort read of a gateway's advertised auth providers, cached per base +// URL for the life of the process. Used by the oauth pre-flight guard to tell +// a password-provider gateway (which cannot satisfy the bearer/cookie checks +// by design) from a real OAuth one. Any failure returns [] so callers keep the +// strict guard — backends predating /api/auth/providers are unaffected. +const gatewayAuthProvidersCache = new Map() - while (Date.now() < deadline) { - if (signal?.aborted) { - const error: any = new Error('SSH bootstrap was superseded by newer connection settings.') - error.kind = 'superseded' - throw error +async function gatewayAuthProviders(baseUrl) { + const cached = gatewayAuthProvidersCache.get(baseUrl) + + if (cached) { + return cached + } + + let providers = [] + + try { + const body = (await fetchPublicJson(`${baseUrl}/api/auth/providers`, { timeoutMs: 8_000 })) as any + + if (Array.isArray(body?.providers)) { + providers = body.providers + .filter(p => p && typeof p === 'object') + .map(p => ({ name: String(p.name || ''), supportsPassword: Boolean(p.supports_password) })) + .filter(p => p.name) } + } catch { + // Optional metadata — an unreadable list keeps the strict guard. + } - try { - await fetchJson(`${baseUrl}/api/status`, token) + gatewayAuthProvidersCache.set(baseUrl, providers) - return - } catch (error) { - lastError = error - await new Promise((resolve, reject) => { - const timer = setTimeout(resolve, 500) - signal?.addEventListener( - 'abort', - () => { - clearTimeout(timer) - const aborted: any = new Error('SSH bootstrap was superseded by newer connection settings.') - aborted.kind = 'superseded' - reject(aborted) - }, - { once: true } - ) - }) + return providers +} + +// Build the readiness probe for a connection's auth mode. A gated gateway +// must be probed with the SAME credentials the rest of the connection uses: +// an anonymous probe 401s forever against a live session, and it can never +// see the 404 that identifies a backend predating /api/health (the auth gate +// answers before the SPA catch-all). `probeIsCredentialed` tells +// waitForHermesReady how to read a 401 — rejected session vs gated route. +async function buildReadinessHealthProbe(baseUrl, authMode, token) { + const nativeAt = authMode === 'oauth' ? await ensureNativeAccessToken(baseUrl).catch(() => null) : null + const probeAuth = resolveReadinessProbeAuth(authMode, nativeAt, token) + + if (probeAuth.kind === 'bearer') { + return { + // fetchJson takes the bearer via `options.bearer` — a raw `headers` + // option is ignored, so passing one here would silently probe + // uncredentialed and reintroduce the 401 loop. + probeHealth: (url, options: any = {}) => fetchJson(url, null, { ...options, bearer: probeAuth.token }), + probeIsCredentialed: true } } - throw new Error(`Hermes backend did not become ready: ${lastError?.message || 'timeout'}`) + if (probeAuth.kind === 'cookie') { + return { + probeHealth: (url, options: any = {}) => fetchJsonViaOauthSession(url, options), + probeIsCredentialed: true + } + } + + if (probeAuth.kind === 'token' && probeAuth.token) { + return { + probeHealth: (url, options: any = {}) => fetchJson(url, probeAuth.token, options), + probeIsCredentialed: true + } + } + + return { probeHealth: fetchPublicJson, probeIsCredentialed: false } +} + +async function waitForHermes(baseUrl, token, signal?, authMode?) { + const { probeHealth, probeIsCredentialed } = await buildReadinessHealthProbe(baseUrl, authMode, token) + + return waitForHermesReady(baseUrl, { + token, + signal, + fetchPublicJson, + fetchJson: probeIsCredentialed ? (url, _token, options) => probeHealth(url, options) : fetchJson, + probeHealth, + probeIsCredentialed + }) } function getWindowButtonPosition() { @@ -6668,7 +6910,10 @@ async function buildRemoteConnection( // here would reject a freshly-completed native sign-in and loop the UI back // into "not signed in" even though mintGatewayWsTicket would succeed with // the stored bearer. - if (!oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl))) { + if ( + !oauthSessionIsLive(hasNativeSession(baseUrl), await hasLiveOauthSession(baseUrl)) && + oauthGuardMayHardFail(await gatewayAuthProviders(baseUrl)) + ) { const err = new Error( 'Remote Hermes gateway uses OAuth, but you are not signed in. ' + 'Open Settings → Gateway and click "Sign in", or switch back to Local.' @@ -6909,7 +7154,7 @@ async function bootstrapSshConnectionInner(profile, sshConfig, reuseToken, sourc forward: (localPort, remotePort) => ssh.forward(localPort, remotePort), cancelForward: (localPort, remotePort) => ssh.cancelForward(localPort, remotePort), pickLocalPort, - waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal), + waitForHermes: (baseUrl, token) => waitForHermes(baseUrl, token, lease.signal, 'token'), probeReuseProof: sshProbeReuseProof, adoptServedToken: adoptServedDashboardToken, rememberLog: sshRememberLog, @@ -7382,6 +7627,7 @@ function stopBackendChild(child) { // switch / crash recovery), which still resets boot progress + reloads. function resetHermesConnection({ soft = false } = {}) { backendStartFailure = null + remoteReauthFailure = null remoteLiveness.clear() const hermesProcess = backendConnectionState.invalidate() stopBackendChild(hermesProcess) @@ -7582,7 +7828,7 @@ async function spawnPoolBackend(profile, entry) { const remote = await resolveRemoteBackend(profile) if (remote) { - await waitForHermes(remote.baseUrl, remote.token) + await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode) return { ...remote, @@ -7774,6 +8020,13 @@ async function startHermes() { throw backendStartFailure } + // A confirmed remote reauth rejection is terminal until the user signs in. + // Short-circuiting here keeps the boot-failure overlay latched and its + // "Sign in" button clickable, instead of re-driving boot on every retry. + if (remoteReauthFailure) { + throw remoteReauthFailure + } + // E2E: simulate a boot failure without breaking the real backend. The boot // progresses a few steps, then fails with the given error message. if (BOOT_FAKE_ERROR) { @@ -7798,16 +8051,9 @@ async function startHermes() { let attemptedRemote = primaryBackendIsRemote() const connectionPromise = (async () => { - await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8) - // Resolve for the desktop's primary profile so a per-profile remote - // override on the active profile is honored (falls back to env / global). - // Re-read once resolved so the classification tracks the value actually used. - attemptedRemote = primaryBackendIsRemote() - const remote = await resolveRemoteBackend(primaryProfileKey()) - - if (remote) { + const connectRemote = async remote => { await advanceBootProgress('backend.remote', `Connecting to remote Hermes backend at ${remote.baseUrl}`, 24) - await waitForHermes(remote.baseUrl, remote.token) + await waitForHermes(remote.baseUrl, remote.token, undefined, remote.authMode) updateBootProgress({ phase: 'backend.ready', message: 'Remote Hermes backend is ready', @@ -7831,14 +8077,9 @@ async function startHermes() { } } - // Mutual exclusion with an in-app update (#50238). If this instance was - // relaunched while the Tauri updater is still applying an update, spawning - // a local backend now re-locks the venv shim and gets killed by the - // updater's straggler cleanup — looping. Park until the update finishes (or - // is detected stale), THEN start the backend. Local backends only; remote - // connections returned above and never touch the install tree. - await waitForUpdateToFinish() - + await advanceBootProgress('backend.resolve', 'Resolving Hermes backend', 8) + // Resolve for the desktop's primary profile so a per-profile remote + // override on the active profile is honored (falls back to env / global). const token = crypto.randomBytes(32).toString('base64url') // --port 0: the OS assigns an ephemeral port; the child announces it on stdout. const backendArgs = ['serve', '--host', '127.0.0.1', '--port', '0'] @@ -7853,8 +8094,32 @@ async function startHermes() { backendArgs.unshift('--profile', activeProfile) } - await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28) - const backend = await ensureRuntime(resolveHermesBackend(backendArgs)) + const setup = await runPrimaryBackendStartup({ + connectRemote, + ensureLocalRuntime: ensureRuntime, + prepareLocalBackend: async () => { + await advanceBootProgress('backend.runtime', 'Resolving Hermes runtime', 28) + + return resolveHermesBackend(backendArgs) + }, + resolveRemote: () => { + // Classify immediately before each throwing resolve. This callback runs + // both for an already-saved remote and after first-run remote Apply. + attemptedRemote = primaryBackendIsRemote() + + return resolveRemoteBackend(primaryProfileKey()) + }, + waitForDecision: waitForFirstRunSetupChoice, + // Mutual exclusion with an in-app update (#50238). Remote connections + // return before this waiter; local starts park until the updater exits. + waitForLocalStart: waitForUpdateToFinish + }) + + if (setup.kind === 'remote') { + return setup.connection + } + + const backend = setup.backend // Route old runtimes (no `serve`) through the legacy `dashboard --no-open`. backend.args = getBackendArgsForRuntime(backend) const hermesCwd = resolveHermesCwd() @@ -8010,6 +8275,10 @@ async function startHermes() { throw error } + if (error instanceof FirstRunSetupResetError) { + throw error + } + const message = error instanceof Error ? error.message : String(error) // Only latch LOCAL boot failures. A remote failure (lapsed session / mint @@ -8021,6 +8290,12 @@ async function startHermes() { backendStartFailure = error instanceof Error ? error : new Error(message) } + // A confirmed reauth rejection latches separately: it can't self-heal, and + // leaving it unlatched hides the overlay's "Sign in" button on every retry. + if (shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) })) { + remoteReauthFailure = error instanceof Error ? error : new Error(message) + } + updateBootProgress( { error: message, @@ -8824,16 +9099,9 @@ ipcMain.handle('hermes:bootstrap:reset', async () => { await teardownPrimaryBackendAndWait() bootstrapFailure = null backendStartFailure = null - bootstrapState = { - active: false, - manifest: null, - stages: {}, - error: null, - log: [], - startedAt: null, - completedAt: null, - unsupportedPlatform: null - } + remoteReauthFailure = null + getFirstRunSetupGate().resetForRetry() + resetBootstrapSnapshot() return { ok: true } }) @@ -8854,10 +9122,18 @@ ipcMain.handle('hermes:bootstrap:repair', async () => { bootstrapFailure = null backendStartFailure = null + remoteReauthFailure = null + getFirstRunSetupGate().resetForRepair() resetHermesConnection() return { ok: true } }) +ipcMain.handle('hermes:bootstrap:continue-local', async () => { + rememberLog('[bootstrap] local install selected by renderer; continuing first-launch bootstrap') + continueFirstRunLocalBootstrap() + + return { ok: true } +}) ipcMain.handle('hermes:bootstrap:cancel', async () => { // Renderer's Cancel button during first-launch install. Abort the running // install script (SIGTERM via the runner's abortSignal). runBootstrap @@ -8956,6 +9232,9 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) => }) _storeNativeTokens(baseUrl, tokens) + // Confirmed sign-in — release the reauth latch so the next + // startHermes() re-dials instead of replaying the stale rejection. + remoteReauthFailure = null return { ok: true, baseUrl, connected: true } } catch (error) { @@ -8972,7 +9251,16 @@ ipcMain.handle('hermes:connection-config:oauth-login', async (_event, rawUrl) => // Legacy embedded-webview cookie flow. await openOauthLoginWindow(baseUrl) - return { ok: true, baseUrl, connected: await hasOauthSessionCookie(baseUrl) } + const connected = await hasOauthSessionCookie(baseUrl) + + // Only a CONFIRMED sign-in releases the latch. A cancelled/closed login + // window must leave it set, or the overlay's "Sign in" button starts + // flickering again on the next retry. + if (connected) { + remoteReauthFailure = null + } + + return { ok: true, baseUrl, connected } }) ipcMain.handle('hermes:connection-config:oauth-logout', async (_event, rawUrl) => { const baseUrl = rawUrl ? normalizeRemoteBaseUrl(rawUrl) : '' @@ -9036,6 +9324,18 @@ ipcMain.handle('hermes:connection-config:apply', async (_event, payload) => { await applyConnectionChange({ cancelAndWait: value => sshBootstrapCoordinator.cancelAndWait(value), isPrimary: !key || key === primaryProfileKey(), + rehomePrimary: () => + rehomePrimaryConnection({ + clearLocalBootstrapFailure: () => { + // A remote connection bypasses local runtime/bootstrap failures. Clear + // the local-install latch so unsupported/failure escape paths can re-home. + bootstrapFailure = null + }, + mode: config.mode, + notifyConnectionApplied: sendConnectionApplied, + resumeFirstRunRemote: abandonFirstRunSetupChoiceForRemoteApply, + teardownPrimaryBackend: teardownPrimaryBackendAndWait + }), scope, sendApplied: sendConnectionApplied, stopPool: stopPoolBackend, diff --git a/apps/desktop/electron/native-auth-decisions.test.ts b/apps/desktop/electron/native-auth-decisions.test.ts index 09f648c8719..d4cfc068cd6 100644 --- a/apps/desktop/electron/native-auth-decisions.test.ts +++ b/apps/desktop/electron/native-auth-decisions.test.ts @@ -1,7 +1,7 @@ /** - * Regression tests for electron/native-auth-decisions.ts — the three pure - * decision seams behind the RFC 8252 native-app auth flow, each of which was a - * real runtime bug that the mocked flow tests could not catch. + * Regression tests for electron/native-auth-decisions.ts — the pure decision + * seams behind the RFC 8252 native-app auth flow, each of which was a real + * runtime bug that the mocked flow tests could not catch. * * Run via the vitest `electron` project (electron/**\/*.test.ts). */ @@ -10,7 +10,13 @@ import assert from 'node:assert/strict' import { test } from 'vitest' -import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions' +import { + oauthGuardMayHardFail, + oauthSessionIsLive, + resolveJsonBody, + resolveOauthRestAuth, + resolveReadinessProbeAuth +} from './native-auth-decisions' // --- 1. body encoding (guards the double-JSON.stringify 422) --- @@ -66,3 +72,61 @@ test('resolveOauthRestAuth falls back to cookie when there is no native token', // Empty string is not a usable bearer — must fall back, not send "Bearer ". assert.deepEqual(resolveOauthRestAuth(''), { kind: 'cookie' }) }) + +// --- 4. readiness-probe auth (guards the credential-free 401 boot loop) --- + +test('resolveReadinessProbeAuth reuses the oauth bearer-vs-cookie choice', () => { + assert.deepEqual(resolveReadinessProbeAuth('oauth', 'native-at'), { kind: 'bearer', token: 'native-at' }) + assert.deepEqual(resolveReadinessProbeAuth('oauth', null), { kind: 'cookie' }) + assert.deepEqual(resolveReadinessProbeAuth('oauth', ''), { kind: 'cookie' }) +}) + +test('resolveReadinessProbeAuth sends the session token for a token gateway', () => { + assert.deepEqual(resolveReadinessProbeAuth('token', null, 'session-token'), { + kind: 'token', + token: 'session-token' + }) + assert.deepEqual(resolveReadinessProbeAuth('token', null, null), { kind: 'token', token: null }) +}) + +test('resolveReadinessProbeAuth stays public for local and unknown modes', () => { + // A loopback backend has no gate; sending credentials it never issued is + // meaningless, and an unknown mode must not invent a credential. + assert.deepEqual(resolveReadinessProbeAuth('local', 'native-at', 'session-token'), { kind: 'public' }) + assert.deepEqual(resolveReadinessProbeAuth(undefined, 'native-at', 'session-token'), { kind: 'public' }) + assert.deepEqual(resolveReadinessProbeAuth('something-new', null, null), { kind: 'public' }) +}) + +// --- 5. oauth guard vs password gateways (guards the false "not signed in") --- + +test('oauthGuardMayHardFail is false only when EVERY provider is password-based', () => { + assert.equal(oauthGuardMayHardFail([{ name: 'basic', supportsPassword: true }]), false) + assert.equal( + oauthGuardMayHardFail([ + { name: 'basic', supportsPassword: true }, + { name: 'ldap', supportsPassword: true } + ]), + false + ) +}) + +test('oauthGuardMayHardFail keeps the strict guard for oauth and mixed deployments', () => { + assert.equal(oauthGuardMayHardFail([{ name: 'nous', supportsPassword: false }]), true) + assert.equal( + oauthGuardMayHardFail([ + { name: 'nous', supportsPassword: false }, + { name: 'basic', supportsPassword: true } + ]), + true + ) +}) + +test('oauthGuardMayHardFail keeps the strict guard when the list is unusable', () => { + // Backends predating /api/auth/providers, or an unreachable probe, must not + // silently weaken the guard. + assert.equal(oauthGuardMayHardFail([]), true) + assert.equal(oauthGuardMayHardFail(null), true) + assert.equal(oauthGuardMayHardFail(undefined), true) + assert.equal(oauthGuardMayHardFail('nonsense' as any), true) + assert.equal(oauthGuardMayHardFail([{ supportsPassword: true }]), true) +}) diff --git a/apps/desktop/electron/native-auth-decisions.ts b/apps/desktop/electron/native-auth-decisions.ts index d76746f669a..c0978c3ecdd 100644 --- a/apps/desktop/electron/native-auth-decisions.ts +++ b/apps/desktop/electron/native-auth-decisions.ts @@ -21,7 +21,17 @@ * native bearer when present, else the cookie partition. Cookie-only * routing returns 401 no_cookie for a cookieless native session. * - * All three are trivial once named; the value is the test that pins the + * 4. resolveReadinessProbeAuth — the boot readiness probe must authenticate + * the same way the rest of the connection does. A credential-free probe + * against a gated gateway 401s forever; worse, it cannot tell a missing + * route from a rejected session (see backend-health.ts). + * + * 5. oauthGuardMayHardFail — `auth_required: true` means "this gateway is + * gated", NOT "this gateway speaks OAuth". A password-provider gateway + * can satisfy neither the native-bearer nor the OAuth-partition-cookie + * check by design, so the pre-flight guard must not hard-fail it. + * + * All five are trivial once named; the value is the test that pins the * contract so the god-file call sites can't drift back to the buggy shape. */ @@ -60,3 +70,75 @@ export function resolveOauthRestAuth(nativeAccessToken: string | null | undefine return { kind: 'cookie' } } + +export type ReadinessProbeAuth = OauthRestAuth | { kind: 'token'; token: string | null } | { kind: 'public' } + +/** + * Decide how the boot readiness probe authenticates. + * + * The probe must present the SAME credentials the rest of the connection + * will use. A credential-free probe against a gated gateway 401s until the + * boot deadline even though the session is perfectly valid — and because the + * dashboard auth gate runs ahead of the SPA catch-all, an unknown `/api/*` + * path answers 401 rather than 404, so the probe also cannot detect a backend + * that predates `/api/health`. Sending credentials is what lets a missing + * route surface as a real 404 (see `isMissingHealthEndpointError`). + * + * `oauth` reuses `resolveOauthRestAuth` so the probe and every other oauth + * REST call make the identical bearer-vs-cookie choice. `token` presents the + * connection's session token. `local` (and anything unrecognized) stays + * public: a loopback backend has no gate, and sending credentials it never + * issued would be meaningless. + */ +export function resolveReadinessProbeAuth( + authMode: string | null | undefined, + nativeAccessToken?: string | null, + connectionToken?: string | null +): ReadinessProbeAuth { + if (authMode === 'oauth') { + return resolveOauthRestAuth(nativeAccessToken) + } + + if (authMode === 'token') { + return { kind: 'token', token: connectionToken ?? null } + } + + return { kind: 'public' } +} + +export interface AdvertisedAuthProvider { + name?: string + supportsPassword?: boolean +} + +/** + * Whether the oauth pre-flight guard may hard-fail a connection for "not + * signed in". + * + * `authModeFromStatus` maps the gateway's `auth_required: true` onto + * `'oauth'`, but that flag only means the dashboard is GATED — it says + * nothing about how you authenticate. A gateway whose providers are all + * username/password cannot satisfy the guard's checks by construction: + * `start_login` raises NotImplementedError, `/auth/native/authorize` rejects + * password providers, and its cookies are set by a plain password-login POST + * rather than the `/auth/callback` redirect the OAuth partition is primed + * for. Hard-failing there rejects a live session one line before the + * ws-ticket mint that would have succeeded against that very partition. + * + * Returns false only when EVERY advertised provider is password-based. An + * unknown or empty list keeps the strict guard, so backends that predate + * `/api/auth/providers` are unaffected. + */ +export function oauthGuardMayHardFail(providers: AdvertisedAuthProvider[] | null | undefined): boolean { + if (!Array.isArray(providers) || providers.length === 0) { + return true + } + + const named = providers.filter(provider => provider && typeof provider === 'object' && provider.name) + + if (named.length === 0) { + return true + } + + return !named.every(provider => provider.supportsPassword) +} diff --git a/apps/desktop/electron/preload.ts b/apps/desktop/electron/preload.ts index 7652a7688dd..47d854a1938 100644 --- a/apps/desktop/electron/preload.ts +++ b/apps/desktop/electron/preload.ts @@ -237,6 +237,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', { // current snapshot via getBootstrapState() to recover after a devtools // reload mid-bootstrap. getBootstrapState: () => ipcRenderer.invoke('hermes:bootstrap:get'), + continueBootstrapLocal: () => ipcRenderer.invoke('hermes:bootstrap:continue-local'), resetBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:reset'), repairBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:repair'), cancelBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:cancel'), diff --git a/apps/desktop/electron/primary-backend-startup.test.ts b/apps/desktop/electron/primary-backend-startup.test.ts new file mode 100644 index 00000000000..6d3f5c8f44d --- /dev/null +++ b/apps/desktop/electron/primary-backend-startup.test.ts @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict' + +import { test, vi } from 'vitest' + +import { createFirstRunSetupGate } from './first-run-setup-gate' +import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup' + +const bootstrapBackend = { + activeRoot: '/tmp/hermes-home/hermes-agent', + kind: 'bootstrap-needed', + platform: 'linux' +} + +function startupOptions(overrides: Record = {}) { + return { + connectRemote: vi.fn(async remote => ({ baseUrl: remote.baseUrl, mode: 'remote' as const })), + ensureLocalRuntime: vi.fn(async backend => ({ ...backend, command: 'hermes' })), + prepareLocalBackend: vi.fn(async () => bootstrapBackend), + resolveRemote: vi.fn(async () => null), + waitForDecision: vi.fn(async () => 'continue-local' as const), + waitForLocalStart: vi.fn(async () => {}), + ...overrides + } +} + +test('remote apply re-resolves the saved connection without ensuring a local runtime', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' } + let configuredRemote: typeof savedRemote | null = null + + const options = startupOptions({ + resolveRemote: vi.fn(async () => configuredRemote), + waitForDecision: gate.wait + }) + + const pending = runPrimaryBackendStartup(options) + + await vi.waitFor(() => assert.equal(gate.hasWaiter(), true)) + configuredRemote = savedRemote + assert.equal(gate.abandonForRemoteApply(), true) + + assert.deepEqual(await pending, { + kind: 'remote', + connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' } + }) + assert.deepEqual(options.resolveRemote.mock.calls, [[], []]) + assert.deepEqual(options.connectRemote.mock.calls, [[savedRemote]]) + assert.equal(options.ensureLocalRuntime.mock.calls.length, 0) +}) + +test('an already-saved remote bypasses every local startup step', async () => { + const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' } + const options = startupOptions({ resolveRemote: vi.fn(async () => savedRemote) }) + + assert.deepEqual(await runPrimaryBackendStartup(options), { + kind: 'remote', + connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' } + }) + assert.equal(options.waitForLocalStart.mock.calls.length, 0) + assert.equal(options.prepareLocalBackend.mock.calls.length, 0) + assert.equal(options.waitForDecision.mock.calls.length, 0) + assert.equal(options.ensureLocalRuntime.mock.calls.length, 0) +}) + +test('remote apply fails clearly when no saved remote can be resolved', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + const options = startupOptions({ waitForDecision: gate.wait }) + const pending = runPrimaryBackendStartup(options) + + await vi.waitFor(() => assert.equal(gate.hasWaiter(), true)) + gate.abandonForRemoteApply() + + await assert.rejects(pending, /without a saved remote backend/) + assert.equal(options.connectRemote.mock.calls.length, 0) + assert.equal(options.ensureLocalRuntime.mock.calls.length, 0) +}) + +test('continue local waits for update exclusion and ensures the prepared runtime exactly once', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + const runtimeBackend = { ...bootstrapBackend, command: 'hermes' } + + const options = startupOptions({ + ensureLocalRuntime: vi.fn(async () => runtimeBackend), + waitForDecision: gate.wait + }) + + const pending = runPrimaryBackendStartup(options) + + await vi.waitFor(() => assert.equal(gate.hasWaiter(), true)) + gate.continueLocal() + + assert.deepEqual(await pending, { kind: 'local', backend: runtimeBackend }) + assert.deepEqual(options.waitForLocalStart.mock.calls, [[]]) + assert.deepEqual(options.prepareLocalBackend.mock.calls, [[]]) + assert.deepEqual(options.ensureLocalRuntime.mock.calls, [[bootstrapBackend]]) + assert.deepEqual(options.resolveRemote.mock.calls, [[]]) +}) + +test('reset rejects with a typed error and never enters either backend', async () => { + const gate = createFirstRunSetupGate({ stuckAfterMs: 0 }) + const options = startupOptions({ waitForDecision: gate.wait }) + const pending = runPrimaryBackendStartup(options) + + await vi.waitFor(() => assert.equal(gate.hasWaiter(), true)) + gate.resetForRetry() + + await assert.rejects(pending, error => error instanceof FirstRunSetupResetError && error.firstRunSetupReset) + assert.equal(options.connectRemote.mock.calls.length, 0) + assert.equal(options.ensureLocalRuntime.mock.calls.length, 0) +}) diff --git a/apps/desktop/electron/primary-backend-startup.ts b/apps/desktop/electron/primary-backend-startup.ts new file mode 100644 index 00000000000..edbe5167b05 --- /dev/null +++ b/apps/desktop/electron/primary-backend-startup.ts @@ -0,0 +1,66 @@ +import type { FirstRunSetupDecision } from './first-run-setup-gate' + +export interface PrimaryBackendStartupOptions { + connectRemote: (remote: Remote) => Promise + ensureLocalRuntime: (backend: Backend) => Promise + prepareLocalBackend: () => Backend | Promise + resolveRemote: () => Promise + waitForDecision: (backend: Backend) => Promise + waitForLocalStart: () => Promise +} + +export type PrimaryBackendStartupResult = + | { kind: 'local'; backend: RuntimeBackend } + | { kind: 'remote'; connection: Connection } + +export class FirstRunSetupResetError extends Error { + readonly firstRunSetupReset = true + + constructor() { + super('First-run setup was reset before a choice completed.') + this.name = 'FirstRunSetupResetError' + } +} + +// Owns the production startHermes path up to the local process spawn. Keeping +// the full ordering here makes the first-run remote boundary executable in a +// test: an already-saved remote wins immediately; otherwise update exclusion +// and local backend resolution happen before the setup gate, and a remote Apply +// re-resolves persisted config without ever entering ensureRuntime/bootstrap. +export async function runPrimaryBackendStartup({ + connectRemote, + ensureLocalRuntime, + prepareLocalBackend, + resolveRemote, + waitForDecision, + waitForLocalStart +}: PrimaryBackendStartupOptions): Promise< + PrimaryBackendStartupResult +> { + const savedRemote = await resolveRemote() + + if (savedRemote) { + return { kind: 'remote', connection: await connectRemote(savedRemote) } + } + + await waitForLocalStart() + + const backend = await prepareLocalBackend() + const decision = await waitForDecision(backend) + + if (decision === 'remote-applied') { + const appliedRemote = await resolveRemote() + + if (!appliedRemote) { + throw new Error('First-run remote setup completed without a saved remote backend.') + } + + return { kind: 'remote', connection: await connectRemote(appliedRemote) } + } + + if (decision === 'reset') { + throw new FirstRunSetupResetError() + } + + return { kind: 'local', backend: await ensureLocalRuntime(backend) } +} diff --git a/apps/desktop/electron/primary-connection-rehome.ts b/apps/desktop/electron/primary-connection-rehome.ts new file mode 100644 index 00000000000..4952b8a9028 --- /dev/null +++ b/apps/desktop/electron/primary-connection-rehome.ts @@ -0,0 +1,35 @@ +export interface PrimaryConnectionRehomeOptions { + clearLocalBootstrapFailure: () => void + mode: string + notifyConnectionApplied: () => void + resumeFirstRunRemote: () => boolean + teardownPrimaryBackend: (options: { soft: boolean }) => Promise +} + +// Production seam shared by the connection-config IPC handler and the +// first-run integration test. A remote apply that resumes the active setup +// gate must keep that connection attempt alive; ordinary mode changes tear the +// current backend down before the renderer is told to reconnect. +export async function rehomePrimaryConnection({ + clearLocalBootstrapFailure, + mode, + notifyConnectionApplied, + resumeFirstRunRemote, + teardownPrimaryBackend +}: PrimaryConnectionRehomeOptions): Promise<{ resumedFirstRunRemote: boolean }> { + let resumedFirstRunRemote = false + + if (mode === 'remote') { + resumedFirstRunRemote = resumeFirstRunRemote() + clearLocalBootstrapFailure() + } + + if (resumedFirstRunRemote) { + return { resumedFirstRunRemote: true } + } + + await teardownPrimaryBackend({ soft: true }) + notifyConnectionApplied() + + return { resumedFirstRunRemote: false } +} diff --git a/apps/desktop/electron/spawn-helper-perms.test.ts b/apps/desktop/electron/spawn-helper-perms.test.ts index 26cb1fefaa4..d8c9ca034b5 100644 --- a/apps/desktop/electron/spawn-helper-perms.test.ts +++ b/apps/desktop/electron/spawn-helper-perms.test.ts @@ -8,7 +8,8 @@ import { needsExecBit, spawnHelperCandidates, type SpawnHelperFs, - withExecBits + withExecBits, + writableNodePtyRoot } from './spawn-helper-perms' interface FakeFile { @@ -56,6 +57,30 @@ function fakeFs( } } +test('rewrites an archived node-pty root to the matching unpacked tree exactly once', () => { + assert.equal( + writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty'), + '/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty' + ) + assert.equal( + writableNodePtyRoot('/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty'), + '/Hermes.app/Contents/Resources/app.asar.unpacked/dist/node_modules/node-pty' + ) +}) + +test('uses the unpacked helper when resolution reports an app.asar node-pty root', () => { + const archivedRoot = '/Hermes.app/Contents/Resources/app.asar/dist/node_modules/node-pty' + const unpackedRoot = writableNodePtyRoot(archivedRoot) + const helper = join(unpackedRoot, 'prebuilds', 'darwin-arm64', 'spawn-helper') + const fs = fakeFs({ [helper]: { mode: 0o644 } }, { [join(unpackedRoot, 'prebuilds')]: ['darwin-arm64'] }) + + const result = ensureSpawnHelperExecutable(archivedRoot, fs) + + assert.deepEqual(result.fixed, [helper]) + assert.deepEqual(result.errors, []) + assert.deepEqual(fs.chmods, [{ path: helper, mode: 0o755 }]) +}) + test('needsExecBit / withExecBits treat any missing exec bit as non-executable', () => { assert.equal(needsExecBit(0o644), true) assert.equal(needsExecBit(0o755), false) diff --git a/apps/desktop/electron/spawn-helper-perms.ts b/apps/desktop/electron/spawn-helper-perms.ts index 6dfd484dc80..684b77096ba 100644 --- a/apps/desktop/electron/spawn-helper-perms.ts +++ b/apps/desktop/electron/spawn-helper-perms.ts @@ -18,6 +18,14 @@ import { join } from 'node:path' const EXEC_BITS = 0o111 +// Electron exposes module paths inside app.asar even when electron-builder has +// unpacked the native payload beside it. `stat` can read an archived path, but +// chmod cannot mutate it (ENOTDIR). Native node-pty helpers belong in the +// writable app.asar.unpacked tree; leave an already-unpacked path unchanged. +export function writableNodePtyRoot(nodePtyRoot: string): string { + return nodePtyRoot.replace(/app\.asar(?!\.unpacked)/, 'app.asar.unpacked') +} + export interface SpawnHelperFs { existsSync(path: string): boolean readdirSync(path: string): string[] @@ -81,8 +89,9 @@ export function ensureSpawnHelperExecutable( fs: SpawnHelperFs = defaultFs ): EnsureSpawnHelperResult { const result: EnsureSpawnHelperResult = { fixed: [], errors: [] } + const writableRoot = writableNodePtyRoot(nodePtyRoot) - for (const path of spawnHelperCandidates(nodePtyRoot, fs)) { + for (const path of spawnHelperCandidates(writableRoot, fs)) { if (!fs.existsSync(path)) { continue } diff --git a/apps/desktop/eslint.config.mjs b/apps/desktop/eslint.config.mjs index 61abc0c90a0..7e02d1d7936 100644 --- a/apps/desktop/eslint.config.mjs +++ b/apps/desktop/eslint.config.mjs @@ -39,5 +39,43 @@ export default [ rules: { 'no-restricted-globals': ['warn', 'document'] } + }, + { + // Ban mirroring reactive values into refs via useEffect — the "atom-mirrored + // ref" antipattern. A ref synced from a nanostores atom via useEffect lags the + // atom by one render, which creates stale-read bugs in callbacks that read the + // ref (cancelRun sent session.interrupt to the wrong session; steerPrompt, + // restoreToMessage, editMessage all had closure-priority stale reads). The fix + // is to read $atom.get() directly in callbacks instead. This rule catches the + // mirroring effect at lint time so the pattern can't reappear. Legitimate + // non-atom ref writes inside useEffect (DOM instance refs, mount flags, request + // tokens, prop mirrors) get an eslint-disable-next-line with a comment. + files: ['src/**/*.{ts,tsx}'], + rules: { + 'no-restricted-syntax': [ + 'error', + { + // useEffect(() => { someRef.current = value }, [value]) + selector: + 'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="AssignmentExpression"][body.left.type="MemberExpression"][body.left.property.name="current"]', + message: + 'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.' + }, + { + // useEffect(() => { someRef.current = value; ... }, [value]) + selector: + 'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(AssignmentExpression[left.type="MemberExpression"][left.property.name="current"])', + message: + 'Do not mirror reactive values into refs via useEffect. Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.' + }, + { + // useEffect(() => { setMutableRef(ref, value) }, [value]) + selector: + 'CallExpression[callee.name="useEffect"] > ArrowFunctionExpression[body.type="BlockStatement"]:has(CallExpression[callee.name="setMutableRef"])', + message: + 'Do not mirror reactive values into refs via useEffect (setMutableRef included). Read $atom.get() directly in callbacks instead — refs synced from atoms lag one render and cause stale-read bugs.' + } + ] + } } ] diff --git a/apps/desktop/package.json b/apps/desktop/package.json index e40be3df818..0d2c07756af 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -111,7 +111,7 @@ "motion": "^12.38.0", "nanostores": "^1.3.0", "node-pty": "1.1.0", - "radix-ui": "^1.4.3", + "radix-ui": "^1.6.5", "react": "^19.2.5", "react-arborist": "^3.5.0", "react-dnd-html5-backend": "^14.0.3", diff --git a/apps/desktop/scripts/before-pack.mjs b/apps/desktop/scripts/before-pack.mjs index 8b2359dfba6..b76acfe32d2 100644 --- a/apps/desktop/scripts/before-pack.mjs +++ b/apps/desktop/scripts/before-pack.mjs @@ -57,7 +57,8 @@ * - electronPlatformName: 'win32' | 'darwin' | 'linux' * - arch: Arch enum (0=ia32, 1=x64, 2=armv7l, 3=arm64, 4=universal) */ -import { existsSync, rmSync } from 'node:fs' +import { existsSync, rmSync, renameSync } from 'node:fs' +import path from 'node:path' import { Arch } from 'electron-builder' import { stageNodePty } from './stage-native-deps.mjs' @@ -75,10 +76,52 @@ export function cleanStaleAppOutDir(appOutDir) { return true } +/** + * Windows rollback material (#69179): before wiping the previous unpacked + * tree, preserve it as `.bak` — but ONLY when it holds the product + * exe (i.e. it is a previously-working build, not the corrupted partial state + * cleanStaleAppOutDir exists to remove). If the fresh pack then produces a + * Hermes.exe that Windows can't load (truncated PE from a corrupt cached + * Electron zip, wrong arch), the updater's integrity gate in + * `hermes desktop --build-only` (hermes_cli/main.py + * `_ensure_desktop_exe_launchable`) restores this .bak instead of leaving the + * user with "This app can't run on your computer". + * + * Returns true when the tree was preserved (appOutDir no longer exists), false + * when there was nothing worth preserving (caller falls through to the wipe). + * A rename failure (AV holding a handle) also returns false — the wipe is the + * safe fallback and matches pre-#69179 behavior exactly. + */ +export function preserveRollbackBackup(appOutDir, productExeName = 'Hermes.exe') { + if (!appOutDir || typeof appOutDir !== 'string' || !existsSync(appOutDir)) { + return false + } + if (!existsSync(path.join(appOutDir, productExeName))) { + // Partial/corrupt tree (interrupted prior pack) — not rollback material. + return false + } + const backupDir = `${appOutDir}.bak` + try { + rmSync(backupDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }) + renameSync(appOutDir, backupDir) + return true + } catch { + return false + } +} + export default async function beforePack(context) { const appOutDir = context && context.appOutDir + const platformName = context && context.electronPlatformName try { - if (cleanStaleAppOutDir(appOutDir)) { + // Windows: keep the previous working build as rollback material for the + // post-build integrity gate (#69179) instead of destroying it. Falls + // through to the plain wipe when the old tree is partial/corrupt or the + // rename fails. + const productExe = `${(context && context.packager?.appInfo?.productFilename) || 'Hermes'}.exe` + if (platformName === 'win32' && preserveRollbackBackup(appOutDir, productExe)) { + console.log(`[before-pack] preserved previous unpacked dir for rollback: ${appOutDir}.bak`) + } else if (cleanStaleAppOutDir(appOutDir)) { console.log(`[before-pack] removed stale unpacked dir before staging: ${appOutDir}`) } } catch (err) { diff --git a/apps/desktop/scripts/before-pack.test.mjs b/apps/desktop/scripts/before-pack.test.mjs index 44adf961618..d082ec4d2ad 100644 --- a/apps/desktop/scripts/before-pack.test.mjs +++ b/apps/desktop/scripts/before-pack.test.mjs @@ -4,7 +4,7 @@ import os from 'node:os' import path from 'node:path' import { test } from 'vitest' -import beforePack, { cleanStaleAppOutDir } from '../scripts/before-pack.mjs' +import beforePack, { cleanStaleAppOutDir, preserveRollbackBackup } from '../scripts/before-pack.mjs' test('cleanStaleAppOutDir removes a populated unpacked directory', () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) @@ -50,3 +50,106 @@ test('beforePack default export resolves even when cleanup throws', async () => // remove; the contract under test is that the hook never rejects. await assert.doesNotReject(beforePack({ appOutDir: '', electronPlatformName: 'linux' })) }) + +// ─── Windows rollback preservation (#69179) ──────────────────────────────── + +test('preserveRollbackBackup moves a working build to .bak', () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) + try { + const appOutDir = path.join(tempRoot, 'win-unpacked') + fs.mkdirSync(appOutDir, { recursive: true }) + fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-old-build', 'utf8') + fs.writeFileSync(path.join(appOutDir, 'resources.pak'), 'x', 'utf8') + + const preserved = preserveRollbackBackup(appOutDir, 'Hermes.exe') + + assert.equal(preserved, true) + // Original slot vacated so electron-builder stages into a clean tree... + assert.equal(fs.existsSync(appOutDir), false) + // ...and the previous working build is intact under .bak for rollback. + assert.equal( + fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'), + 'MZ-old-build' + ) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('preserveRollbackBackup replaces a stale .bak from an older update', () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) + try { + const appOutDir = path.join(tempRoot, 'win-unpacked') + fs.mkdirSync(appOutDir, { recursive: true }) + fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'current', 'utf8') + fs.mkdirSync(`${appOutDir}.bak`, { recursive: true }) + fs.writeFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'two-updates-ago', 'utf8') + + assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), true) + assert.equal(fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'), 'current') + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('preserveRollbackBackup refuses a partial tree missing the product exe', () => { + // The corrupted partial state (interrupted prior pack) must NOT become + // rollback material — it is exactly what cleanStaleAppOutDir exists to wipe. + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) + try { + const appOutDir = path.join(tempRoot, 'win-unpacked') + fs.mkdirSync(appOutDir, { recursive: true }) + fs.writeFileSync(path.join(appOutDir, 'LICENSE.electron.txt'), 'x', 'utf8') + + assert.equal(preserveRollbackBackup(appOutDir, 'Hermes.exe'), false) + // Tree untouched; the caller's wipe path handles it. + assert.equal(fs.existsSync(appOutDir), true) + assert.equal(fs.existsSync(`${appOutDir}.bak`), false) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('preserveRollbackBackup ignores missing or invalid input', () => { + assert.equal(preserveRollbackBackup(''), false) + assert.equal(preserveRollbackBackup(undefined), false) + assert.equal(preserveRollbackBackup(null), false) + assert.equal(preserveRollbackBackup(path.join(os.tmpdir(), 'does-not-exist-xyz')), false) +}) + +test('beforePack on win32 preserves the previous build instead of wiping it', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) + try { + const appOutDir = path.join(tempRoot, 'win-unpacked') + fs.mkdirSync(appOutDir, { recursive: true }) + fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'MZ-working', 'utf8') + + // No packager info in the context → default 'Hermes.exe' product name. + // node-pty staging is skipped because arch is not a number here. + await beforePack({ appOutDir, electronPlatformName: 'win32' }) + + assert.equal(fs.existsSync(appOutDir), false) + assert.equal( + fs.readFileSync(path.join(`${appOutDir}.bak`, 'Hermes.exe'), 'utf8'), + 'MZ-working' + ) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('beforePack on linux keeps the plain wipe (no .bak)', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-before-pack-')) + try { + const appOutDir = path.join(tempRoot, 'linux-unpacked') + fs.mkdirSync(appOutDir, { recursive: true }) + fs.writeFileSync(path.join(appOutDir, 'Hermes.exe'), 'x', 'utf8') + + await beforePack({ appOutDir, electronPlatformName: 'linux' }) + + assert.equal(fs.existsSync(appOutDir), false) + assert.equal(fs.existsSync(`${appOutDir}.bak`), false) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs index d9aeece1eff..d69ecce2fbc 100644 --- a/apps/desktop/scripts/perf/scenarios/index.mjs +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -4,16 +4,20 @@ import coldStart from './cold-start.mjs' import firstToken from './first-token.mjs' import keystroke from './keystroke.mjs' +import multitab from './multitab.mjs' import profileSwitch from './profile-switch.mjs' import sessionSwitch from './session-switch.mjs' import stream from './stream.mjs' +import streamHistory from './stream-history.mjs' import submit from './submit.mjs' import transcript from './transcript.mjs' export const SCENARIOS = { [stream.name]: stream, + [streamHistory.name]: streamHistory, [keystroke.name]: keystroke, [transcript.name]: transcript, + [multitab.name]: multitab, [coldStart.name]: coldStart, [firstToken.name]: firstToken, [submit.name]: submit, diff --git a/apps/desktop/scripts/perf/scenarios/multitab.mjs b/apps/desktop/scripts/perf/scenarios/multitab.mjs new file mode 100644 index 00000000000..d93116f1285 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/multitab.mjs @@ -0,0 +1,246 @@ +// Multi-tab working sessions: N session tiles stacked as tabs in the main +// zone, EVERY tab mounted (keep-alive), all streaming concurrently — the +// "5 tabs doing PR review" workload. Measures frame pacing + longtasks while +// the whole stack streams, which is where multitab renderers crawl. +// +// Drives the real pipeline synthetically (no backend, no credits): +// publishSessionState per session per flush — exactly what the gateway's +// delta flush does — via the __HERMES_SESSION_TILES__ hook. +// +// node scripts/perf/run.mjs multitab --spawn [--tiles 5] [--tokens 240] + +import { sleep } from '../lib/cdp.mjs' +import { frameHistogram, percentile } from '../lib/stats.mjs' + +// Same recorder pattern as stream.mjs (generation-guarded rAF + longtasks). +const RECORDERS = ` + (() => { + window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1 + const ftGen = window.__FT_GEN__ + window.__FT__ = { times: [], stop: false } + let last = performance.now() + const tick = () => { + if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return + const now = performance.now() + window.__FT__.times.push(now - last) + last = now + requestAnimationFrame(tick) + } + requestAnimationFrame(tick) + + window.__LT__ = { entries: [], stop: false } + try { + const po = new PerformanceObserver((list) => { + if (window.__LT__.stop) return + for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime }) + }) + po.observe({ entryTypes: ['longtask'] }) + window.__LT__.po = po + } catch {} + return 'armed' + })() +` + +const COLLECT = ` + (() => { + window.__FT__.stop = true + window.__LT__.stop = true + try { window.__LT__.po && window.__LT__.po.disconnect() } catch {} + return JSON.stringify({ frames: window.__FT__.times, longtasks: window.__LT__.entries }) + })() +` + +/** Page-side setup: open `tiles` session tiles stacked into the main zone, + * bind fake runtime ids, and seed each with a realistic transcript. */ +const setup = (tiles, seedTurns, streamSeed) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + if (!hook) return 'no-hook' + + const turn = (sid, i) => ([ + { id: sid + '-u' + i, role: 'user', timestamp: Date.now(), + parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff in module ' + i + ' handle the error path?' }] }, + { id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false, + parts: [{ type: 'text', text: [ + '## Finding ' + i, '', + 'The handler swallows the rejection. Key points for hunk \\\`' + i + '\\\`:', '', + '- The catch block drops the original error.', + '- Retries are unbounded — see [the loop](https://example.com/loop).', '', + '\\\`\\\`\\\`ts', + 'async function retry' + i + '(fn: () => Promise) {', + ' for (;;) { try { return await fn() } catch {} }', + '}', + '\\\`\\\`\\\`', '', + '| path | covered |', '|---|---|', '| happy | yes |', '| error | no |', '' + ].join('\\n') }] } + ]) + + const state = (sid, rid) => { + const messages = [] + for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i)) + // Streaming tail the driver grows (--code seeds an open fence). + messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true, + parts: [{ type: 'text', text: ${JSON.stringify(streamSeed)} }] }) + return { + storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '', + reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '', + busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true, + pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false, + needsInput: false, turnStartedAt: Date.now(), usage: null + } + } + + window.__MT__ = { ids: [], timer: null } + for (let n = 1; n <= ${tiles}; n++) { + const sid = 'perf-tile-' + n + const rid = 'perf-rt-' + n + window.__MT__.ids.push({ sid, rid }) + hook.open(sid, 'center') + hook.patch(sid, { runtimeId: rid }) + hook.publish(rid, state(sid, rid)) + } + return 'ok' + })() +` + +// Activate every tab once so keep-alive mounts the full stack (lazy mount: +// a never-activated tab stays unmounted, which would understate the cost). +const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})` + +/** Page-side driver: grow every tile's streaming tail by `chunk` each + * `intervalMs`, through the same publish path the gateway flush uses. */ +const drive = (chunk, intervalMs, totalTokens) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + let pushed = 0 + const tick = () => { + const states = hook.states() + for (const { rid } of window.__MT__.ids) { + const prev = states[rid] + if (!prev) continue + const messages = prev.messages.map(m => { + if (m.id !== prev.streamId) return m + const head = m.parts.slice(0, -1) + const last = m.parts[m.parts.length - 1] + return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] } + }) + hook.publish(rid, { ...prev, messages }) + } + pushed += 1 + if (pushed < ${totalTokens}) window.__MT__.timer = setTimeout(tick, ${intervalMs}) + else window.__MT__.done = true + } + window.__MT__.timer = setTimeout(tick, ${intervalMs}) + return 'driving' + })() +` + +const CLEANUP = ` + (() => { + if (window.__MT__) { + clearTimeout(window.__MT__.timer) + for (const { sid, rid } of window.__MT__.ids) { + window.__HERMES_SESSION_TILES__.publish(rid, { + ...window.__HERMES_SESSION_TILES__.states()[rid], busy: false, streamId: null + }) + window.__HERMES_SESSION_TILES__.close(sid) + } + window.__MT__ = null + } + return 'cleaned' + })() +` + +export default { + name: 'multitab', + tier: 'ci', + description: 'N mounted session-tile tabs all streaming: frame pacing + longtasks.', + async run(cdp, opts = {}) { + const tiles = Number(opts.tiles ?? 5) + const seedTurns = Number(opts.turns ?? 20) + const tokens = Number(opts.tokens ?? 240) + // Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush. + const intervalMs = Number(opts.intervalMs ?? 33) + // --code: every tile grows ONE giant fenced code block with no settle + // boundaries — what a coding agent streams. The block re-parses and + // re-renders fully every flush (block memoization can't settle it), the + // documented worst case and the "5 tabs all coding" crawl. + const chunk = opts.code + ? ' const value = await resolve(ctx, { retry: true }) // step\n' + : (opts.chunk ?? 'A streamed review sentence with **bold**, `code`, and ordinary prose.\n\n') + const streamSeed = opts.code ? '```ts\n' : '' + + await cdp.send('Runtime.enable') + + const ok = await cdp.eval(setup(tiles, seedTurns, streamSeed)) + + if (ok !== 'ok') { + throw new Error(`multitab setup failed (${ok}) — dev hooks missing? (needs a dev/probe renderer)`) + } + + // Mount every tab (keep-alive mounts on first activation), then settle. + for (let n = 1; n <= tiles; n++) { + await cdp.eval(reveal(`perf-tile-${n}`)) + await sleep(350) + } + + await sleep(1000) + await cdp.eval(RECORDERS) + await cdp.eval(drive(chunk, intervalMs, tokens)) + await sleep(tokens * intervalMs + 1500) + + const data = JSON.parse(await cdp.eval(COLLECT)) + await cdp.eval(CLEANUP) + + // Drop the first 500ms (recorder install + settle). + const frames = [] + let acc = 0 + + for (const f of data.frames) { + acc += f + + if (acc >= 500) { + frames.push(f) + } + } + + const ltDurations = data.longtasks.map(e => e.duration) + const windowS = frames.reduce((a, b) => a + b, 0) / 1000 + // The felt numbers: sustained fps over the window, and the fps of the + // worst 1-second slice (a 333ms frame IS "3fps" even if the average looks + // fine). Worst slice = max summed frame time in any sliding 1s window. + const avgFps = windowS ? frames.length / windowS : 0 + let worstFps = avgFps + + for (let i = 0, j = 0, sum = 0; j < frames.length; j++) { + sum += frames[j] + + while (sum > 1000) { + sum -= frames[i++] + } + + // Only a window that actually spans ~1s counts; short prefixes don't. + if (sum >= 900) { + worstFps = Math.min(worstFps, ((j - i + 1) / sum) * 1000) + } + } + + return { + metrics: { + longtasks_n: data.longtasks.length, + longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10, + frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10, + frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10, + slow_frames_33: frames.filter(f => f > 33).length + }, + detail: { + tiles, + code: Boolean(opts.code), + windowS: Math.round(windowS * 10) / 10, + avgFps: Math.round(avgFps * 10) / 10, + worstSecondFps: Math.round(worstFps * 10) / 10, + frameHistogram: frameHistogram(frames) + } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/stream-history.mjs b/apps/desktop/scripts/perf/scenarios/stream-history.mjs new file mode 100644 index 00000000000..d807f3f3573 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/stream-history.mjs @@ -0,0 +1,18 @@ +// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but +// the history is mounted and allowed to settle before the recorders start, so +// what it captures is the per-delta cost that scales with transcript length — +// the regression reported in #69120. +// +// Report-only (tier: manual): the number depends on how much history the host +// can mount, so it is not gated against the committed baseline. + +import stream from './stream.mjs' + +export default { + name: 'stream-history', + tier: 'manual', + description: 'Streaming cost with a long settled transcript already mounted.', + run(cdp, opts = {}) { + return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) }) + } +} diff --git a/apps/desktop/scripts/perf/scenarios/stream.mjs b/apps/desktop/scripts/perf/scenarios/stream.mjs index 66ae3ca58d5..f32a4831b1e 100644 --- a/apps/desktop/scripts/perf/scenarios/stream.mjs +++ b/apps/desktop/scripts/perf/scenarios/stream.mjs @@ -70,7 +70,7 @@ const COLLECT = ` })() ` -function analyze(data, warmupMs) { +function analyze(data, warmupMs, extra = {}) { // Drop warm-up frames (recorder installs before the stream starts). const frames = [] let acc = 0 @@ -102,6 +102,7 @@ function analyze(data, warmupMs) { intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10 }, detail: { + ...extra, windowS: Math.round(windowS * 10) / 10, avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0, frameHistogram: frameHistogram(frames), @@ -128,13 +129,36 @@ export default { // noise unrelated to render cost). const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n' const real = Boolean(opts.real) + const historyTurns = Number(opts.historyTurns ?? 0) + const historySettleMs = Number(opts.historySettleMs ?? 1500) await cdp.send('Runtime.enable') + + // Mount the settled history BEFORE the recorders start, so the measurement + // window contains only streaming work — not the one-off mount cost. + if (historyTurns > 0) { + if (real) { + throw new Error('--historyTurns is only supported by the synthetic stream path') + } + + await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`) + await sleep(historySettleMs) + + const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()')) + const expected = historyTurns * 2 + + if (mounted !== expected) { + throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`) + } + } + await cdp.eval(RECORDERS) if (real) { // Backend path: fire a real prompt and wait for the stream to appear. - const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`) + const baseCount = await cdp.eval( + `document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length` + ) await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 }) await cdp.eval(`(() => { const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) @@ -186,6 +210,6 @@ export default { await cdp.eval('window.__PERF_DRIVE__.reset()') } - return analyze(data, real ? 0 : 500) + return analyze(data, real ? 0 : 500, { historyTurns }) } } diff --git a/apps/desktop/scripts/probe-model-picker.mjs b/apps/desktop/scripts/probe-model-picker.mjs new file mode 100644 index 00000000000..d5585a36835 --- /dev/null +++ b/apps/desktop/scripts/probe-model-picker.mjs @@ -0,0 +1,80 @@ +// Model picker open latency probe: click the composer model pill, time until +// the dropdown/dialog content paints, repeat. Run with --cpuprofile via the +// harness runner once promoted; standalone for iteration. +// node scripts/perf/probe-model-picker.mjs [--port 9222] [--rounds 5] +import { CDP } from './perf/lib/cdp.mjs' + +const args = process.argv.slice(2) +const flag = name => { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} +const port = Number(flag('port') ?? 9222) +const rounds = Number(flag('rounds') ?? 5) +const sleep = ms => new Promise(r => setTimeout(r, ms)) + +const cdp = await CDP.connect({ port }) +await cdp.send('Runtime.enable') + +// Find the pill (dropdown path) — the composer model selector button. +const PILL = `(() => { + const btns = [...document.querySelectorAll('button[aria-label]')] + const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || '') && b.closest('[data-slot]')) + return pill ? (pill.getAttribute('aria-label') || 'found') : null +})()` + +console.log('pill:', await cdp.eval(PILL)) + +const MEASURE = ` + (async () => { + const btns = [...document.querySelectorAll('button[aria-label]')] + const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || '')) + if (!pill) return JSON.stringify({ error: 'no pill' }) + + const t0 = performance.now() + pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })) + pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true })) + pill.click() + + // Wait for menu/dialog content to exist AND paint (double rAF after found). + const found = await new Promise(resolve => { + const deadline = performance.now() + 5000 + const check = () => { + const menu = document.querySelector('[role="menu"], [role="dialog"] [cmdk-list]') + if (menu && menu.childElementCount > 0) { + requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now()))) + return + } + if (performance.now() > deadline) { resolve(null); return } + requestAnimationFrame(check) + } + check() + }) + + const openMs = found ? found - t0 : null + const rows = document.querySelectorAll('[role="menu"] [role="menuitem"], [cmdk-item]').length + + // Close: Escape. + document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })) + const menu = document.querySelector('[role="menu"], [role="dialog"]') + menu?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })) + await new Promise(r => setTimeout(r, 300)) + + return JSON.stringify({ openMs, rows }) + })() +` + +const samples = [] + +for (let i = 0; i < rounds; i++) { + const raw = await cdp.eval(MEASURE, { awaitPromise: true }) + const r = JSON.parse(raw) + console.log(`round ${i}:`, r) + if (typeof r.openMs === 'number') samples.push(r.openMs) + await sleep(500) +} + +samples.sort((a, b) => a - b) +console.log('\nopen latency ms — min/median/max:', + Math.round(samples[0]), '/', Math.round(samples[Math.floor(samples.length / 2)]), '/', Math.round(samples.at(-1))) +cdp.close() diff --git a/apps/desktop/scripts/profile-model-picker.mjs b/apps/desktop/scripts/profile-model-picker.mjs new file mode 100644 index 00000000000..26e16d6e7c9 --- /dev/null +++ b/apps/desktop/scripts/profile-model-picker.mjs @@ -0,0 +1,60 @@ +// CPU-profile one model-picker open. +// node scripts/profile-model-picker.mjs [--port 9222] +import { writeFileSync } from 'node:fs' + +import { CDP } from './perf/lib/cdp.mjs' +import { cpuProfileTopSelf } from './perf/lib/stats.mjs' + +const port = Number(process.argv.includes('--port') ? process.argv[process.argv.indexOf('--port') + 1] : 9222) +const cdp = await CDP.connect({ port }) +await cdp.send('Runtime.enable') +await cdp.send('Profiler.enable') +await cdp.send('Profiler.setSamplingInterval', { interval: 100 }) + +const OPEN = ` + (async () => { + // Reset: close any open menu first. + document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })) + await new Promise(r => setTimeout(r, 250)) + + const btns = [...document.querySelectorAll('button[aria-label]')] + const pill = btns.find(b => /model/i.test(b.getAttribute('aria-label') || '')) + if (!pill) return -1 + const t0 = performance.now() + pill.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })) + pill.dispatchEvent(new PointerEvent('pointerup', { bubbles: true })) + pill.click() + const found = await new Promise(resolve => { + const deadline = performance.now() + 8000 + const check = () => { + const menu = document.querySelector('[role="menu"]') + if (menu && menu.childElementCount > 0) { + requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now()))) + return + } + if (performance.now() > deadline) { resolve(-1); return } + requestAnimationFrame(check) + } + check() + }) + return found < 0 ? -1 : found - t0 + })() +` + +await cdp.send('Profiler.start') +const openMs = await cdp.eval(OPEN) +const { profile } = await cdp.send('Profiler.stop') + +console.log('openMs:', Math.round(openMs)) +const out = `/tmp/model-picker-open.cpuprofile` +writeFileSync(out, JSON.stringify(profile)) +console.log('wrote', out) +console.log('top self-time (ms):') + +for (const r of cpuProfileTopSelf(profile, 20)) { + console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(44)} ${r.url.split('/').slice(-2).join('/')}:${r.line}`) +} + +// Close the menu again. +await cdp.eval(`document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))`) +cdp.close() diff --git a/apps/desktop/src/app/chat/composer/composer-utils.ts b/apps/desktop/src/app/chat/composer/composer-utils.ts index 7939be35b6b..547a210f06f 100644 --- a/apps/desktop/src/app/chat/composer/composer-utils.ts +++ b/apps/desktop/src/app/chat/composer/composer-utils.ts @@ -50,6 +50,9 @@ export function slashChipKindForItem(item: Unstable_TriggerItem): SlashChipKind return 'command' } +/** True for a skill completion — the only kind offered mid-message. */ +export const isSkillItem = (item: Unstable_TriggerItem) => slashChipKindForItem(item) === 'skill' + /** A `/` query is at its arg stage once it's past the command name. */ export const slashArgStage = (query: string) => query.includes(' ') diff --git a/apps/desktop/src/app/chat/composer/focus.test.ts b/apps/desktop/src/app/chat/composer/focus.test.ts new file mode 100644 index 00000000000..f3987982274 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/focus.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { blurComposerInput } from './focus' +import { RICH_INPUT_SLOT } from './rich-editor' + +/** + * Inactive tabs keep their composer mounted, so an unscoped lookup can blur a + * background input and leave the one the user is typing in focused. + */ + +/** A composer input inside its own pane layer, hidden or not. */ +function mountInput(hidden = false) { + const layer = document.createElement('div') + const input = document.createElement('div') + input.dataset.slot = RICH_INPUT_SLOT + input.tabIndex = 0 + layer.toggleAttribute('data-pane-hidden', hidden) + layer.append(input) + document.body.append(layer) + + return input +} + +afterEach(() => { + document.body.innerHTML = '' +}) + +describe('blurComposerInput', () => { + it('blurs the foreground composer while a hidden tab matches first', () => { + const background = mountInput(true) + const foreground = mountInput() + + foreground.focus() + blurComposerInput() + + expect(document.activeElement).not.toBe(foreground) + expect(document.activeElement).not.toBe(background) + }) + + it('leaves focus alone when the composer does not hold it', () => { + const outside = document.createElement('button') + document.body.append(outside) + mountInput() + + outside.focus() + blurComposerInput() + + expect(document.activeElement).toBe(outside) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/focus.ts b/apps/desktop/src/app/chat/composer/focus.ts index a470edfcd95..7120fde2b1a 100644 --- a/apps/desktop/src/app/chat/composer/focus.ts +++ b/apps/desktop/src/app/chat/composer/focus.ts @@ -10,6 +10,8 @@ * steal focus from the composer effect. */ +import { queryVisible } from '@/components/pane-shell/pane-visibility' + import type { InlineRefInput } from './inline-refs' import { RICH_INPUT_SLOT } from './rich-editor' @@ -175,9 +177,11 @@ export const focusComposerInput = (el: HTMLElement | null) => { window.setTimeout(focus, 0) } -/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). */ +/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). + * Skips inactive tabs — they stay mounted, so an unscoped lookup can land on a + * background composer and leave the visible one focused. */ export const blurComposerInput = () => { - const el = document.querySelector(`[data-slot="${RICH_INPUT_SLOT}"]`) as HTMLElement | null + const el = queryVisible(`[data-slot="${RICH_INPUT_SLOT}"]`) if (el && document.activeElement === el) { el.blur() diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx new file mode 100644 index 00000000000..81401482f69 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx @@ -0,0 +1,130 @@ +import { act, cleanup, render } from '@testing-library/react' +import { useLayoutEffect } from 'react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { type ComposerAttachment, mainComposerScope, stashSessionDraft } from '@/store/composer' + +import type { QueueEditState } from '../composer-utils' + +import { useComposerDraft } from './use-composer-draft' + +const mockComposerApi = { setText: vi.fn() } + +vi.mock('@assistant-ui/react', () => ({ + useAui: () => ({ composer: () => mockComposerApi }), + useAuiState: (selector: (state: { composer: { text: string } }) => unknown) => selector({ composer: { text: '' } }), + useComposerRuntime: () => ({ + getState: () => ({ text: '' }), + subscribe: () => () => undefined + }) +})) + +interface ProbeHarnessProps { + activeQueueSessionKey: string | null + onLayoutSnapshot: (attachments: ComposerAttachment[]) => void + sessionId: string +} + +function ProbeHarness({ activeQueueSessionKey, onLayoutSnapshot, sessionId }: ProbeHarnessProps) { + useComposerDraft({ + activeQueueSessionKey, + focusKey: null, + inputDisabled: false, + queueEditRef: { current: null as QueueEditState | null }, + sessionId + }) + + // useLayoutEffect fires synchronously right after the DOM commit, BEFORE + // the hook's per-thread scope-swap useEffect (a passive effect) has a + // chance to swap attachmentScope.$attachments over to the new session. A + // synchronous read here — the same read ChatBar's `attachments` prop + // performs at render time — observes the OUTGOING session's attachments. + useLayoutEffect(() => { + onLayoutSnapshot(mainComposerScope.$attachments.get()) + }) + + return null +} + +describe('useComposerDraft — attachment scope stays coherent with the committed session on switch (#59305)', () => { + afterEach(() => { + cleanup() + mainComposerScope.clear() + }) + + it('clears the outgoing session attachments by the layout phase right after switching sessions', () => { + const attachmentA: ComposerAttachment = { id: 'url-A', kind: 'url', label: 'A' } + stashSessionDraft('session-A', 'hi from A', [attachmentA]) + + const snapshots: ComposerAttachment[][] = [] + + const { rerender } = render( + snapshots.push(s)} sessionId="session-A" /> + ) + + // Mount loads session A's stashed attachment into the (module-level) main + // scope — confirms the fixture actually seeded the leak precondition. + expect(mainComposerScope.$attachments.get()).toEqual([attachmentA]) + + snapshots.length = 0 // drop the initial-mount snapshot; only the switch matters + + act(() => { + rerender( + snapshots.push(s)} + sessionId="session-B" + /> + ) + }) + + // By the layout phase the scope must already be B's (empty) — a submit + // fired the instant B renders must never ship session A's attachment. + expect(snapshots[0]).toEqual([]) + }) +}) + +describe('useComposerDraft — rehydrate diagnostic log stays redacted', () => { + afterEach(() => { + cleanup() + mainComposerScope.clear() + vi.restoreAllMocks() + }) + + it('logs counts/kinds/scope on restore but never the raw url, refText, or label', () => { + const secretUrl = 'https://secret.example.com/private-workspace-path' + + const attachment: ComposerAttachment = { + id: 'url-secret', + kind: 'url', + label: 'do-not-leak-label', + refText: `@url:${secretUrl}` + } + + stashSessionDraft('session-secret', '', [attachment]) + + const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined) + + render( + undefined} + sessionId="session-secret" + /> + ) + + const rehydrateCalls = debugSpy.mock.calls.filter(call => call[0] === '[composer-rehydrate]') + expect(rehydrateCalls.length).toBeGreaterThan(0) + + const serialized = JSON.stringify(rehydrateCalls) + expect(serialized).not.toContain(secretUrl) + expect(serialized).not.toContain(attachment.label) + expect(serialized).not.toContain(attachment.refText) + + expect(rehydrateCalls[0]?.[1]).toMatchObject({ + attachmentCount: 1, + attachmentKinds: ['url'], + scope: 'session-secret' + }) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts index 11dc9534df0..11aa35dfaf7 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts @@ -1,5 +1,5 @@ import { useAui, useAuiState, useComposerRuntime } from '@assistant-ui/react' -import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' +import { type RefObject, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' import { SLASH_COMMAND_RE } from '@/lib/chat-runtime' import { type ComposerAttachment, stashSessionDraft, takeSessionDraft } from '@/store/composer' @@ -20,7 +20,7 @@ import { onComposerInsertRequest } from '../focus' import { type InlineRefInput, insertInlineRefsIntoEditor } from '../inline-refs' -import { composerPlainText, placeCaretEnd, renderComposerContents } from '../rich-editor' +import { composerPlainText, placeCaretEnd, REF_RE, renderComposerContents } from '../rich-editor' import { useComposerScope } from '../scope' import type { ChatBarProps } from '../types' @@ -189,6 +189,21 @@ export function useComposerDraft({ stashSessionDraft(scope, text, attachments) const loadIntoComposer = (text: string, attachments: ComposerAttachment[]) => { + // Diagnostic breadcrumb for #59305-class reports: identifies WHAT kind of + // state got restored into the composer (session switch, queue-edit + // restore, history browse) without logging any raw content. REF_RE has the + // global flag — testing against a throwaway clone avoids mutating the + // shared instance's lastIndex, which would otherwise corrupt this check on + // the next call. + if (attachments.length > 0 || new RegExp(REF_RE.source, REF_RE.flags).test(text)) { + console.debug('[composer-rehydrate]', { + attachmentCount: attachments.length, + attachmentKinds: attachments.map(a => a.kind), + hasTextRefs: new RegExp(REF_RE.source, REF_RE.flags).test(text), + scope: activeQueueSessionKeyRef.current + }) + } + attachmentScope.$attachments.set(cloneAttachments(attachments)) paintDraft(text, false) } @@ -231,6 +246,7 @@ export function useComposerDraft({ // source otherwise), and (3) schedule the debounced per-session stash. // Browsing history / editing a queued prompt suppress the stash so recalled // text never clobbers the draft. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const sync = () => { const text = composerRuntime.getState().text @@ -318,7 +334,17 @@ export function useComposerDraft({ // Per-thread draft swap — the composer's only session coupling. Lifecycle // never clears composer state; this effect alone stashes on leave, restores // on enter. Keyed writes are idempotent, so no skip-sentinel. - useEffect(() => { + // + // MUST be a layout effect, not a passive one: it swaps attachmentScope's + // module-level $attachments atom, and a passive effect fires only after the + // browser paints the new session's view — leaving a window where the DOM + // already shows session B while $attachments (and therefore ChatBar's + // `attachments` prop) still holds session A's chips. A submit fired in that + // window (e.g. a fast session switch immediately followed by Enter) would + // ship A's attachments into B's turn (#59305). useLayoutEffect closes the + // window by running before paint. + + useLayoutEffect(() => { // A pending debounce timer from the outgoing session is now stale — its // scope was correct when scheduled, but the authoritative stash below // (and the cleanup on the way out) already covers that text. Letting it @@ -344,6 +370,7 @@ export function useComposerDraft({ // pagehide is load-bearing: React skips effect cleanups on reload, so Cmd+R // inside the debounce/rAF window would drop trailing keystrokes without this. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const flushPendingDraftPersist = () => { const scope = draftScopeRef.current diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts index 318802bbab8..228758dd905 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts @@ -1,6 +1,12 @@ import { useAuiState } from '@assistant-ui/react' import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' +import { + clearSurfaceVar, + COMPOSER_HEIGHT_VAR, + COMPOSER_SURFACE_HEIGHT_VAR, + setSurfaceVar +} from '@/app/chat/surface-vars' import { useMediaQuery } from '@/hooks/use-media-query' import { useResizeObserver } from '@/hooks/use-resize-observer' import { $composerPoppedOut } from '@/store/composer-popout' @@ -89,18 +95,16 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, // (Read globals here so the callback stays stable; mirror the popoutAllowed // gate since secondary windows are forced docked.) if ($composerPoppedOut.get() && !isSecondaryWindow()) { - const root = document.documentElement lastBucketedHeightRef.current = 0 lastBucketedSurfaceHeightRef.current = 0 - root.style.setProperty('--composer-measured-height', '0px') - root.style.setProperty('--composer-surface-measured-height', '0px') + setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, '0px') + setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, '0px') return } const { height, width } = composer.getBoundingClientRect() const surfaceHeight = composerSurfaceRef.current?.getBoundingClientRect().height - const root = document.documentElement if (width > 0) { const nextTight = width < COMPOSER_STACK_BREAKPOINT_PX @@ -135,7 +139,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, if (bucket !== lastBucketedHeightRef.current) { lastBucketedHeightRef.current = bucket - root.style.setProperty('--composer-measured-height', `${bucket}px`) + setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, `${bucket}px`) } } @@ -144,7 +148,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, if (bucket !== lastBucketedSurfaceHeightRef.current) { lastBucketedSurfaceHeightRef.current = bucket - root.style.setProperty('--composer-surface-measured-height', `${bucket}px`) + setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, `${bucket}px`) } } }, [composerRef, composerSurfaceRef, editorRef]) @@ -160,12 +164,13 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, }, [poppedOut, syncComposerMetrics]) useEffect(() => { + const composer = composerRef.current + return () => { - const root = document.documentElement - root.style.removeProperty('--composer-measured-height') - root.style.removeProperty('--composer-surface-measured-height') + clearSurfaceVar(composer, COMPOSER_HEIGHT_VAR) + clearSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR) } - }, []) + }, [composerRef]) // Pill compacts on real width (tile/pane), OR when stacked for any reason // (viewport-narrow / wrapped) so the controls row never over-runs. diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts index 0c2e4b61927..8d1bf8c1ee5 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts @@ -29,6 +29,7 @@ export function useComposerPlaceholder({ disabled, reconnecting, sessionId }: Us const prevSessionIdRef = useRef(sessionId) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const prev = prevSessionIdRef.current prevSessionIdRef.current = sessionId diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts index 4e813f548ae..dff3804bb69 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts @@ -322,6 +322,7 @@ export function useComposerQueue({ // never churns, so a change there is a real session switch and must NOT // migrate; only the runtime-derived key (queueSessionKey falsy → key is // sessionId) churns on a backend bounce/resume of the same conversation. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const prev = prevQueueKeyRef.current prevQueueKeyRef.current = activeQueueSessionKey diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx index a09cd10ef29..b3bcaaa463e 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx @@ -113,7 +113,9 @@ describe('useComposerSubmit busy-turn routing', () => { hook.result.current.submitDraft() }) - await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('/compress preserve context')) + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('/compress preserve context', { composerScope: 'stored-session' }) + ) expect(clearDraft).toHaveBeenCalledTimes(1) expect(onSteer).not.toHaveBeenCalled() expect(queueCurrentDraft).not.toHaveBeenCalled() @@ -159,9 +161,26 @@ describe('useComposerSubmit busy-turn routing', () => { hook.result.current.submitDraft() }) - await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('ordinary question', { attachments: [] })) + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('ordinary question', { + attachments: [], + composerScope: 'stored-session' + }) + ) expect(onSteer).not.toHaveBeenCalled() expect(queueCurrentDraft).not.toHaveBeenCalled() expect(onCancel).not.toHaveBeenCalled() }) + + it('threads the loaded composer scope through onSubmit for the #59305 submit-time guard', async () => { + const { hook, onSubmit } = renderSubmitHook({ text: 'hello' }) + + act(() => { + hook.result.current.submitDraft() + }) + + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('hello', expect.objectContaining({ composerScope: 'stored-session' })) + ) + }) }) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts index fb1cec2cf04..315157a96e5 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts @@ -89,7 +89,11 @@ export function useComposerSubmit({ stashAt(submittedScope, text, submittedAttachments) } - void Promise.resolve(attachments ? onSubmit(text, { attachments }) : onSubmit(text)) + void Promise.resolve( + attachments + ? onSubmit(text, { attachments, composerScope: submittedScope }) + : onSubmit(text, { composerScope: submittedScope }) + ) .then(accepted => void (accepted === false ? restore() : clearSessionDraft(submittedScope))) .catch(restore) } diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts new file mode 100644 index 00000000000..c356fae3af8 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts @@ -0,0 +1,123 @@ +import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core' +import { act, renderHook } from '@testing-library/react' +import { createRef } from 'react' +import { describe, expect, it, vi } from 'vitest' + +import { composerPlainText, renderComposerContents, RICH_INPUT_SLOT } from '../rich-editor' + +import { useComposerTrigger } from './use-composer-trigger' + +/** A live contentEditable seeded with `text`, caret parked at the end. */ +function mountEditor(text: string) { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.contentEditable = 'true' + document.body.append(editor) + renderComposerContents(editor, text) + + const range = document.createRange() + range.selectNodeContents(editor) + range.collapse(false) + const selection = window.getSelection()! + selection.removeAllRanges() + selection.addRange(range) + + return editor +} + +const item = (command: string, group = 'Skills'): Unstable_TriggerItem => ({ + id: command, + type: 'slash', + label: command.slice(1), + metadata: { command, display: command, meta: '', group, action: '', rawText: command } +}) + +function mountTrigger(editor: HTMLDivElement, items: Unstable_TriggerItem[]) { + const editorRef = createRef() as { current: HTMLDivElement | null } + editorRef.current = editor + + const draftRef = { current: composerPlainText(editor) } + + const adapter: Unstable_TriggerAdapter = { + categories: () => [], + categoryItems: () => [], + search: () => items + } + + const setComposerText = vi.fn() + + const hook = renderHook(() => + useComposerTrigger({ + at: { adapter: null, loading: false }, + draftRef, + editorRef, + requestMainFocus: vi.fn(), + setComposerText, + slash: { adapter, loading: false } + }) + ) + + return { draftRef, hook, setComposerText } +} + +describe('useComposerTrigger — slash anywhere in the prompt', () => { + it('opens the completion list for a slash typed mid-message', () => { + const editor = mountEditor('please run /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toMatchObject({ kind: '/', inline: true, query: 'cle' }) + expect(hook.result.current.triggerItems).toHaveLength(1) + }) + + it('inserts the picked skill inline and keeps the surrounding prose intact', () => { + const editor = mountEditor('please run /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(item('/clean'))) + + // The `/cle` the user typed is replaced by the full command; "please run" + // in front of it survives untouched. + expect(composerPlainText(editor)).toBe('please run /clean ') + }) + + it('offers only skills mid-message, not app commands', () => { + // `/model` and `/new` act on the app — meaningless as a reference in prose. + const editor = mountEditor('please run /') + const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands'), item('/new', 'Commands')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean']) + }) + + it('still offers the full command set at the start of the prompt', () => { + const editor = mountEditor('/') + const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean', 'model']) + }) + + it('still opens the list for a slash at the start of the prompt', () => { + const editor = mountEditor('/cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toMatchObject({ kind: '/', query: 'cle' }) + expect(hook.result.current.trigger?.inline).toBeUndefined() + }) + + it('leaves a mid-message file path alone', () => { + const editor = mountEditor('open src/foo/bar') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toBeNull() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts index 20abc03309f..3c22fddb634 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts @@ -4,7 +4,13 @@ import { type MutableRefObject, type RefObject, useCallback, useEffect, useRef, import { hermesDirectiveFormatter } from '@/components/assistant-ui/directive-text' import { desktopSlashCommandTakesArgs } from '@/lib/desktop-slash-commands' -import { COMPLETION_ACTIONS, slashArgStage, slashChipKindForItem, slashCommandToken } from '../composer-utils' +import { + COMPLETION_ACTIONS, + isSkillItem, + slashArgStage, + slashChipKindForItem, + slashCommandToken +} from '../composer-utils' import { composerPlainText, placeCaretEnd, @@ -112,7 +118,13 @@ export function useComposerTrigger({ return } - setTriggerItems(triggerAdapter.search(trigger.query)) + const items = triggerAdapter.search(trigger.query) + + // Mid-message only offers SKILLS. A built-in like `/model` or `/new` acts + // on the app, so it's meaningless as a reference inside prose — only a + // skill reads as "handle this part with X". Filtering here rather than in + // the fetcher keeps one completion source for both shapes. + setTriggerItems(trigger.inline ? items.filter(isSkillItem) : items) }, [trigger, triggerAdapter]) const triggerLoading = trigger?.kind === '@' ? at.loading : trigger?.kind === '/' ? slash.loading : false @@ -191,10 +203,13 @@ export function useComposerTrigger({ // Picking a bare arg-taking command (e.g. `/personality`) shouldn't commit // it — expand to its options step so the popover shows the inline list, just // as typing `/personality ` by hand would. A serialized value with a space is - // already an arg pick (`/personality alice`), so it commits normally. + // already an arg pick (`/personality alice`), so it commits normally. An + // inline (mid-message) pick never expands: it's a reference inside prose, so + // there's no command invocation for the args to belong to. const command = (item.metadata as { command?: string } | undefined)?.command ?? '' - const expandsToArgs = trigger.kind === '/' && !serialized.includes(' ') && desktopSlashCommandTakesArgs(command) + const expandsToArgs = + trigger.kind === '/' && !trigger.inline && !serialized.includes(' ') && desktopSlashCommandTakesArgs(command) const text = starter || serialized.endsWith(' ') ? serialized : `${serialized} ` const directive = !starter && serialized.match(/^@([^:]+):(.+)$/) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts b/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts index 6da699b602a..32301cc8294 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts @@ -49,6 +49,7 @@ export function useLiveCompletionAdapter(options: { useEffect(() => () => cancelTimer(), [cancelTimer]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (enabled) { return diff --git a/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts b/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts index 0b71507bfd1..79e1cc067da 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts @@ -231,6 +231,7 @@ export function useComposerPopoutGestures({ [clearTimer, poppedOut] ) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { // Coalesce drag updates to one per frame — pointermove can fire several times // between paints on high-Hz mice, and each update re-renders + clamps. diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index 2ec43c83c0b..33285f05aa1 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -60,18 +60,22 @@ export function useVoiceConversation({ const statusRef = useRef('idle') const wasEnabledRef = useRef(enabled) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { enabledRef.current = enabled }, [enabled]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { mutedRef.current = muted }, [muted]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { busyRef.current = busy }, [busy]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { statusRef.current = status }, [status]) @@ -508,6 +512,7 @@ export function useVoiceConversation({ // Drive the loop: when a voice-submitted reply appears, open a live speech // session (which feeds itself from then on). Otherwise start listening when // idle between turns. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!enabled || muted) { return @@ -542,6 +547,7 @@ export function useVoiceConversation({ } }, [busy, enabled, muted, openLiveSpeech, pendingResponse, startListening, status]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (enabled && !wasEnabledRef.current) { void start() diff --git a/apps/desktop/src/app/chat/composer/model-pill.tsx b/apps/desktop/src/app/chat/composer/model-pill.tsx index b5a6e976805..2a26db4094c 100644 --- a/apps/desktop/src/app/chat/composer/model-pill.tsx +++ b/apps/desktop/src/app/chat/composer/model-pill.tsx @@ -11,7 +11,7 @@ import { useI18n } from '@/i18n' import { ChevronDown } from '@/lib/icons' import { formatModelStatusLabel } from '@/lib/model-status-label' import { cn } from '@/lib/utils' -import { $currentModelSource, setModelPickerOpen } from '@/store/session' +import { $currentModelSource, $defaultReasoningEffort, setModelPickerOpen } from '@/store/session' import type { ChatBarState } from './types' @@ -48,6 +48,7 @@ export function ModelPill({ const fastMode = useStore(view.$fast) const reasoningEffort = useStore(view.$reasoningEffort) const modelSource = useStore($currentModelSource) + const defaultEffort = useStore($defaultReasoningEffort) const runtimeId = useStore(view.$runtimeId) const [open, setOpen] = useState(false) @@ -68,7 +69,9 @@ export function ModelPill({ ) : ( <> {currentModel.trim() ? ( - {formatModelStatusLabel(currentModel, { fastMode, reasoningEffort })} + + {formatModelStatusLabel(currentModel, { defaultEffort, fastMode, reasoningEffort })} + ) : ( )} diff --git a/apps/desktop/src/app/chat/composer/rich-editor.ts b/apps/desktop/src/app/chat/composer/rich-editor.ts index 71491b87496..21f0286c9f8 100644 --- a/apps/desktop/src/app/chat/composer/rich-editor.ts +++ b/apps/desktop/src/app/chat/composer/rich-editor.ts @@ -15,6 +15,7 @@ import { type SlashChipKind, slashIconElement } from '@/components/assistant-ui/directive-text' +import { sessionRefFallbackLabel } from '@/lib/session-refs' export const RICH_INPUT_SLOT = 'composer-rich-input' @@ -59,7 +60,9 @@ export function refChipHtml(kind: string, rawValue: string, displayLabel?: strin const id = unquoteRef(rawValue) const text = `@${kind}:${quoteRefValue(id)}` - return `${directiveIconSvg(kind)}${escapeHtml(displayLabel || refLabel(id))}` + const label = displayLabel || (kind === 'session' ? sessionRefFallbackLabel(id) : refLabel(id)) + + return `${directiveIconSvg(kind)}${escapeHtml(label)}` } export function refChipElement(kind: string, rawValue: string, displayLabel?: string) { @@ -74,7 +77,7 @@ export function refChipElement(kind: string, rawValue: string, displayLabel?: st chip.dataset.refKind = kind chip.className = DIRECTIVE_CHIP_CLASS label.className = 'truncate' - label.textContent = displayLabel || refLabel(id) + label.textContent = displayLabel || (kind === 'session' ? sessionRefFallbackLabel(id) : refLabel(id)) chip.append(directiveIconElement(kind), label) return chip diff --git a/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx b/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx index aee735e4aea..5093658af88 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx @@ -91,6 +91,7 @@ export const CodingStatusRow = memo(function CodingStatusRow({ const worktreeReq = useStore($newWorktreeRequest) const lastWorktreeReqRef = useRef(worktreeReq) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (worktreeReq === lastWorktreeReqRef.current) { return diff --git a/apps/desktop/src/app/chat/composer/status-stack/index.tsx b/apps/desktop/src/app/chat/composer/status-stack/index.tsx index 739080be8b8..10b7a0776bf 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/index.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/index.tsx @@ -3,6 +3,7 @@ import { type ReactNode, useEffect, useLayoutEffect, useMemo, useRef } from 'rea import { useNavigate } from 'react-router-dom' import { blurComposerInput } from '@/app/chat/composer/focus' +import { clearSurfaceVar, setSurfaceVar, STATUS_STACK_VAR } from '@/app/chat/surface-vars' import { AGENTS_ROUTE } from '@/app/routes' import { BillingBanner } from '@/components/billing-banner' import { composerDockCard } from '@/components/chat/composer-dock' @@ -197,12 +198,12 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro // height never sees it. Publish our own measured height — bucketed like the // composer's, to avoid style invalidation churn — so the thread's // last-message clearance can add it and the stack never hides messages. + // Scoped to THIS surface: tiles render their own stack (see surface-vars.ts). useLayoutEffect(() => { - const root = document.documentElement const el = stackRef.current if (!visible || !el) { - root.style.removeProperty('--status-stack-measured-height') + clearSurfaceVar(el, STATUS_STACK_VAR) return } @@ -214,7 +215,7 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro if (bucket !== last) { last = bucket - root.style.setProperty('--status-stack-measured-height', `${bucket}px`) + setSurfaceVar(el, STATUS_STACK_VAR, `${bucket}px`) } } @@ -224,7 +225,7 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro return () => { observer.disconnect() - root.style.removeProperty('--status-stack-measured-height') + clearSurfaceVar(el, STATUS_STACK_VAR) } }, [visible]) diff --git a/apps/desktop/src/app/chat/composer/text-utils.test.ts b/apps/desktop/src/app/chat/composer/text-utils.test.ts index 6c6a20780f6..ca2ac803576 100644 --- a/apps/desktop/src/app/chat/composer/text-utils.test.ts +++ b/apps/desktop/src/app/chat/composer/text-utils.test.ts @@ -44,14 +44,25 @@ describe('detectTrigger', () => { it('does not treat file-style paths as slash triggers', () => { expect(detectTrigger('src/foo/bar')).toBeNull() expect(detectTrigger('/path/to/file')).toBeNull() + // Mid-message paths stay excluded too: a path keeps going past the command + // token, so the trailing-anchored inline trigger never matches it. + expect(detectTrigger('check src/foo/bar')).toBeNull() + expect(detectTrigger('look at /usr/local/bin')).toBeNull() + expect(detectTrigger('and/or')).toBeNull() }) - it('does not trigger slash popover mid-message', () => { - expect(detectTrigger('hello /')).toBeNull() - expect(detectTrigger('hello /skill')).toBeNull() + it('treats a mid-message slash as an inline reference', () => { + // Skills have to be reachable anywhere in a prompt, not just at position 0. + expect(detectTrigger('hello /')).toEqual({ kind: '/', inline: true, query: '', tokenLength: 1 }) + expect(detectTrigger('hello /clean')).toEqual({ kind: '/', inline: true, query: 'clean', tokenLength: 6 }) + expect(detectTrigger('text\n/skill')).toEqual({ kind: '/', inline: true, query: 'skill', tokenLength: 6 }) + }) + + it('does not carry arg completion into an inline slash reference', () => { + // Only a position-0 slash is a real invocation, so `/personality alic` + // mid-message is prose — the trigger ends at the command token. expect(detectTrigger('hello there /personality alic')).toBeNull() - expect(detectTrigger('text\n/skill')).toBeNull() - expect(detectTrigger('multi word message /')).toBeNull() + expect(detectTrigger('run /tools enable foo')).toBeNull() }) it('still anchors at-mention triggers strictly at the token edge', () => { diff --git a/apps/desktop/src/app/chat/composer/text-utils.ts b/apps/desktop/src/app/chat/composer/text-utils.ts index b9b6adc07f1..8bc6663210e 100644 --- a/apps/desktop/src/app/chat/composer/text-utils.ts +++ b/apps/desktop/src/app/chat/composer/text-utils.ts @@ -1,22 +1,35 @@ import { DATA_IMAGE_URL_RE, dataUrlToBlob } from '@/lib/embedded-images' export interface TriggerState { + /** True for a `/` typed mid-message — an inline skill/command reference in + * prose rather than a command invocation. Arg completion doesn't apply. */ + inline?: boolean kind: '@' | '/' query: string tokenLength: number } // `@` triggers stop at the first whitespace — `@file:path` and `@diff` are -// single tokens. `/` triggers keep going so the popover stays live while the -// user types args (`/personality alic` → arg completer suggests `alice`). -// Restricting the slash command name to `[a-zA-Z][\w-]*` avoids matching file -// paths like `src/foo/bar`. +// single tokens. Restricting the slash command name to `[a-zA-Z][\w-]*` avoids +// matching file paths like `src/foo/bar`. // -// Slash commands only execute at the beginning of a message, so the `/` -// trigger is anchored strictly at position 0 — not after whitespace — to -// avoid opening the popover mid-message (e.g. `hello /`). +// `/` triggers fire in two shapes, because a slash means two different things +// depending on where it sits: +// +// - At position 0 it's a COMMAND invocation the app executes (SLASH_COMMAND_RE +// is `^`-anchored, and so is the backend's). The popover stays live past the +// command name so arg completion works (`/personality alic` → `alice`). +// - After whitespace it's an inline REFERENCE the user is dropping into prose +// ("clean this up with /clean"). The text submits as an ordinary message, so +// there are no args to complete — the trigger is a single token that ends at +// the next space, exactly like `@`. +// +// The inline shape is what makes skills reachable anywhere in a prompt. Both +// shapes need the trailing `$`: detection runs against the text BEFORE the +// caret, so the match must end where the user is typing. const AT_TRIGGER_RE = /(?:^|[\s])(@)([^\s@/]*)$/ -const SLASH_TRIGGER_RE = /^(\/)((?:[a-zA-Z][\w-]*(?:\s+\S*)*)?)$/ +const SLASH_COMMAND_TRIGGER_RE = /^(\/)((?:[a-zA-Z][\w-]*(?:\s+\S*)*)?)$/ +const SLASH_INLINE_TRIGGER_RE = /[\s](\/)([a-zA-Z][\w-]*)?$/ /** Stable key for paste dedupe — `items` and `files` often mirror the same image as different objects. */ export function blobDedupeKey(blob: Blob): string { @@ -107,10 +120,20 @@ export function textBeforeCaret(editor: HTMLDivElement): string | null { } export function detectTrigger(textBefore: string): TriggerState | null { - const slash = SLASH_TRIGGER_RE.exec(textBefore) + const command = SLASH_COMMAND_TRIGGER_RE.exec(textBefore) - if (slash) { - return { kind: '/', query: slash[2], tokenLength: 1 + slash[2].length } + if (command) { + return { kind: '/', query: command[2], tokenLength: 1 + command[2].length } + } + + // An inline `/skill` is a reference dropped into prose, so it carries no args + // and the whole match is the token the chip replaces. + const inline = SLASH_INLINE_TRIGGER_RE.exec(textBefore) + + if (inline) { + const query = inline[2] ?? '' + + return { inline: true, kind: '/', query, tokenLength: 1 + query.length } } const at = AT_TRIGGER_RE.exec(textBefore) diff --git a/apps/desktop/src/app/chat/hooks/use-file-drop-zone.ts b/apps/desktop/src/app/chat/hooks/use-file-drop-zone.ts index 42e4554b992..186c226b84a 100644 --- a/apps/desktop/src/app/chat/hooks/use-file-drop-zone.ts +++ b/apps/desktop/src/app/chat/hooks/use-file-drop-zone.ts @@ -44,6 +44,7 @@ export function useFileDropZone({ enabled = true, onDropFiles }: FileDropZoneOpt // DnD can't be cancelled at the OS level, so we drop the overlay and arm a // guard that swallows the trailing drop instead. Top escape layer + capture // stop so it doesn't also fire a handler behind the drag (see drag-session). + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (dragKind === null) { return diff --git a/apps/desktop/src/app/chat/index.tsx b/apps/desktop/src/app/chat/index.tsx index bd18be0c483..5fdf5a0487b 100644 --- a/apps/desktop/src/app/chat/index.tsx +++ b/apps/desktop/src/app/chat/index.tsx @@ -1,14 +1,16 @@ import { type AppendMessage, AssistantRuntimeProvider, type ThreadMessage } from '@assistant-ui/react' import { useStore } from '@nanostores/react' import { useQuery } from '@tanstack/react-query' +import type { ReadableAtom } from 'nanostores' import type * as React from 'react' -import { Suspense, useCallback, useEffect, useMemo } from 'react' +import { Suspense, useCallback, useEffect, useMemo, useState } from 'react' import { useLocation } from 'react-router-dom' import type { SubmitTextOptions } from '@/app/session/hooks/use-prompt-actions/utils' import { Thread } from '@/components/assistant-ui/thread' import { Backdrop } from '@/components/Backdrop' import { COMPOSER_HEART_CONFIG, HeartField } from '@/components/chat/vibe-hearts' +import { usePaneVisible } from '@/components/pane-shell/pane-visibility' import { $sessionTileDragging, $sessionTileEdgeHover } from '@/components/pane-shell/tree/store' import { PromptOverlays } from '@/components/prompt-overlays' import { Button } from '@/components/ui/button' @@ -42,7 +44,7 @@ import { import { isSecondaryWindow, isWatchWindow } from '@/store/windows' import type { ModelOptionsResponse } from '@/types/hermes' -import { routeSessionId } from '../routes' +import { primaryRouteSelectedSessionId, routeSessionId } from '../routes' import { titlebarHeaderBaseClass, titlebarHeaderShadowClass, titlebarHeaderTitleClass } from '../shell/titlebar' import { ChatDropOverlay } from './chat-drop-overlay' @@ -174,6 +176,30 @@ interface ChatRuntimeBoundaryProps { const NO_MESSAGES: ChatMessage[] = [] +/** + * The view's $messages, live only while this surface is the VISIBLE tab. + * + * Keep-alive keeps every ever-active tab MOUNTED (tree-group.tsx), so without + * this gate a hidden tab re-renders its entire thread on every streaming + * delta flush (~30×/s) — five busy tabs quintuple the per-token render cost + * and the app crawls. Hidden tabs freeze their transcript instead (status + * dots stay live through the separate status atoms) and catch up in one + * commit on reveal — the subscribe fires immediately with the current value. + */ +function useMessagesWhileVisible($messages: ReadableAtom): ChatMessage[] { + const visible = usePaneVisible() + const [messages, setMessages] = useState(() => $messages.get()) + + // nanostores types the listener value ReadonlyIfObject; the store publishes + // a fresh array per flush, so the cast is safe and avoids a per-token clone. + useEffect( + () => (visible ? $messages.subscribe(value => setMessages(value as ChatMessage[])) : undefined), + [$messages, visible] + ) + + return messages +} + /** * Owns the $messages subscription and the assistant-ui external-store runtime. * @@ -193,7 +219,7 @@ function ChatRuntimeBoundary({ onThreadMessagesChange, suppressMessages }: ChatRuntimeBoundaryProps) { - const storeMessages = useStore(useSessionView().$messages) + const storeMessages = useMessagesWhileVisible(useSessionView().$messages) const messages = suppressMessages ? NO_MESSAGES : storeMessages const runtimeMessageRepository = useRuntimeMessageRepository(messages) @@ -285,11 +311,18 @@ export function ChatView({ const resumeExhaustedSessionId = useStore($resumeExhaustedSessionId) // Durable composer/queue scope (lineage root) so auto-compression tip rotation - // does not wipe an in-progress draft or orphan /queue entries. - const queueSessionKey = useMemo( - () => resolveComposerSessionKey(selectedSessionId, sessions), - [selectedSessionId, sessions] - ) + // does not wipe an in-progress draft or orphan /queue entries. For the + // primary view, the route is authoritative over the store selection — the + // latter can be momentarily null/stale mid-switch, which used to leak into + // the composer's scope key (#59305). A tile has no route, so it always uses + // its own selection directly. + const queueSessionKey = useMemo(() => { + const effectiveSelectedSessionId = isPrimary + ? primaryRouteSelectedSessionId(location.pathname, selectedSessionId) + : selectedSessionId + + return resolveComposerSessionKey(effectiveSelectedSessionId, sessions) + }, [isPrimary, location.pathname, selectedSessionId, sessions]) // When the tip row arrives after compression, migrate any tip-keyed stash onto // the durable lineage key before the composer remounts onto that key. @@ -437,6 +470,7 @@ export function ChatView({ 'relative isolate flex h-full min-w-0 flex-col overflow-hidden bg-(--ui-chat-surface-background)', className )} + data-chat-surface="" data-composer-target={composerScope.target} data-session-anchor={sessionAnchor} > diff --git a/apps/desktop/src/app/chat/right-rail/preview-console.tsx b/apps/desktop/src/app/chat/right-rail/preview-console.tsx index 67df7fefc2f..35001ab0029 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-console.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-console.tsx @@ -166,6 +166,7 @@ export function PreviewConsolePanel({ const sendableLogs = visibleSelection.length > 0 ? visibleSelection : logs const stickScrollRafRef = useRef(null) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!consoleShouldStickRef.current) { return diff --git a/apps/desktop/src/app/chat/right-rail/preview-file.tsx b/apps/desktop/src/app/chat/right-rail/preview-file.tsx index 13a20f61d62..a4eab020f69 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-file.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-file.tsx @@ -591,6 +591,7 @@ export function LocalFilePreview({ reloadKey, target }: { reloadKey: number; tar const filePath = filePathForTarget(target) const isImage = target.previewKind === 'image' + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { setUserMode(null) setEditing(false) diff --git a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx index f240c4b2067..9c8e4caf9ca 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx @@ -319,6 +319,7 @@ export function PreviewPane({ return () => setTitlebarToolGroup(TITLEBAR_GROUP_ID, []) }, [consoleOpen, consoleState, copy, devtoolsOpen, isWebPreview, setTitlebarToolGroup, toggleDevTools]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!consoleOpen) { return @@ -334,6 +335,7 @@ export function PreviewPane({ return () => window.cancelAnimationFrame(handle) }, [consoleOpen]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if ( !previewServerRestart || @@ -392,6 +394,7 @@ export function PreviewPane({ return () => window.clearTimeout(timer) }, [copy.stillWorking, previewServerRestart, restartingServer]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (reloadRequest === lastReloadRequestRef.current) { return @@ -498,6 +501,7 @@ export function PreviewPane({ } }, [appendConsoleEntry, copy, reloadPreview, target.kind, target.url]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const host = hostRef.current diff --git a/apps/desktop/src/app/chat/runtime-repository.ts b/apps/desktop/src/app/chat/runtime-repository.ts index 9dc94d42114..3dad84dac13 100644 --- a/apps/desktop/src/app/chat/runtime-repository.ts +++ b/apps/desktop/src/app/chat/runtime-repository.ts @@ -1,14 +1,27 @@ -import { ExportedMessageRepository, type ThreadMessage } from '@assistant-ui/react' +import { fromThreadMessageLike, getAutoStatus } from '@assistant-ui/core/internal' +import type { ExportedMessageRepository, ThreadMessage } from '@assistant-ui/react' import { useMemo, useRef } from 'react' import type { ChatMessage } from '@/lib/chat-messages' import { coalesceToolOnlyAssistants, createToolMergeCache, toRuntimeMessage } from '@/lib/chat-runtime' +// The exact fallback status ExportedMessageRepository.fromBranchableArray uses. +// Normalization happens HERE, once per message, so the cached record below is +// already the final ThreadMessage the runtime consumes. +const FALLBACK_STATUS = getAutoStatus(false, false, false, false, undefined) + /** * ChatMessage[] -> assistant-ui message repository, with a WeakMap identity * cache so unchanged messages convert once (and a tool-merge cache that folds * tool-only assistant turns into their neighbour). Shared by the main chat's * runtime boundary and session tiles — one transcript pipeline, N surfaces. + * + * The cache stores NORMALIZED messages. `fromBranchableArray` maps the whole + * array through `fromThreadMessageLike` on every call, so building the export + * with it threw away the cache's reference identity once per streamed delta — + * re-normalizing the entire settled transcript ~30x/s. Normalizing inside the + * cache miss keeps identity stable for settled turns, which is what lets the + * runtime reconcile detect that only the tail moved. */ export function useRuntimeMessageRepository(messages: ChatMessage[]): ExportedMessageRepository { const cacheRef = useRef(new WeakMap()) @@ -32,7 +45,9 @@ export function useRuntimeMessageRepository(messages: ChatMessage[]): ExportedMe } const cachedMessage = cacheRef.current.get(message) - const runtimeMessage = cachedMessage ?? toRuntimeMessage(message) + + const runtimeMessage = + cachedMessage ?? fromThreadMessageLike(toRuntimeMessage(message), message.id, FALLBACK_STATUS) if (!cachedMessage) { cacheRef.current.set(message, runtimeMessage) @@ -46,6 +61,6 @@ export function useRuntimeMessageRepository(messages: ChatMessage[]): ExportedMe } } - return ExportedMessageRepository.fromBranchableArray(items, { headId }) + return { headId, messages: items } }, [messages]) } diff --git a/apps/desktop/src/app/chat/session-drag.test.ts b/apps/desktop/src/app/chat/session-drag.test.ts new file mode 100644 index 00000000000..5e4b7955ee5 --- /dev/null +++ b/apps/desktop/src/app/chat/session-drag.test.ts @@ -0,0 +1,113 @@ +import type { PointerEvent as ReactPointerEvent } from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { group } from '@/components/pane-shell/tree/model' +import { $layoutTree } from '@/components/pane-shell/tree/store' +import { openSessionTile } from '@/store/session-states' + +import { requestComposerInsertRefs } from './composer/focus' +import { startSessionDrag } from './session-drag' + +/** + * A session drop resolves its target by rect-testing the chat surfaces in the + * document. A tab group keeps inactive tabs MOUNTED with their layout box + * intact, so a background tab's rect is identical to the foreground tab's — + * the drop has to land on the tab the user can actually see. + */ + +vi.mock('@/store/session-states', () => ({ openSessionTile: vi.fn() })) +vi.mock('./composer/focus', () => ({ requestComposerInsertRefs: vi.fn() })) + +const ZONE = { left: 0, top: 0, right: 1000, bottom: 800 } +const COMPOSER = { left: 100, top: 700, right: 900, bottom: 780 } + +const stubRect = (el: Element, box: { left: number; top: number; right: number; bottom: number }) => { + el.getBoundingClientRect = () => + ({ ...box, width: box.right - box.left, height: box.bottom - box.top, x: box.left, y: box.top }) as DOMRect +} + +/** The workspace tab kept alive behind an active session tile tab. */ +function mountStackedTabs() { + document.body.innerHTML = ` +
+
+
+
+
+
+
+
+
+
+
+
+
+ ` + + stubRect(document.querySelector('[data-tree-group]')!, ZONE) + + for (const surface of document.querySelectorAll('[data-session-anchor]')) { + stubRect(surface, ZONE) + } + + for (const composer of document.querySelectorAll('[data-slot="composer-root"]')) { + stubRect(composer, COMPOSER) + } + + $layoutTree.set(group(['workspace', 'session-tile:visible'], { id: 'g1' })) + + return document.getElementById('row')! +} + +/** Press on `source`, drag to (x, y), release. The drag session flushes its + * pending move synchronously on release, so no frame wait is needed. */ +function dragTo(source: HTMLElement, x: number, y: number) { + startSessionDrag({ id: 'dragged', profile: 'default', title: 'Dragged chat' }, { + button: 0, + clientX: 0, + clientY: 0, + currentTarget: source, + pointerId: 1 + } as unknown as ReactPointerEvent) + + window.dispatchEvent(new MouseEvent('pointermove', { bubbles: true, clientX: x, clientY: y })) + window.dispatchEvent(new MouseEvent('pointerup', { bubbles: true, clientX: x, clientY: y })) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +afterEach(() => { + document.body.innerHTML = '' + $layoutTree.set(null) +}) + +describe('session drop targeting across stacked tabs', () => { + it('links into the visible tab’s composer, not the tab kept alive behind it', () => { + const row = mountStackedTabs() + + dragTo(row, 500, 740) + + expect(requestComposerInsertRefs).toHaveBeenCalledWith(expect.anything(), { target: 'tile:visible' }) + }) + + it('docks a split against the visible tab’s pane', () => { + const row = mountStackedTabs() + + dragTo(row, 980, 400) + + expect(openSessionTile).toHaveBeenCalledWith('dragged', 'right', 'session-tile:visible', undefined) + expect(requestComposerInsertRefs).not.toHaveBeenCalled() + }) + + it('commits nothing over a zone that hosts no chat surface', () => { + mountStackedTabs() + $layoutTree.set(group(['terminal'], { id: 'g1' })) + + dragTo(document.getElementById('row')!, 500, 740) + + expect(requestComposerInsertRefs).not.toHaveBeenCalled() + expect(openSessionTile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/desktop/src/app/chat/session-drag.ts b/apps/desktop/src/app/chat/session-drag.ts index f99bfa47325..b4ca62802de 100644 --- a/apps/desktop/src/app/chat/session-drag.ts +++ b/apps/desktop/src/app/chat/session-drag.ts @@ -27,6 +27,7 @@ import type { PointerEvent as ReactPointerEvent } from 'react' +import { queryAllVisible } from '@/components/pane-shell/pane-visibility' import { findGroup } from '@/components/pane-shell/tree/model' import { type DoubleTapContext, @@ -66,8 +67,11 @@ const snapRect = (el: HTMLElement): ZoneRect => { return { left: r.left, top: r.top, right: r.right, bottom: r.bottom } } +/** Chat surfaces the pointer can land on. Inactive tabs are excluded: they stay + * mounted with their layout box intact, so their rect is identical to the + * visible tab's and a hit-test alone would pick whichever came first. */ function snapshotSurfaces(): SurfaceSnapshot[] { - return [...document.querySelectorAll('[data-session-anchor]')].map(el => ({ + return queryAllVisible('[data-session-anchor]').map(el => ({ anchor: el.dataset.sessionAnchor || 'workspace', composerTarget: el.dataset.composerTarget || 'main', rect: snapRect(el) @@ -123,7 +127,7 @@ export function startSessionDrag( zones = snapshotZones() strips = snapshotStrips() surfaces = snapshotSurfaces() - composers = [...document.querySelectorAll('[data-slot="composer-root"]')].map(snapRect) + composers = queryAllVisible('[data-slot="composer-root"]').map(snapRect) zoneHost = new Map(zones.map(zone => [zone.id, chatZonePane(zone.id)])) source?.style.setProperty('opacity', '0.45') // The same sentinel the zone overlay + chat surfaces key off — the diff --git a/apps/desktop/src/app/chat/session-tile-actions.ts b/apps/desktop/src/app/chat/session-tile-actions.ts index 4eca47f5c09..912878e53b0 100644 --- a/apps/desktop/src/app/chat/session-tile-actions.ts +++ b/apps/desktop/src/app/chat/session-tile-actions.ts @@ -24,10 +24,12 @@ import { resetSessionBackground } from '@/store/composer-status' import { notifyError } from '@/store/notifications' import { clearPreviewArtifacts } from '@/store/preview-status' import { clearAllPrompts } from '@/store/prompts' -import { $connection } from '@/store/session' +import { $connection, $sessions, sessionMatchesStoredId } from '@/store/session' import { $sessionStates, sessionTileDelegate } from '@/store/session-states' +import { broadcastSessionsChanged } from '@/store/session-sync' import { clearSessionSubagents } from '@/store/subagents' import { clearSessionTodos } from '@/store/todos' +import type { SessionInfo } from '@/types/hermes' import { uploadComposerAttachment } from '../session/hooks/use-prompt-actions' import { @@ -38,13 +40,54 @@ import { planEdit, planReload, planRestore, - runRewindSubmit + runRewindSubmit, + truncateSubmitParams } from '../session/hooks/use-prompt-actions/rewind' import { useSubmitPrompt } from '../session/hooks/use-prompt-actions/submit' import { type SubmitTextOptions } from '../session/hooks/use-prompt-actions/utils' +import { upsertOptimisticSession } from '../session/hooks/use-session-actions/utils' import type { ComposerScope } from './composer/scope' +/** + * List a tile's session in the sidebar/tab strip on its first send. + * + * A ⌘T tab's session is created UNLISTED (see `openNewSessionTile`), so it has + * no `$sessions` row until its first turn persists and a refresh surfaces it — + * for that whole first exchange the tab and the sidebar read "New session". + * ⌘N has no such gap: its session is created per-send and seeded with the + * user's text as the row preview. Seeding the same way here names the session + * within the first message; the server's auto-title supersedes it once the turn + * completes. + * + * No-ops on empty text and on a session that is already listed, so re-sends + * never clobber a real title with a raw message preview. + */ +export function listTileSessionRow(deps: { + cwd?: string + model?: string + preview: string + runtimeId: string + sessions: readonly SessionInfo[] + storedSessionId: string +}): boolean { + const preview = deps.preview.trim() + + if (!preview || deps.sessions.some(session => sessionMatchesStoredId(session, deps.storedSessionId))) { + return false + } + + upsertOptimisticSession( + { info: { cwd: deps.cwd, model: deps.model }, session_id: deps.runtimeId, stored_session_id: deps.storedSessionId }, + deps.storedSessionId, + null, + preview + ) + broadcastSessionsChanged() + + return true +} + interface SessionTileActionsArgs { runtimeId: string scope: ComposerScope @@ -89,6 +132,23 @@ export function useSessionTileActions({ runtimeId, scope, storedSessionId }: Ses const readState = useCallback(() => $sessionStates.get()[runtimeIdRef.current], []) const readMessages = useCallback(() => readState()?.messages ?? [], [readState]) + // A ⌘T tab's session is unlisted until its first turn persists — seed the + // row from the user's first message so the tab and sidebar name it right + // away (see listTileSessionRow). + const listTileSession = useCallback((preview: string) => { + const runtimeId = runtimeIdRef.current + const state = $sessionStates.get()[runtimeId] + + listTileSessionRow({ + cwd: state?.cwd, + model: state?.model, + preview, + runtimeId, + sessions: $sessions.get(), + storedSessionId: storedIdRef.current + }) + }, []) + // Tile-side attachment staging: same upload rules as the primary submit // (skip synced/pathless, byte-upload files+images), against the tile scope. const syncAttachmentsForSubmit = useCallback( @@ -162,6 +222,8 @@ export function useSessionTileActions({ runtimeId, scope, storedSessionId }: Ses const visibleText = rawText.trim() const attachments = options?.attachments ?? scope.attachments.$attachments.get() + listTileSession(visibleText) + if (!attachments.length && SLASH_COMMAND_RE.test(visibleText)) { triggerHaptic('selection') await sessionTileDelegate()?.executeSlash(visibleText, runtimeIdRef.current) @@ -171,24 +233,7 @@ export function useSessionTileActions({ runtimeId, scope, storedSessionId }: Ses return await submitPromptText(rawText, options) }, - [scope.attachments.$attachments, submitPromptText] - ) - - const appendSystemNote = useCallback( - (text: string) => { - update(state => ({ - ...state, - messages: [ - ...state.messages, - { - id: `system-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, - role: 'system', - parts: [textPart(text)] - } - ] - })) - }, - [update] + [listTileSession, scope.attachments.$attachments, submitPromptText] ) const cancelRun = useCallback(async () => { @@ -221,30 +266,84 @@ export function useSessionTileActions({ runtimeId, scope, storedSessionId }: Ses const steerPrompt = useCallback( async (rawText: string): Promise => { const text = rawText.trim() + const sessionId = runtimeIdRef.current - if (!text) { + if (!text || !sessionId) { return false } + const messageId = `user-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + + const mutate = (updater: (state: ClientSessionState) => ClientSessionState) => + sessionTileDelegate()?.updateSession(sessionId, updater) + + // Match the primary composer: insert the correction before the active + // reply before awaiting the redirect RPC, whose completion can race us. + mutate(state => { + const message = { + id: messageId, + role: 'user' as const, + parts: [textPart(text)] + } + + const streamIndex = state.streamId ? state.messages.findIndex(candidate => candidate.id === state.streamId) : -1 + + const lastAssistantIndex = state.messages.map(candidate => candidate.role).lastIndexOf('assistant') + const insertionIndex = streamIndex >= 0 ? streamIndex : lastAssistantIndex + + const messages = + insertionIndex >= 0 + ? [...state.messages.slice(0, insertionIndex), message, ...state.messages.slice(insertionIndex)] + : [...state.messages, message] + + return { ...state, messages } + }) + + const discardOptimisticMessage = () => + mutate(state => ({ + ...state, + messages: state.messages.filter(message => message.id !== messageId) + })) + + const moveOptimisticMessageToEnd = () => + mutate(state => { + const message = state.messages.find(candidate => candidate.id === messageId) + + return message + ? { ...state, messages: [...state.messages.filter(candidate => candidate.id !== messageId), message] } + : state + }) + try { - const result = await requestGateway<{ status?: string }>('session.steer', { - session_id: runtimeIdRef.current, + const result = await requestGateway<{ status?: string }>('session.redirect', { + session_id: sessionId, text }) - if (result?.status === 'queued') { + if (result?.status === 'redirected') { + triggerHaptic('submit') + + return true + } + + if (result?.status === 'queued') { + moveOptimisticMessageToEnd() triggerHaptic('submit') - appendSystemNote(`steer:${text}`) return true } } catch { + discardOptimisticMessage() // Swallow — the caller queues the text so nothing is lost. + + return false } + discardOptimisticMessage() + return false }, - [appendSystemNote, requestGateway] + [requestGateway] ) // Rewind primitive (interrupt-first for live turns, busy-retry) — shared with @@ -274,7 +373,11 @@ export function useSessionTileActions({ runtimeId, scope, storedSessionId }: Ses try { await requestGateway( 'prompt.submit', - { session_id: runtimeIdRef.current, text: plan.text, truncate_before_user_ordinal: plan.truncateOrdinal }, + { + session_id: runtimeIdRef.current, + text: plan.text, + ...truncateSubmitParams(plan.truncateOrdinal) + }, PROMPT_SUBMIT_REQUEST_TIMEOUT_MS ) } catch (err) { diff --git a/apps/desktop/src/app/chat/session-tile-row.test.ts b/apps/desktop/src/app/chat/session-tile-row.test.ts new file mode 100644 index 00000000000..2c6eb0479e6 --- /dev/null +++ b/apps/desktop/src/app/chat/session-tile-row.test.ts @@ -0,0 +1,82 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { $sessions } from '@/store/session' +import type { SessionInfo } from '@/types/hermes' + +import { listTileSessionRow } from './session-tile-actions' + +const STORED = 'stored-tab' +const RUNTIME = 'runtime-tab' + +function row(overrides: Partial = {}): SessionInfo { + return { + cwd: null, + ended_at: null, + id: STORED, + input_tokens: 0, + is_active: true, + last_active: 1, + message_count: 1, + model: null, + output_tokens: 0, + parent_session_id: null, + preview: null, + source: 'desktop', + started_at: 1, + title: null, + tool_call_count: 0, + ...overrides + } +} + +function seed(preview: string, sessions: SessionInfo[] = $sessions.get()) { + return listTileSessionRow({ + cwd: '/work/repo', + model: 'claude-opus-5', + preview, + runtimeId: RUNTIME, + sessions, + storedSessionId: STORED + }) +} + +describe('listTileSessionRow', () => { + beforeEach(() => $sessions.set([])) + afterEach(() => $sessions.set([])) + + it("lists an unlisted ⌘T tab from the user's first message", () => { + expect(seed('fix the tab titles')).toBe(true) + + const listed = $sessions.get().find(session => session.id === STORED) + + expect(listed?.preview).toBe('fix the tab titles') + expect(listed?.cwd).toBe('/work/repo') + // Title stays null so the server's auto-title is what eventually wins. + expect(listed?.title).toBeNull() + }) + + it('leaves an already-listed session alone so a real title survives re-sends', () => { + const titled = row({ title: 'Tab title fix' }) + + expect(seed('second message', [titled])).toBe(false) + expect($sessions.get()).toHaveLength(0) + }) + + it('matches a listed session across compression by lineage root', () => { + const rotated = row({ _lineage_root_id: STORED, id: 'stored-tab-compressed-2' }) + + expect(seed('after compression', [rotated])).toBe(false) + }) + + it('does not list a session on an empty or whitespace-only send', () => { + expect(seed('')).toBe(false) + expect(seed(' ')).toBe(false) + expect($sessions.get()).toHaveLength(0) + }) + + it('trims the seeded preview', () => { + seed(' padded prompt ') + + expect($sessions.get()[0]?.preview).toBe('padded prompt') + }) +}) diff --git a/apps/desktop/src/app/chat/session-tile.tsx b/apps/desktop/src/app/chat/session-tile.tsx index 6562880b58f..6ca29e70ade 100644 --- a/apps/desktop/src/app/chat/session-tile.tsx +++ b/apps/desktop/src/app/chat/session-tile.tsx @@ -113,7 +113,7 @@ function TileChat({ storedSessionId: string view: SessionView }) { - const { gatewayRef, requestGateway } = useGatewayRequest() + const { gateway, requestGateway } = useGatewayRequest() const queryClient = useQueryClient() const { selectModel } = useModelControls({ queryClient, requestGateway }) const activeGatewayProfile = useStore($activeGatewayProfile) @@ -151,20 +151,20 @@ function TileChat({ () => gatewayOpen ? ( ) : null, - [activeGatewayProfile, gatewayOpen, gatewayRef, requestGateway, selectModel] + [activeGatewayProfile, gateway, gatewayOpen, requestGateway, selectModel] ) return ( composer.addContextRefAttachment(`@url:${formatRefValue(url)}`, url)} @@ -253,6 +253,7 @@ export function SessionTilePane({ storedSessionId }: { storedSessionId: string } // session.resume before the gateway is OPEN. Persisted tiles mount at boot // while it's still connecting — an ungated resume rejected there and // latched every restored tile into the error card. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!gatewayOpen || runtimeId || tile?.error || resumingRef.current) { return diff --git a/apps/desktop/src/app/chat/session-view.test.ts b/apps/desktop/src/app/chat/session-view.test.ts new file mode 100644 index 00000000000..366af2327bc --- /dev/null +++ b/apps/desktop/src/app/chat/session-view.test.ts @@ -0,0 +1,88 @@ +import { cleanup } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { createClientSessionState } from '@/lib/chat-runtime' +import { $activeSessionId, $busy, $messages } from '@/store/session' +import { $sessionStates, dropSessionState, publishSessionState } from '@/store/session-states' + +import { PRIMARY_SESSION_VIEW } from './session-view' + +const message = (id: string, text: string) => ({ + id, + parts: [{ type: 'text' as const, text }], + role: 'assistant' as const +}) + +const stateWith = (runtimeId: string, text: string, busy: boolean) => ({ + ...createClientSessionState(`stored-${runtimeId}`), + messages: [message(`${runtimeId}-msg`, text)], + busy +}) + +/** + * The workspace pane is just the first tab: it renders from the active + * session's own `$sessionStates` slice, exactly like a ⌘T tile. + * + * The regression this guards: the pane used to render straight off the global + * `$messages`/`$busy` atoms — a mirror of whichever session was active. With + * two turns in flight, navigating away from a still-streaming session left it + * painting into the surface now showing a different conversation. + */ +describe('primary session view reads its own session slice', () => { + beforeEach(() => { + $sessionStates.set({}) + $activeSessionId.set(null) + $messages.set([]) + $busy.set(false) + }) + + afterEach(cleanup) + + it('shows the active session transcript, not a background session still streaming', () => { + publishSessionState('runtime-background', stateWith('runtime-background', 'background turn', true)) + publishSessionState('runtime-foreground', stateWith('runtime-foreground', 'foreground turn', false)) + + $activeSessionId.set('runtime-foreground') + + expect(PRIMARY_SESSION_VIEW.$messages.get()).toEqual([message('runtime-foreground-msg', 'foreground turn')]) + expect(PRIMARY_SESSION_VIEW.$busy.get()).toBe(false) + }) + + it('ignores a background session that keeps streaming after the user switches away', () => { + publishSessionState('runtime-a', stateWith('runtime-a', 'session A turn', true)) + $activeSessionId.set('runtime-b') + publishSessionState('runtime-b', stateWith('runtime-b', 'session B turn', false)) + + // Session A streams on: another delta lands for the session the user left. + publishSessionState('runtime-a', { + ...stateWith('runtime-a', 'session A turn', true), + messages: [message('runtime-a-msg', 'session A turn'), message('runtime-a-late', 'late delta')] + }) + + expect(PRIMARY_SESSION_VIEW.$messages.get()).toEqual([message('runtime-b-msg', 'session B turn')]) + expect(PRIMARY_SESSION_VIEW.$lastVisibleIsUser.get()).toBe(false) + expect(PRIMARY_SESSION_VIEW.$busy.get()).toBe(false) + }) + + it('falls back to the draft atoms while the chat has no runtime session yet', () => { + $messages.set([message('draft-msg', 'unsent draft')]) + $busy.set(true) + + expect(PRIMARY_SESSION_VIEW.$runtimeId.get()).toBeNull() + expect(PRIMARY_SESSION_VIEW.$messages.get()).toEqual([message('draft-msg', 'unsent draft')]) + expect(PRIMARY_SESSION_VIEW.$busy.get()).toBe(true) + expect(PRIMARY_SESSION_VIEW.$messagesEmpty.get()).toBe(false) + }) + + it('returns to the draft atoms when the active session state is dropped', () => { + publishSessionState('runtime-a', stateWith('runtime-a', 'session A turn', true)) + $activeSessionId.set('runtime-a') + + expect(PRIMARY_SESSION_VIEW.$messages.get()).toEqual([message('runtime-a-msg', 'session A turn')]) + + dropSessionState('runtime-a') + + expect(PRIMARY_SESSION_VIEW.$messages.get()).toEqual([]) + expect(PRIMARY_SESSION_VIEW.$messagesEmpty.get()).toBe(true) + }) +}) diff --git a/apps/desktop/src/app/chat/session-view.tsx b/apps/desktop/src/app/chat/session-view.tsx index 9ca39ace67e..97ec1d0a376 100644 --- a/apps/desktop/src/app/chat/session-view.tsx +++ b/apps/desktop/src/app/chat/session-view.tsx @@ -1,6 +1,7 @@ -import type { ReadableAtom } from 'nanostores' +import { computed, type ReadableAtom } from 'nanostores' import { createContext, useContext } from 'react' +import type { ClientSessionState } from '@/app/types' import type { ChatMessage } from '@/lib/chat-messages' import { $activeSessionId, @@ -11,18 +12,28 @@ import { $currentModel, $currentProvider, $currentReasoningEffort, - $lastVisibleMessageIsUser, $messages, - $messagesEmpty, $selectedStoredSessionId } from '@/store/session' +import { $sessionStates } from '@/store/session-states' + +import { lastVisibleMessageIsUser } from './thread-loading' /** - * SESSION VIEW — the store surface a ChatView renders from. The PRIMARY view - * is the app's classic global atoms (route-driven active session, untouched - * fast path). A session TILE provides the same shape computed from its - * session's slice of `$sessionStates`, so the identical ChatView tree renders - * either — one chat surface, N sessions on screen. + * SESSION VIEW — the store surface a ChatView renders from. Every session, + * including the one in the workspace pane, renders from ITS OWN slice of + * `$sessionStates`. The workspace pane is just the first tab: a session + * surface with no privileged state of its own. + * + * That symmetry is load-bearing. The pane used to render off the global + * `$messages`/`$busy` atoms — a mirror of whichever session was active — so + * with two turns in flight (⌘T tabs made that routine), navigating away from + * a still-streaming session left it painting into the surface now showing a + * different conversation. Reading the per-session slice makes that + * structurally impossible rather than merely guarded. + * + * The global atoms stay the DRAFT surface: a new chat has no runtime id, and + * therefore no slice, until its first turn creates one. * * Everything is atoms (not values) so subscription granularity survives: * ChatView subscribes only to the coarse edges; `$messages` stays boundary- @@ -44,18 +55,39 @@ export interface SessionView { $reasoningEffort: ReadableAtom } +/** The active session's own slice, or `undefined` while it's a draft. */ +const $primaryState = computed([$activeSessionId, $sessionStates], (runtimeId, states) => + runtimeId ? states[runtimeId] : undefined +) + +/** + * Read one field from the active session's slice, falling back to the global + * draft atom while no runtime exists yet. Once a session HAS a slice, that + * slice is authoritative — a background session publishing its own state can + * never reach this view. + */ +function primaryField(select: (state: ClientSessionState) => T, $draft: ReadableAtom): ReadableAtom { + const $field: ReadableAtom = computed([$primaryState, $draft], (state, draft: T) => + state ? select(state) : draft + ) + + return $field +} + +const $primaryMessages = primaryField(state => state.messages, $messages) + export const PRIMARY_SESSION_VIEW: SessionView = { kind: 'primary', - $awaitingResponse, - $busy, - $cwd: $currentCwd, - $fast: $currentFastMode, - $lastVisibleIsUser: $lastVisibleMessageIsUser, - $messages, - $messagesEmpty, - $model: $currentModel, - $provider: $currentProvider, - $reasoningEffort: $currentReasoningEffort, + $awaitingResponse: primaryField(state => state.awaitingResponse, $awaitingResponse), + $busy: primaryField(state => state.busy, $busy), + $cwd: primaryField(state => state.cwd, $currentCwd), + $fast: primaryField(state => state.fast, $currentFastMode), + $lastVisibleIsUser: computed($primaryMessages, lastVisibleMessageIsUser), + $messages: $primaryMessages, + $messagesEmpty: computed($primaryMessages, messages => messages.length === 0), + $model: primaryField(state => state.model, $currentModel), + $provider: primaryField(state => state.provider, $currentProvider), + $reasoningEffort: primaryField(state => state.reasoningEffort, $currentReasoningEffort), $runtimeId: $activeSessionId, $storedId: $selectedStoredSessionId } diff --git a/apps/desktop/src/app/chat/sidebar/chrome.tsx b/apps/desktop/src/app/chat/sidebar/chrome.tsx index 7815d1fafcf..2a0f728503a 100644 --- a/apps/desktop/src/app/chat/sidebar/chrome.tsx +++ b/apps/desktop/src/app/chat/sidebar/chrome.tsx @@ -43,6 +43,22 @@ export function SidebarRowNest({ className, ...props }: React.ComponentProps<'di return } +/** + * Chronological date-bucket separator ("Yesterday" / "Last week" / "June") for + * the session list. One flat row — a small caption plus a hairline rule — so it + * groups sessions by recency without adding a level of indentation. + */ +export function SidebarDateDivider({ className, label, ...props }: React.ComponentProps<'div'> & { label: string }) { + return ( +
+ + {label} + +
+ ) +} + /** Outer grid — sole owner of row height. */ export function SidebarRowShell({ actions, diff --git a/apps/desktop/src/app/chat/sidebar/index.tsx b/apps/desktop/src/app/chat/sidebar/index.tsx index 5b3f2b4b30c..91dec1e2d58 100644 --- a/apps/desktop/src/app/chat/sidebar/index.tsx +++ b/apps/desktop/src/app/chat/sidebar/index.tsx @@ -115,6 +115,7 @@ import { orderByIds, reconcileOrderIds, resolveManualSessionOrderIds, sameIds } import { ProfileRail } from './profile-switcher' import { ProjectDialog } from './project-dialog' import { + orderProjectsByIds, overlayLiveLanes, overlayLivePreviews, PROJECT_PREVIEW_COUNT, @@ -622,8 +623,9 @@ export function ChatSidebar({ ) // Layer the user's manual drag-order on top of the deterministic sort. Empty - // (default) returns `sorted` untouched; new projects surface on top. - return orderByIds(sorted, project => project.id, projectOrderIds) + // (default) returns `sorted` untouched; projects the user hasn't ordered yet + // keep their sorted position rather than jumping the hand-picked list. + return orderProjectsByIds(sorted, projectOrderIds) }, [showAllProfiles, projectTree, dismissedAutoProjects, orderRepos, activeProjectId, projectOrderIds]) // The overview only renders in grouped mode; the model stays live regardless @@ -718,6 +720,7 @@ export function ChatSidebar({ // only the cheap per-repo `git worktree list`, never the heavy tree scan. const prevWorkingIdsRef = useRef(workingSessionIds) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const prev = prevWorkingIdsRef.current prevWorkingIdsRef.current = workingSessionIds @@ -754,6 +757,7 @@ export function ChatSidebar({ [currentCwd] ) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!inProject || !enteredProject) { lastProjectCwdSyncRef.current = null @@ -1239,7 +1243,7 @@ export function ChatSidebar({ {!trimmedQuery && ( } label={s.pinned} @@ -1275,6 +1279,7 @@ export function ChatSidebar({ // virtualized long list, which must keep its own scroller. !recentsVirtualizes && COMPACT_FLAT )} + dateGrouped={inProject || !agentOrderManual} dndSensors={dndSensors} emptyState={ showSessionSkeletons ? ( diff --git a/apps/desktop/src/app/chat/sidebar/profile-switcher.tsx b/apps/desktop/src/app/chat/sidebar/profile-switcher.tsx index 52fa8de19c0..e57fec12079 100644 --- a/apps/desktop/src/app/chat/sidebar/profile-switcher.tsx +++ b/apps/desktop/src/app/chat/sidebar/profile-switcher.tsx @@ -213,6 +213,7 @@ export function ProfileRail() { const createRequest = useStore($profileCreateRequest) const lastCreateRef = useRef(createRequest) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (createRequest === lastCreateRef.current) { return diff --git a/apps/desktop/src/app/chat/sidebar/projects/index.ts b/apps/desktop/src/app/chat/sidebar/projects/index.ts index dea53d1fe6f..f30d075a0f5 100644 --- a/apps/desktop/src/app/chat/sidebar/projects/index.ts +++ b/apps/desktop/src/app/chat/sidebar/projects/index.ts @@ -1,6 +1,12 @@ // Public surface of the project/worktree sidebar, consumed by the sidebar root. export { EnteredProjectContent } from './entered-content' -export { PROJECT_PREVIEW_COUNT, projectTreeCwd, sortProjectsForOverview, useRepoWorktreeMap } from './model' +export { + orderProjectsByIds, + PROJECT_PREVIEW_COUNT, + projectTreeCwd, + sortProjectsForOverview, + useRepoWorktreeMap +} from './model' export { ProjectBackRow, ProjectOverviewRow } from './overview-row' export { ProjectMenu } from './project-menu' export { SidebarWorkspaceGroup } from './workspace-group' diff --git a/apps/desktop/src/app/chat/sidebar/projects/model.test.ts b/apps/desktop/src/app/chat/sidebar/projects/model.test.ts new file mode 100644 index 00000000000..c56912fc862 --- /dev/null +++ b/apps/desktop/src/app/chat/sidebar/projects/model.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest' + +import { orderProjectsByIds } from './model' +import type { SidebarProjectTree } from './workspace-groups' + +function makeProject(id: string, sessionCount: number): SidebarProjectTree { + return { + id, + isAuto: true, + label: id, + lastActive: 0, + path: `/repos/${id}`, + previewSessions: [], + repos: [], + sessionCount + } +} + +const ids = (projects: SidebarProjectTree[]) => projects.map(project => project.id) + +describe('orderProjectsByIds', () => { + it('leaves the deterministic sort alone when nothing has been dragged', () => { + const projects = [makeProject('a', 0), makeProject('b', 2)] + + expect(orderProjectsByIds(projects, [])).toBe(projects) + }) + + it('applies the saved manual order', () => { + const projects = [makeProject('a', 1), makeProject('b', 1), makeProject('c', 1)] + + expect(ids(orderProjectsByIds(projects, ['c', 'a', 'b']))).toEqual(['c', 'a', 'b']) + }) + + it('keeps freshly-scanned zero-session repos below the hand-ordered list', () => { + // The regression: a disk scan keeps finding git checkouts the user has + // never opened in Hermes. Surfacing every unsaved id at the top buried the + // projects they deliberately dragged into place. + const projects = [makeProject('scanned-1', 0), makeProject('mine', 4), makeProject('scanned-2', 0)] + + expect(ids(orderProjectsByIds(projects, ['mine']))).toEqual(['mine', 'scanned-1', 'scanned-2']) + }) + + it('still surfaces a new project that has real activity', () => { + // A project you just started working in should not sink beneath the saved + // order — only the zero-session discoveries do. + const projects = [makeProject('ordered', 1), makeProject('just-started', 3)] + + expect(ids(orderProjectsByIds(projects, ['ordered']))).toEqual(['just-started', 'ordered']) + }) + + it('drops ids that are no longer present', () => { + const projects = [makeProject('a', 1)] + + expect(ids(orderProjectsByIds(projects, ['gone', 'a']))).toEqual(['a']) + }) +}) diff --git a/apps/desktop/src/app/chat/sidebar/projects/model.ts b/apps/desktop/src/app/chat/sidebar/projects/model.ts index 1e55db2fe41..f3510e2061d 100644 --- a/apps/desktop/src/app/chat/sidebar/projects/model.ts +++ b/apps/desktop/src/app/chat/sidebar/projects/model.ts @@ -75,6 +75,39 @@ export function sortProjectsForOverview( }) } +// Layer the user's manual drag-order over the deterministic sort. +// +// This can't just be `orderByIds`: that surfaces every id missing from the saved +// order at the TOP, which is right for sessions (a new chat should not sink) but +// wrong here. The overview also lists repos found by the disk scan that have +// zero Hermes sessions, and those arrive continuously — so once the user dragged +// anything, every freshly-scanned checkout jumped above the projects they +// actually work in. +// +// Fresh projects keep their place in the deterministic sort instead: ones with +// real activity go on top (a project you just started still surfaces), and +// zero-session discoveries sink below the hand-ordered list. +export function orderProjectsByIds(projects: SidebarProjectTree[], orderIds: string[]): SidebarProjectTree[] { + if (!orderIds.length) { + return projects + } + + const byId = new Map(projects.map(project => [project.id, project])) + const ordered = orderIds.map(id => byId.get(id)).filter((p): p is SidebarProjectTree => Boolean(p)) + const seen = new Set(ordered.map(project => project.id)) + const fresh = projects.filter(project => !seen.has(project.id)) + + if (!fresh.length) { + return ordered + } + + return [ + ...fresh.filter(project => project.sessionCount > 0), + ...ordered, + ...fresh.filter(project => project.sessionCount <= 0) + ] +} + // Project drill-in lanes are git-driven: source them from `git worktree list` so // linked worktrees still appear even when their sessions aren't in the recents // payload currently loaded in memory. diff --git a/apps/desktop/src/app/chat/sidebar/session-row.tsx b/apps/desktop/src/app/chat/sidebar/session-row.tsx index c1291a27b15..7de51f3de43 100644 --- a/apps/desktop/src/app/chat/sidebar/session-row.tsx +++ b/apps/desktop/src/app/chat/sidebar/session-row.tsx @@ -163,7 +163,9 @@ export function SidebarSessionRow({ style={style} {...rest} > - {sessionShowsRunningArc({ isWorking, needsInput }) &&