mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Merge remote-tracking branch 'origin/main' into wake-toggle-config
# Conflicts: # uv.lock
This commit is contained in:
commit
7d951032b2
1147 changed files with 113320 additions and 6464 deletions
5
.github/workflows/ci.yml
vendored
5
.github/workflows/ci.yml
vendored
|
|
@ -115,6 +115,11 @@ jobs:
|
|||
needs: detect
|
||||
uses: ./.github/workflows/uv-lockfile-check.yml
|
||||
|
||||
infographic-check:
|
||||
name: Check no committed infographics
|
||||
needs: detect
|
||||
uses: ./.github/workflows/infographic-check.yml
|
||||
|
||||
lockfile-diff:
|
||||
name: package-lock.json diff
|
||||
needs: detect
|
||||
|
|
|
|||
78
.github/workflows/infographic-check.yml
vendored
Normal file
78
.github/workflows/infographic-check.yml
vendored
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
name: Infographic Check
|
||||
|
||||
# Rejects PRs that commit PR-infographic images into the repo.
|
||||
#
|
||||
# PR infographics are rendered to an image-provider URL (fal.media) and
|
||||
# embedded in the PR *description*. The PR body is the archive; the binary
|
||||
# never belongs in git history.
|
||||
#
|
||||
# This has now leaked twice. PR #48261 removed the first batch, PR #54564
|
||||
# removed a second batch and added `infographic/` to `.gitignore` — but
|
||||
# `.gitignore` only stops *accidental* `git add`. It does nothing against
|
||||
# `git add -f`, and it does nothing for a path that does not literally match
|
||||
# the ignore pattern. Nine more PNGs (~14MB) were committed in the four
|
||||
# weeks AFTER that rule landed, plus PR #70552 caught an `infograficos/`
|
||||
# spelling that sidestepped the pattern entirely.
|
||||
#
|
||||
# A passive ignore rule cannot enforce a policy. This check can.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
review_status:
|
||||
description: "JSON array of review_status objects for the synthesizer."
|
||||
value: ${{ jobs.check-no-committed-infographics.outputs.review_status }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check-no-committed-infographics:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
review_status: ${{ steps.infographic-check.outputs.review_status }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- id: infographic-check
|
||||
name: Reject committed PR-infographic images
|
||||
run: |
|
||||
# Match on the IMAGE, not on a directory name. Keying this to
|
||||
# `infographic/` is what let `infograficos/` through in #70552 —
|
||||
# any localized or typo'd directory would sidestep it again.
|
||||
# Instead: find tracked raster images whose path contains an
|
||||
# infographic-ish segment, in any spelling, at any depth.
|
||||
#
|
||||
# `docs/assets` and `website/` legitimately hold product imagery
|
||||
# and are excluded; those are referenced from shipped docs pages.
|
||||
OFFENDERS=$(git ls-files -z \
|
||||
| tr '\0' '\n' \
|
||||
| grep -iE '(^|/)(infograph|infograf)[^/]*/' \
|
||||
| grep -iE '\.(png|jpe?g|webp|gif)$' \
|
||||
|| true)
|
||||
|
||||
if [ -n "$OFFENDERS" ]; then
|
||||
COUNT=$(printf '%s\n' "$OFFENDERS" | wc -l | tr -d ' ')
|
||||
STATUS='[{"source":"committed infographics","results":[{"kind":"action_required","title":"PR infographic committed to the repo","summary":"Infographic images belong in the PR description, never in git.","detail":"","how_to_fix":"Untrack the image and reference the provider URL from the PR body instead:\n```\ngit rm --cached <path-to-image>\n```\nThen put it in the PR description:\n```\n## Infographic\n\n\n```\n"}]}]'
|
||||
echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT"
|
||||
echo ""
|
||||
echo "::error::${COUNT} PR-infographic image(s) are tracked in git."
|
||||
echo ""
|
||||
printf '%s\n' "$OFFENDERS" | sed 's/^/ /'
|
||||
echo ""
|
||||
echo "PR infographics are rendered to an image-provider URL and"
|
||||
echo "embedded in the PR DESCRIPTION. The PR body is the archive —"
|
||||
echo "the binary never enters git history."
|
||||
echo ""
|
||||
echo "This rule has been re-established twice already (#48261,"
|
||||
echo "#54564) and leaked both times, because .gitignore cannot stop"
|
||||
echo "'git add -f' or a differently-spelled directory (#70552)."
|
||||
echo ""
|
||||
echo "To fix:"
|
||||
echo " git rm --cached <path> # keeps your local copy"
|
||||
echo " # then embed the provider URL in the PR description"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::No committed PR-infographic images."
|
||||
echo "review_status=[]" >> "$GITHUB_OUTPUT"
|
||||
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -1,6 +1,8 @@
|
|||
.DS_Store
|
||||
/venv/
|
||||
/venv.old/
|
||||
/venv.stale.runtime-*/
|
||||
/.hermes-runtime/
|
||||
/_pycache/
|
||||
*.pyc*
|
||||
__pycache__/
|
||||
|
|
@ -150,6 +152,11 @@ docs/superpowers/*
|
|||
.update-incomplete
|
||||
.update-incomplete.lock
|
||||
|
||||
# Checkout fingerprint the __pycache__ tree was last validated against
|
||||
# (launch-time stale-bytecode sweep). Runtime state, never a code change.
|
||||
.bytecode-fingerprint
|
||||
.bytecode-fingerprint.tmp
|
||||
|
||||
# Installer-written method stamp in the managed checkout root (scripts/install.sh).
|
||||
# Runtime metadata only — never a code change. Ignore so `git status` stays clean
|
||||
# and `hermes update`'s untracked autostash does not treat it as a local edit (#66189 / #54855).
|
||||
|
|
@ -173,5 +180,13 @@ apps/desktop/demo/
|
|||
# image-provider (fal.media) URL — they are NEVER committed to the repo. The
|
||||
# PR body is the archive. See the hermes-agent-dev skill's
|
||||
# pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1).
|
||||
#
|
||||
# Spelling variants are listed because a single `infographic/` pattern was
|
||||
# sidestepped by an `infograficos/` directory (#70552). .gitignore is only
|
||||
# the first line of defence and cannot stop `git add -f` at all — the
|
||||
# infographic-check CI job is what actually enforces this.
|
||||
infographic/
|
||||
infographics/
|
||||
infograficos/
|
||||
infografico/
|
||||
native/fts5_cjk/*.so
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ class AIAgent:
|
|||
provider: str = None,
|
||||
api_mode: str = None, # "chat_completions" | "codex_responses" | ...
|
||||
model: str = "", # empty → resolved from config/provider later
|
||||
max_iterations: int = 90, # tool-calling iterations (shared with subagents)
|
||||
max_iterations: int = 500, # tool-calling iterations (shared with subagents)
|
||||
enabled_toolsets: list = None,
|
||||
disabled_toolsets: list = None,
|
||||
quiet_mode: bool = False,
|
||||
|
|
|
|||
59
Dockerfile
59
Dockerfile
|
|
@ -1,3 +1,45 @@
|
|||
# Debian 13 still ships SQLite 3.46.1, which contains the upstream WAL-reset
|
||||
# corruption bug. Build a pinned shared library for the runtime image instead
|
||||
# of relying on a distro backport that trixie does not currently provide.
|
||||
# See #70480 and https://sqlite.org/wal.html#walresetbug.
|
||||
FROM debian:13.4 AS sqlite_build
|
||||
ARG SQLITE_AUTOCONF_VERSION=3530400
|
||||
ARG SQLITE_SHA256=0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c
|
||||
RUN apt-get -o Acquire::Retries=3 update && \
|
||||
apt-get -o Acquire::Retries=3 install -y --no-install-recommends \
|
||||
build-essential ca-certificates curl && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
(curl -fsSL --retry 1 --retry-all-errors --connect-timeout 15 --max-time 60 \
|
||||
-o /tmp/sqlite.tar.gz \
|
||||
"https://sqlite.org/2026/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz" || \
|
||||
curl -fsSL --retry 3 --retry-all-errors --connect-timeout 15 --max-time 120 \
|
||||
-o /tmp/sqlite.tar.gz \
|
||||
"https://sources.buildroot.net/sqlite/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz") && \
|
||||
printf '%s %s\n' "${SQLITE_SHA256}" /tmp/sqlite.tar.gz > /tmp/sqlite.sha256 && \
|
||||
sha256sum -c /tmp/sqlite.sha256 && \
|
||||
tar -xzf /tmp/sqlite.tar.gz -C /tmp && \
|
||||
cd "/tmp/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}" && \
|
||||
CFLAGS="-O2 \
|
||||
-DSQLITE_ENABLE_FTS3 \
|
||||
-DSQLITE_ENABLE_FTS3_PARENTHESIS \
|
||||
-DSQLITE_ENABLE_FTS4 \
|
||||
-DSQLITE_ENABLE_FTS5 \
|
||||
-DSQLITE_ENABLE_RTREE \
|
||||
-DSQLITE_ENABLE_GEOPOLY \
|
||||
-DSQLITE_ENABLE_COLUMN_METADATA \
|
||||
-DSQLITE_ENABLE_UNLOCK_NOTIFY \
|
||||
-DSQLITE_ENABLE_DBSTAT_VTAB \
|
||||
-DSQLITE_ENABLE_DBPAGE_VTAB \
|
||||
-DSQLITE_ENABLE_MATH_FUNCTIONS \
|
||||
-DSQLITE_ENABLE_PREUPDATE_HOOK \
|
||||
-DSQLITE_ENABLE_SESSION \
|
||||
-DSQLITE_SECURE_DELETE \
|
||||
-DSQLITE_THREADSAFE=1 \
|
||||
-DSQLITE_MAX_VARIABLE_NUMBER=250000" \
|
||||
./configure --prefix=/opt/sqlite-fixed --disable-static && \
|
||||
make -j"$(nproc)" && \
|
||||
make install
|
||||
|
||||
FROM ghcr.io/astral-sh/uv:0.11.6-python3.13-trixie@sha256:b3c543b6c4f23a5f2df22866bd7857e5d304b67a564f4feab6ac22044dde719b AS uv_source
|
||||
# Node 22 LTS source stage. Debian trixie's bundled nodejs is pinned to 20.x
|
||||
# which reached EOL in April 2026 — we copy node + npm + corepack from the
|
||||
|
|
@ -31,6 +73,23 @@ RUN apt-get -o Acquire::Retries=3 update && \
|
|||
ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev procps git openssh-client docker-cli xz-utils && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Prefer the fixed SQLite over Debian's vulnerable libsqlite3.so.0. Keep the
|
||||
# public library name stable so both the system interpreter and the uv-created
|
||||
# venv resolve the replacement without changing Python import paths.
|
||||
COPY --from=sqlite_build /opt/sqlite-fixed/lib/libsqlite3.so.3.53.4 /usr/local/lib/
|
||||
RUN ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so.0 && \
|
||||
ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so && \
|
||||
printf '/usr/local/lib\n' > /etc/ld.so.conf.d/000-sqlite-fixed.conf && \
|
||||
ldconfig && \
|
||||
python3 -c "import sqlite3, sys; \
|
||||
v = sqlite3.sqlite_version_info; \
|
||||
sys.exit(f'linked SQLite {sqlite3.sqlite_version} still has the WAL-reset bug') if v < (3, 51, 3) else None; \
|
||||
db = sqlite3.connect(':memory:'); \
|
||||
db.execute(\"CREATE VIRTUAL TABLE docs USING fts5(content, tokenize='trigram')\"); \
|
||||
db.execute(\"INSERT INTO docs VALUES ('hermes')\"); \
|
||||
sys.exit('SQLite FTS5 trigram self-test failed') if db.execute(\"SELECT count(*) FROM docs WHERE docs MATCH 'erm'\").fetchone()[0] != 1 else None; \
|
||||
db.close()"
|
||||
|
||||
# ---------- s6-overlay install ----------
|
||||
# s6-overlay provides supervision for the main hermes process, the dashboard,
|
||||
# and per-profile gateways. /init becomes PID 1 below — see ENTRYPOINT.
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ else:
|
|||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from hermes_constants import get_hermes_home
|
||||
|
|
@ -251,11 +252,13 @@ def main(argv: list[str] | None = None) -> None:
|
|||
# MCP servers dynamically via asyncio.to_thread inside the event
|
||||
# loop; that path is unaffected.) Moved from model_tools.py module
|
||||
# scope to avoid freezing the gateway's loop on lazy import (#16856).
|
||||
try:
|
||||
from tools.mcp_tool import discover_mcp_tools
|
||||
discover_mcp_tools()
|
||||
except Exception:
|
||||
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
|
||||
# Metadata-only hosts can opt out of unrelated global MCP startup.
|
||||
if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1":
|
||||
try:
|
||||
from tools.mcp_tool import discover_mcp_tools
|
||||
discover_mcp_tools()
|
||||
except Exception:
|
||||
logger.debug("MCP tool discovery failed at ACP startup", exc_info=True)
|
||||
|
||||
agent = HermesACPAgent()
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,110 @@ from tools.approval import (
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]:
|
||||
"""Return ``(slug, label, [(model_id, description), ...])`` for named endpoints.
|
||||
|
||||
Covers both the v12 ``providers:`` mapping and the legacy
|
||||
``custom_providers:`` list. These endpoints never appear in canonical
|
||||
provider enumeration, so without this the ACP model selector hides every
|
||||
named endpoint that the TUI ``/model`` picker already renders (#47039
|
||||
implemented named-endpoint rows for the TUI surface only).
|
||||
|
||||
Model lists come from the entry's declared models (``default_model`` +
|
||||
``models``), refreshed from the endpoint's live ``/models`` listing when a
|
||||
credential is available and ``discover_models`` is not disabled. Declared
|
||||
models are kept even when live discovery fails — some OpenAI-compatible
|
||||
endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at
|
||||
all yet serve the declared models fine.
|
||||
|
||||
Slugs use the ``custom:<name>`` shape that ``parse_model_input`` and
|
||||
``resolve_runtime_provider`` already resolve, so encoded choice ids
|
||||
(``custom:<name>:<model>``) round-trip through ``set_session_model``
|
||||
unchanged.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import (
|
||||
get_compatible_custom_providers,
|
||||
is_provider_enabled,
|
||||
load_config,
|
||||
)
|
||||
from hermes_cli.models import fetch_api_models
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
try:
|
||||
cfg = load_config()
|
||||
entries = get_compatible_custom_providers(cfg)
|
||||
except Exception:
|
||||
logger.debug("Could not load named custom providers", exc_info=True)
|
||||
return []
|
||||
|
||||
# ``get_compatible_custom_providers`` drops the ``enabled`` flag during
|
||||
# normalization, so collect explicitly disabled provider keys from the
|
||||
# raw config and skip their entries below.
|
||||
disabled_keys: set[str] = set()
|
||||
raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None
|
||||
if isinstance(raw_providers, dict):
|
||||
for raw_key, raw_entry in raw_providers.items():
|
||||
if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry):
|
||||
disabled_keys.add(str(raw_key).strip().lower())
|
||||
|
||||
catalogs: list[tuple[str, str, list[tuple[str, str]]]] = []
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
provider_key = str(entry.get("provider_key", "") or "").strip()
|
||||
if provider_key.lower() in disabled_keys:
|
||||
continue
|
||||
name = str(entry.get("name", "") or "").strip()
|
||||
base_url = str(entry.get("base_url", "") or "").strip()
|
||||
if not name or not base_url:
|
||||
continue
|
||||
slug_source = provider_key or name
|
||||
slug = "custom:" + slug_source.strip().lower().replace(" ", "-")
|
||||
|
||||
api_key = str(entry.get("api_key", "") or "").strip()
|
||||
if not api_key:
|
||||
key_env = str(entry.get("key_env", "") or "").strip()
|
||||
api_key = os.environ.get(key_env, "").strip() if key_env else ""
|
||||
|
||||
declared: list[str] = []
|
||||
default_model = str(entry.get("model", "") or "").strip()
|
||||
if default_model:
|
||||
declared.append(default_model)
|
||||
models_cfg = entry.get("models")
|
||||
if isinstance(models_cfg, dict):
|
||||
for mid in models_cfg:
|
||||
mid = str(mid or "").strip()
|
||||
if mid and mid not in declared:
|
||||
declared.append(mid)
|
||||
|
||||
if not api_key and not declared:
|
||||
# No credential to discover with and nothing declared:
|
||||
# not addressable from the selector.
|
||||
continue
|
||||
|
||||
model_ids = list(declared)
|
||||
discover = entry.get("discover_models", True)
|
||||
if isinstance(discover, str):
|
||||
discover = discover.lower() not in {"false", "no", "0"}
|
||||
if discover and api_key:
|
||||
try:
|
||||
live = fetch_api_models(
|
||||
api_key, base_url, api_mode=entry.get("api_mode")
|
||||
)
|
||||
except Exception:
|
||||
live = None
|
||||
if live:
|
||||
model_ids = declared + [m for m in live if m not in declared]
|
||||
|
||||
if not model_ids:
|
||||
continue
|
||||
catalogs.append((slug, name, [(mid, "") for mid in model_ids]))
|
||||
|
||||
return catalogs
|
||||
|
||||
try:
|
||||
from hermes_cli import __version__ as HERMES_VERSION
|
||||
except Exception:
|
||||
|
|
@ -97,6 +201,13 @@ _executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="acp-agent")
|
|||
# does not expose a client-side limit, so this is a fixed cap that clients
|
||||
# paginate against using `cursor` / `next_cursor`.
|
||||
_LIST_SESSIONS_PAGE_SIZE = 50
|
||||
# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render
|
||||
# the whole `availableModels` array in one dropdown, so an unbounded
|
||||
# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker
|
||||
# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not
|
||||
# the total; aggregator providers stay intentionally uncapped inside the shared
|
||||
# inventory, and the current model is always kept via the fallback insert below.
|
||||
ACP_MAX_MODELS_PER_PROVIDER = 200
|
||||
_MAX_ACP_RESOURCE_BYTES = 512 * 1024
|
||||
_TEXT_RESOURCE_MIME_PREFIXES = ("text/",)
|
||||
_TEXT_RESOURCE_MIME_TYPES = {
|
||||
|
|
@ -585,46 +696,108 @@ class HermesACPAgent(acp.Agent):
|
|||
return f"{raw_provider}:{raw_model}"
|
||||
|
||||
def _build_model_state(self, state: SessionState) -> SessionModelState | None:
|
||||
"""Return the ACP model selector payload for editors like Zed."""
|
||||
"""Return authenticated providers and their models for ACP clients.
|
||||
|
||||
The shared Hermes inventory is also used by ``hermes model``, the TUI,
|
||||
and the dashboard. Keeping ACP on that substrate prevents its selector
|
||||
from silently collapsing to the current provider's curated list.
|
||||
"""
|
||||
model = str(state.model or getattr(state.agent, "model", "") or "").strip()
|
||||
provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter"
|
||||
|
||||
try:
|
||||
from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label
|
||||
from hermes_cli.inventory import build_models_payload, load_picker_context
|
||||
from hermes_cli.models import normalize_provider, provider_label
|
||||
|
||||
normalized_provider = normalize_provider(provider)
|
||||
provider_name = provider_label(normalized_provider)
|
||||
context = load_picker_context().with_overrides(
|
||||
current_provider=normalized_provider,
|
||||
current_model=model,
|
||||
current_base_url=str(getattr(state.agent, "base_url", "") or ""),
|
||||
)
|
||||
payload = build_models_payload(
|
||||
context,
|
||||
explicit_only=True,
|
||||
include_unconfigured=False,
|
||||
picker_hints=False,
|
||||
canonical_order=True,
|
||||
pricing=False,
|
||||
capabilities=False,
|
||||
refresh=False,
|
||||
probe_custom_providers=False,
|
||||
probe_current_custom_provider=False,
|
||||
max_models=ACP_MAX_MODELS_PER_PROVIDER,
|
||||
)
|
||||
|
||||
available_models: list[ModelInfo] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
for model_id, description in curated_models_for_provider(normalized_provider):
|
||||
rendered_model = str(model_id or "").strip()
|
||||
if not rendered_model:
|
||||
for row in payload.get("providers") or []:
|
||||
row_provider = normalize_provider(str(row.get("slug") or "").strip())
|
||||
if not row_provider:
|
||||
continue
|
||||
choice_id = self._encode_model_choice(normalized_provider, rendered_model)
|
||||
if choice_id in seen_ids:
|
||||
continue
|
||||
desc_parts = [f"Provider: {provider_name}"]
|
||||
if description:
|
||||
desc_parts.append(str(description).strip())
|
||||
if rendered_model == model:
|
||||
desc_parts.append("current")
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=choice_id,
|
||||
name=rendered_model,
|
||||
description=" • ".join(part for part in desc_parts if part),
|
||||
)
|
||||
provider_name = str(row.get("name") or "").strip() or provider_label(
|
||||
row_provider
|
||||
)
|
||||
seen_ids.add(choice_id)
|
||||
for model_entry in row.get("models") or []:
|
||||
if isinstance(model_entry, dict):
|
||||
rendered_model = str(
|
||||
model_entry.get("id")
|
||||
or model_entry.get("model")
|
||||
or model_entry.get("name")
|
||||
or ""
|
||||
).strip()
|
||||
else:
|
||||
rendered_model = str(model_entry or "").strip()
|
||||
if not rendered_model:
|
||||
continue
|
||||
choice_id = self._encode_model_choice(row_provider, rendered_model)
|
||||
if choice_id in seen_ids:
|
||||
continue
|
||||
is_current = (
|
||||
row_provider == normalized_provider and rendered_model == model
|
||||
)
|
||||
description = f"Provider: {provider_name}"
|
||||
if is_current:
|
||||
description += " • current"
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=choice_id,
|
||||
name=f"{provider_name} · {rendered_model}",
|
||||
description=description,
|
||||
)
|
||||
)
|
||||
seen_ids.add(choice_id)
|
||||
|
||||
# Named user-defined endpoints (providers: / custom_providers:)
|
||||
# are invisible to canonical provider enumeration — append them
|
||||
# so editor clients can select them like the TUI /model picker.
|
||||
for named_slug, named_label, named_catalog in _named_custom_provider_catalogs():
|
||||
for named_model, named_desc in named_catalog:
|
||||
named_choice = self._encode_model_choice(named_slug, named_model)
|
||||
if not named_choice or named_choice in seen_ids:
|
||||
continue
|
||||
named_parts = [f"Provider: {named_label}"]
|
||||
if named_desc:
|
||||
named_parts.append(str(named_desc).strip())
|
||||
if named_slug == normalized_provider and named_model == model:
|
||||
named_parts.append("current")
|
||||
available_models.append(
|
||||
ModelInfo(
|
||||
model_id=named_choice,
|
||||
name=named_model,
|
||||
description=" • ".join(part for part in named_parts if part),
|
||||
)
|
||||
)
|
||||
seen_ids.add(named_choice)
|
||||
|
||||
current_model_id = self._encode_model_choice(normalized_provider, model)
|
||||
if current_model_id and current_model_id not in seen_ids:
|
||||
provider_name = provider_label(normalized_provider)
|
||||
available_models.insert(
|
||||
0,
|
||||
ModelInfo(
|
||||
model_id=current_model_id,
|
||||
name=model,
|
||||
name=f"{provider_name} · {model}",
|
||||
description=f"Provider: {provider_name} • current",
|
||||
),
|
||||
)
|
||||
|
|
@ -1588,7 +1761,16 @@ class HermesACPAgent(acp.Agent):
|
|||
clear_session_vars,
|
||||
set_session_vars,
|
||||
)
|
||||
session_tokens = set_session_vars(session_key=session_id)
|
||||
# ``cwd`` pins the logical working directory for this context,
|
||||
# which is what the system prompt's "Current working directory"
|
||||
# line reports (agent/prompt_builder.py -> resolve_agent_cwd).
|
||||
# Without it the prompt advertises the global Hermes workspace
|
||||
# while the tools are rooted at the client's project, so the
|
||||
# model emits absolute paths under ~/.hermes/workspace and the
|
||||
# edit silently lands outside the editor's workspace.
|
||||
session_tokens = set_session_vars(
|
||||
session_key=session_id, cwd=state.cwd,
|
||||
)
|
||||
except Exception:
|
||||
session_tokens = None
|
||||
clear_session_vars = None # type: ignore[assignment]
|
||||
|
|
@ -1875,8 +2057,26 @@ class HermesACPAgent(acp.Agent):
|
|||
if handler is None:
|
||||
return None # not a known command — let the LLM handle it
|
||||
|
||||
try:
|
||||
# Slash handlers run on the event-loop thread, OUTSIDE the per-turn
|
||||
# contextvars.copy_context() that pins the session cwd for the agent
|
||||
# call. ``/compress`` and ``/model`` reach code that REBUILDS the
|
||||
# system prompt (agent._build_system_prompt -> resolve_agent_cwd), so
|
||||
# an unpinned handler bakes the Hermes install tree into the session's
|
||||
# cached prompt — persisted, and therefore poisoning every later turn
|
||||
# even though the turn itself is pinned. Pin inside a fresh context so
|
||||
# the write can't leak into other concurrent ACP sessions and needs no
|
||||
# teardown.
|
||||
def _dispatch() -> str | None:
|
||||
try:
|
||||
from agent.runtime_cwd import set_session_cwd
|
||||
|
||||
set_session_cwd(state.cwd)
|
||||
except Exception:
|
||||
logger.debug("Could not pin ACP session cwd for slash command", exc_info=True)
|
||||
return handler(args, state)
|
||||
|
||||
try:
|
||||
return contextvars.copy_context().run(_dispatch)
|
||||
except Exception as e:
|
||||
logger.error("Slash command /%s error: %s", cmd, e, exc_info=True)
|
||||
return f"Error executing /{cmd}: {e}"
|
||||
|
|
|
|||
|
|
@ -455,7 +455,7 @@ def init_agent(
|
|||
command: str = None,
|
||||
args: list[str] | None = None,
|
||||
model: str = "",
|
||||
max_iterations: int = 90, # Default tool-calling iterations (shared with subagents)
|
||||
max_iterations: int = 500, # Default tool-calling iterations (shared with subagents)
|
||||
tool_delay: float = 1.0,
|
||||
enabled_toolsets: List[str] = None,
|
||||
disabled_toolsets: List[str] = None,
|
||||
|
|
@ -529,7 +529,7 @@ def init_agent(
|
|||
requested_provider (str): Original provider identity before runtime canonicalization
|
||||
api_mode (str): API mode override: "chat_completions" or "codex_responses"
|
||||
model (str): Model name to use (default: "anthropic/claude-opus-4.6")
|
||||
max_iterations (int): Maximum number of tool calling iterations (default: 90)
|
||||
max_iterations (int): Maximum number of tool calling iterations (default: 500)
|
||||
tool_delay (float): Delay between tool calls in seconds (default: 1.0)
|
||||
enabled_toolsets (List[str]): Only enable tools from these toolsets (optional)
|
||||
disabled_toolsets (List[str]): Disable tools from these toolsets (optional)
|
||||
|
|
@ -645,6 +645,13 @@ def init_agent(
|
|||
# AWS Bedrock — auto-detect from provider name or base URL
|
||||
# (bedrock-runtime.<region>.amazonaws.com).
|
||||
agent.api_mode = "bedrock_converse"
|
||||
elif agent.provider in {"nous", "nous-portal", "nousresearch"}:
|
||||
# Portal is dual-wire: anthropic/* → Messages, everything else →
|
||||
# chat_completions. Callers that already pass api_mode win above;
|
||||
# this covers direct AIAgent construction without a resolved runtime.
|
||||
from hermes_cli.providers import nous_api_mode
|
||||
|
||||
agent.api_mode = nous_api_mode(agent.model)
|
||||
else:
|
||||
agent.api_mode = "chat_completions"
|
||||
|
||||
|
|
@ -823,9 +830,10 @@ def init_agent(
|
|||
# Anthropic prompt caching: auto-enabled for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party gateways that speak the
|
||||
# Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces
|
||||
# input costs by ~75% on multi-turn conversations. Uses system_and_3
|
||||
# strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy``
|
||||
# for the layout-vs-transport decision.
|
||||
# input costs by ~75% on multi-turn conversations. Uses four breakpoints:
|
||||
# the static system prefix, full system prompt, and last two messages
|
||||
# (falling back to system-and-3 when no static prefix is available). See
|
||||
# ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision.
|
||||
agent._use_prompt_caching, agent._use_native_cache_layout = (
|
||||
agent._anthropic_prompt_cache_policy()
|
||||
)
|
||||
|
|
@ -1341,6 +1349,13 @@ def init_agent(
|
|||
print("⚠️ Warning: API key appears invalid or missing")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to initialize OpenAI client: {e}")
|
||||
|
||||
# Keep a stable identity for the pool entry that supplied this runtime.
|
||||
# OAuth refreshes can replace the runtime token before a failed request is
|
||||
# recovered, so the mutable API-key value alone cannot reliably attribute
|
||||
# the failure to its source entry.
|
||||
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
|
||||
sync_credential_pool_entry_id(agent)
|
||||
|
||||
# Provider fallback chain — ordered list of backup providers tried
|
||||
# when the primary is exhausted (rate-limit, overload, connection
|
||||
|
|
@ -1487,6 +1502,9 @@ def init_agent(
|
|||
|
||||
# Cached system prompt -- built once per session, only rebuilt on compression
|
||||
agent._cached_system_prompt: Optional[str] = None
|
||||
# Cross-session-stable prefix of the cached prompt. It remains separate
|
||||
# from the persisted string and is used only to place an early cache marker.
|
||||
agent._cached_system_prompt_static: Optional[str] = None
|
||||
|
||||
# Filesystem checkpoint manager (transparent — not a tool)
|
||||
from tools.checkpoint_manager import CheckpointManager
|
||||
|
|
@ -1941,8 +1959,12 @@ def init_agent(
|
|||
# parent_session_id chain, no `name #N` renumber). See #38763 and
|
||||
# agent/conversation_compression.py. Consumed by compress_context(), not the
|
||||
# compressor, so it rides on the agent.
|
||||
# Default True must match DEFAULT_CONFIG["compression"]["in_place"]
|
||||
# (#38763). default=False here previously flipped agents into rotation
|
||||
# mode whenever the merged config omitted the key (partial configs,
|
||||
# load_config failure → {}), re-arming the pre-lease drift abort.
|
||||
compression_in_place = is_truthy_value(
|
||||
_compression_cfg.get("in_place"), default=False
|
||||
_compression_cfg.get("in_place"), default=True
|
||||
)
|
||||
codex_app_server_auto_compaction = str(
|
||||
_compression_cfg.get("codex_app_server_auto", "native") or "native"
|
||||
|
|
|
|||
|
|
@ -850,6 +850,25 @@ def strip_think_blocks(agent, content: str) -> str:
|
|||
|
||||
|
||||
|
||||
def sync_credential_pool_entry_id(agent) -> None:
|
||||
"""Rebind ``agent._credential_pool_entry_id`` from the current pool + key.
|
||||
|
||||
OAuth refreshes can replace the runtime token before a failed request is
|
||||
recovered, so the mutable API-key value alone cannot reliably attribute
|
||||
the failure to its source entry. This resolves the stable pool-entry ID
|
||||
for the agent's current ``api_key`` and clears it when no pool is bound.
|
||||
"""
|
||||
pool = getattr(agent, "_credential_pool", None)
|
||||
try:
|
||||
agent._credential_pool_entry_id = (
|
||||
pool.entry_id_for_api_key(getattr(agent, "api_key", None))
|
||||
if pool is not None
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
agent._credential_pool_entry_id = None
|
||||
|
||||
|
||||
def recover_with_credential_pool(
|
||||
agent,
|
||||
*,
|
||||
|
|
@ -934,10 +953,30 @@ def recover_with_credential_pool(
|
|||
# failing entry exactly; fall back to current()'s key only when the agent
|
||||
# carries no key at all.
|
||||
_api_key_hint = getattr(agent, "api_key", None) or None
|
||||
_raw_credential_id = getattr(agent, "_credential_pool_entry_id", None)
|
||||
_credential_id = (
|
||||
_raw_credential_id
|
||||
if isinstance(_raw_credential_id, str) and _raw_credential_id
|
||||
else None
|
||||
)
|
||||
if not _api_key_hint:
|
||||
_cur = pool.current()
|
||||
if _cur:
|
||||
_api_key_hint = getattr(_cur, "runtime_api_key", None)
|
||||
if not _credential_id:
|
||||
_current_id = getattr(_cur, "id", None)
|
||||
if isinstance(_current_id, str) and _current_id:
|
||||
_credential_id = _current_id
|
||||
|
||||
def _rotate_failed_credential(rotate_status: int):
|
||||
kwargs = {
|
||||
"status_code": rotate_status,
|
||||
"error_context": error_context,
|
||||
"api_key_hint": _api_key_hint,
|
||||
}
|
||||
if _credential_id:
|
||||
kwargs["credential_id"] = _credential_id
|
||||
return pool.mark_exhausted_and_rotate(**kwargs)
|
||||
|
||||
effective_reason = classified_reason
|
||||
if effective_reason is None:
|
||||
|
|
@ -972,11 +1011,7 @@ def recover_with_credential_pool(
|
|||
# Runtime credentials can be resolved by a separate pool instance,
|
||||
# leaving this recovery pool without ``current_id``. Match the key
|
||||
# that actually failed instead of quarantining a different account.
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (billing) — rotated to pool entry %s",
|
||||
|
|
@ -995,8 +1030,13 @@ def recover_with_credential_pool(
|
|||
# Prefer the entry matching the failing key over the shared current()
|
||||
# pointer, for the same attribution reason as above.
|
||||
current_entry = None
|
||||
if _api_key_hint:
|
||||
if _credential_id:
|
||||
current_entry = next(
|
||||
(e for e in pool.entries() if e.id == _credential_id),
|
||||
None,
|
||||
)
|
||||
if _api_key_hint:
|
||||
current_entry = current_entry or next(
|
||||
(e for e in pool.entries() if e.runtime_api_key == _api_key_hint),
|
||||
None,
|
||||
)
|
||||
|
|
@ -1009,11 +1049,7 @@ def recover_with_credential_pool(
|
|||
current_last_status,
|
||||
)
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s",
|
||||
|
|
@ -1037,11 +1073,7 @@ def recover_with_credential_pool(
|
|||
if not has_retried_429 and not usage_limit_reached:
|
||||
return False, True
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit) — rotated to pool entry %s",
|
||||
|
|
@ -1113,7 +1145,10 @@ def recover_with_credential_pool(
|
|||
# the shared pointer can reference a different, healthy entry, and
|
||||
# refreshing it would consume that entry's single-use refresh token
|
||||
# (or mark it exhausted on failure) for a failure it never had.
|
||||
refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint)
|
||||
refresh_kwargs = {"api_key_hint": _api_key_hint}
|
||||
if _credential_id:
|
||||
refresh_kwargs["credential_id"] = _credential_id
|
||||
refreshed = pool.try_refresh_matching(**refresh_kwargs)
|
||||
if refreshed is not None:
|
||||
# ``try_refresh_matching()`` re-mints a fresh OAuth token and reports
|
||||
# success even when the upstream keeps rejecting it — a single-entry
|
||||
|
|
@ -1145,11 +1180,7 @@ def recover_with_credential_pool(
|
|||
# Refresh failed — rotate to next credential instead of giving up.
|
||||
# The failed entry is already marked exhausted by the refresh attempt.
|
||||
rotate_status = status_code if status_code is not None else 401
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
next_entry = _rotate_failed_credential(rotate_status)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (auth refresh failed) — rotated to pool entry %s",
|
||||
|
|
@ -1190,15 +1221,29 @@ def try_recover_primary_transport(
|
|||
if agent._is_openrouter_url():
|
||||
return False
|
||||
provider_lower = (agent.provider or "").strip().lower()
|
||||
if provider_lower in {"nous", "nous-research"}:
|
||||
# Portal OpenAI-wire traffic still rides aggregator retry infra, so one
|
||||
# more rebuilt OpenAI client won't help. Portal Claude on the native
|
||||
# Messages route holds a local Anthropic SDK client whose connection
|
||||
# pool *does* need the rebuild every other anthropic_messages provider
|
||||
# already gets — don't blanket-skip the dual-wire path.
|
||||
if (
|
||||
provider_lower in {"nous", "nous-portal", "nousresearch"}
|
||||
and getattr(agent, "api_mode", None) != "anthropic_messages"
|
||||
):
|
||||
return False
|
||||
|
||||
try:
|
||||
# Close existing client to release stale connections
|
||||
# Retire the existing client to release stale connections. #70773:
|
||||
# never hard-close the shared client here — this runs on the
|
||||
# conversation-loop thread while workers from stale-killed streaming
|
||||
# attempts may still be unwinding their SSL BIOs on the old pool.
|
||||
# ``_retire_shared_openai_client`` shuts the sockets down (FD-safe
|
||||
# from any thread) and defers the FD release to GC, which cannot
|
||||
# complete until every borrowing thread has unwound.
|
||||
if getattr(agent, "client", None) is not None:
|
||||
try:
|
||||
agent._close_openai_client(
|
||||
agent.client, reason="primary_recovery", shared=True,
|
||||
agent._retire_shared_openai_client(
|
||||
agent.client, reason="primary_recovery",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -1460,6 +1505,7 @@ def restore_primary_runtime(agent) -> bool:
|
|||
pool_matches_primary = False
|
||||
if pool is not None and pool_provider and not pool_matches_primary:
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
|
||||
|
|
@ -1479,6 +1525,7 @@ def restore_primary_runtime(agent) -> bool:
|
|||
# the pool for its current best entry and swap the live credential in.
|
||||
# When the pool is absent, empty, or the entry has no usable key, we
|
||||
# keep the snapshot key (the existing behavior). Fixes #25205.
|
||||
agent._credential_pool_entry_id = None
|
||||
pool = getattr(agent, "_credential_pool", None)
|
||||
if pool is not None and pool.has_available():
|
||||
entry = pool.select()
|
||||
|
|
@ -1856,7 +1903,15 @@ def anthropic_prompt_cache_policy(
|
|||
|
||||
if is_native_anthropic:
|
||||
return True, True
|
||||
if (is_openrouter or is_nous_portal) and (is_claude or is_kimi):
|
||||
# Envelope layout is an OpenAI-wire construct. Portal Claude on the native
|
||||
# Messages route must fall through to the third-party anthropic_messages
|
||||
# branch below, which emits inner-block cache_control breakpoints; the
|
||||
# envelope form would be dropped and serve 0% cache hits.
|
||||
if (
|
||||
(is_openrouter or is_nous_portal)
|
||||
and (is_claude or is_kimi)
|
||||
and not is_anthropic_wire
|
||||
):
|
||||
return True, False
|
||||
# Nous Portal Qwen (e.g. qwen3.6-plus) takes the same envelope-layout
|
||||
# cache_control path as Portal Claude. Portal proxies to OpenRouter
|
||||
|
|
@ -2020,8 +2075,11 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
from hermes_cli.providers import determine_api_mode
|
||||
|
||||
# ── Determine api_mode if not provided ──
|
||||
# Pass model so dual-wire providers (Nous Portal anthropic/* → Messages)
|
||||
# resolve correctly; without it determine_api_mode falls back to the
|
||||
# openai_chat overlay default.
|
||||
if not api_mode:
|
||||
api_mode = determine_api_mode(new_provider, base_url)
|
||||
api_mode = determine_api_mode(new_provider, base_url, model=new_model)
|
||||
|
||||
# Defense-in-depth: ensure OpenCode base_url doesn't carry a trailing
|
||||
# /v1 into the anthropic_messages client, which would cause the SDK to
|
||||
|
|
@ -2075,6 +2133,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
# restore the original pool (issue #52727: pool reload is part of this
|
||||
# switch and must be reversible on rollback).
|
||||
_snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING)
|
||||
_snapshot["_credential_pool_entry_id"] = getattr(
|
||||
agent, "_credential_pool_entry_id", _MISSING
|
||||
)
|
||||
|
||||
try:
|
||||
# Clear the per-config context_length override so the new model's
|
||||
|
|
@ -2131,6 +2192,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
# A pool bound to the old provider is worse than no pool: the
|
||||
# recovery guard rejects it and every later 401/429 skips rotation.
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
agent._credential_pool = load_pool(new_provider)
|
||||
|
|
@ -2140,7 +2202,6 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
"continuing without pool rotation this turn",
|
||||
new_provider, _pool_exc,
|
||||
)
|
||||
|
||||
# ── Build new client ──
|
||||
if (new_provider or "").strip().lower() == "moa":
|
||||
from agent.moa_loop import build_moa_facade
|
||||
|
|
@ -2236,6 +2297,8 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
reason="switch_model",
|
||||
shared=True,
|
||||
)
|
||||
|
||||
sync_credential_pool_entry_id(agent)
|
||||
except Exception:
|
||||
# Rollback every mutated field to the pre-swap snapshot so the agent
|
||||
# is left consistent (old model + old provider + old client) and the
|
||||
|
|
@ -2570,6 +2633,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i
|
|||
_clarify_tool(
|
||||
question=next_args.get("question", ""),
|
||||
choices=next_args.get("choices"),
|
||||
multi_select=next_args.get("multi_select", False),
|
||||
callback=agent.clarify_callback,
|
||||
),
|
||||
next_args,
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from urllib.parse import urlparse
|
|||
|
||||
from hermes_constants import get_hermes_home
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from utils import base_url_host_matches, normalize_proxy_env_vars
|
||||
from utils import base_url_host_matches, base_url_hostname, normalize_proxy_env_vars
|
||||
|
||||
# NOTE: `import anthropic` is deliberately NOT at module top — the SDK pulls
|
||||
# ~220 ms of imports (anthropic.types, anthropic.lib.tools._beta_runner, etc.)
|
||||
|
|
@ -368,7 +368,7 @@ def _detect_claude_code_version() -> str:
|
|||
try:
|
||||
result = _sp.run(
|
||||
[cmd, "--version"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
# Output is like "2.1.74 (Claude Code)" or just "2.1.74"
|
||||
|
|
@ -546,15 +546,49 @@ def _is_deepseek_anthropic_endpoint(base_url: str | None) -> bool:
|
|||
return "/anthropic" in normalized.rstrip("/").lower()
|
||||
|
||||
|
||||
def _is_nous_portal_endpoint(base_url: str | None) -> bool:
|
||||
"""Return True for Nous Portal's Anthropic Messages route.
|
||||
|
||||
Portal serves its ``anthropic/*`` catalog natively at
|
||||
``https://inference-api.nousresearch.com/v1/messages``. Portal-specific
|
||||
behaviours key off this: Bearer JWT auth, verbatim catalog model ids,
|
||||
and native thinking-signature replay.
|
||||
|
||||
Trusted hosts only:
|
||||
|
||||
1. Prod hostname ``inference-api.nousresearch.com``
|
||||
2. The operator-set ``NOUS_INFERENCE_BASE_URL`` hostname (staging/preview)
|
||||
|
||||
Lookalikes such as ``inference-api.nousresearch.com.attacker.test`` are
|
||||
rejected (hostname match, not substring).
|
||||
"""
|
||||
if base_url_host_matches(base_url or "", "inference-api.nousresearch.com"):
|
||||
return True
|
||||
try:
|
||||
from hermes_cli.auth import _nous_inference_env_override
|
||||
|
||||
override = _nous_inference_env_override()
|
||||
except Exception:
|
||||
return False
|
||||
if not override:
|
||||
return False
|
||||
# Exact host equality (not subdomain) so the env override can't broaden
|
||||
# into sibling hosts the operator did not set.
|
||||
override_host = base_url_hostname(override)
|
||||
return bool(override_host) and base_url_hostname(base_url or "") == override_host
|
||||
|
||||
|
||||
def _requires_bearer_auth(base_url: str | None) -> bool:
|
||||
"""Return True for Anthropic-compatible providers that require Bearer auth.
|
||||
|
||||
Some third-party /anthropic endpoints implement Anthropic's Messages API but
|
||||
require Authorization: Bearer instead of Anthropic's native x-api-key header.
|
||||
MiniMax's global and China Anthropic-compatible endpoints, Azure AI
|
||||
Foundry's Anthropic-style endpoint, and Palantir Foundry's LLM proxy
|
||||
follow this pattern.
|
||||
Foundry's Anthropic-style endpoint, Palantir Foundry's LLM proxy, and Nous
|
||||
Portal's Messages route follow this pattern.
|
||||
"""
|
||||
if _is_nous_portal_endpoint(base_url):
|
||||
return True
|
||||
normalized = _normalize_base_url_text(base_url)
|
||||
if not normalized:
|
||||
return False
|
||||
|
|
@ -721,7 +755,11 @@ def _build_anthropic_client_with_bearer_hook(
|
|||
if common_betas:
|
||||
kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)}
|
||||
|
||||
return _anthropic_sdk.Anthropic(**kwargs)
|
||||
client = _anthropic_sdk.Anthropic(**kwargs)
|
||||
# Same env-inference trap as build_anthropic_client: auth_token-only
|
||||
# construction would otherwise also send ANTHROPIC_API_KEY as X-Api-Key.
|
||||
client.api_key = None
|
||||
return client
|
||||
|
||||
|
||||
def build_anthropic_client(
|
||||
|
|
@ -850,7 +888,16 @@ def build_anthropic_client(
|
|||
if common_betas:
|
||||
kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)}
|
||||
|
||||
return _anthropic_sdk.Anthropic(**kwargs)
|
||||
client = _anthropic_sdk.Anthropic(**kwargs)
|
||||
# Bearer-only construction leaves ``api_key`` unset, so the SDK fills it
|
||||
# from ``ANTHROPIC_API_KEY`` (Hermes loads that into the process env from
|
||||
# ``~/.hermes/.env``). The result is dual auth —
|
||||
# ``X-Api-Key: sk-ant-…`` *and* ``Authorization: Bearer <portal-jwt>`` —
|
||||
# on every Portal / MiniMax / OAuth Messages request. Clear the env-filled
|
||||
# key whenever we intentionally authenticated via auth_token alone.
|
||||
if "auth_token" in kwargs and "api_key" not in kwargs:
|
||||
client.api_key = None
|
||||
return client
|
||||
|
||||
|
||||
def build_anthropic_bedrock_client(region: str):
|
||||
|
|
@ -914,7 +961,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]:
|
|||
"-s", "Claude Code-credentials",
|
||||
"-w"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=5,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
@ -1920,10 +1967,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|||
return None
|
||||
btype = b.get("type")
|
||||
if btype == "text":
|
||||
# Coerce empty/whitespace-only text to a non-whitespace placeholder;
|
||||
# the Messages input schema rejects blank text blocks (#69512), and a
|
||||
# blank block stored in history replays on every turn → permanent 400.
|
||||
out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))}
|
||||
text_val = b.get("text", "")
|
||||
# Bedrock and strict Anthropic-compatible endpoints reject text
|
||||
# blocks where "text" is empty or whitespace-only (#69512). Drop the
|
||||
# blank block (the caller relocates any cache_control it carried and
|
||||
# falls back to a non-whitespace placeholder when nothing survives)
|
||||
# rather than coercing in place — a coerced "(empty)" block would be
|
||||
# model-visible noise next to surviving thinking/tool_use blocks.
|
||||
# Type-safe: captured blocks can carry text=None from an invalid
|
||||
# upstream payload, which a bare .strip() would crash on.
|
||||
if not isinstance(text_val, str) or not text_val.strip():
|
||||
return None
|
||||
out: Dict[str, Any] = {"type": "text", "text": text_val}
|
||||
# citations is input-valid ONLY when it's a non-empty list; the SDK
|
||||
# emits citations=None on responses, which the input schema rejects.
|
||||
cits = b.get("citations")
|
||||
|
|
@ -2011,9 +2066,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
parsed_args = {}
|
||||
redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args
|
||||
replayed: List[Dict[str, Any]] = []
|
||||
_relocated_replay_cache_control = None
|
||||
_dropped_blank_text = False
|
||||
for b in ordered_blocks:
|
||||
clean = _sanitize_replay_block(b)
|
||||
if clean is None:
|
||||
if isinstance(b, dict) and b.get("type") == "text":
|
||||
_dropped_blank_text = True
|
||||
if isinstance(b, dict) and isinstance(b.get("cache_control"), dict):
|
||||
# A dropped blank text block can still carry the cache
|
||||
# breakpoint marker -- relocate it rather than losing it.
|
||||
_relocated_replay_cache_control = b["cache_control"]
|
||||
continue
|
||||
if clean.get("type") == "tool_use":
|
||||
# Override raw (un-redacted) input with the redacted copy when
|
||||
|
|
@ -2023,20 +2086,90 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
if redacted is not None:
|
||||
clean["input"] = redacted
|
||||
replayed.append(clean)
|
||||
# When every text block was blank and nothing cacheable survived
|
||||
# (e.g. signed thinking + a blank text block, or a SOLE blank
|
||||
# cache-marked block), emit the non-whitespace placeholder so the
|
||||
# replayed message stays schema-valid (#69512) and a relocated cache
|
||||
# marker still has a carrier instead of being silently lost.
|
||||
_has_cacheable_replay = any(
|
||||
isinstance(b, dict) and b.get("type") in {"text", "tool_use"}
|
||||
for b in replayed
|
||||
)
|
||||
if not _has_cacheable_replay and (
|
||||
_dropped_blank_text or _relocated_replay_cache_control is not None
|
||||
):
|
||||
replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER})
|
||||
if replayed:
|
||||
if _relocated_replay_cache_control is not None:
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
replayed, _relocated_replay_cache_control
|
||||
)
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
replayed, m.get("cache_control")
|
||||
)
|
||||
# apply_anthropic_cache_control marks an assistant turn with
|
||||
# non-empty text by writing cache_control INTO ``content`` (see
|
||||
# _apply_cache_marker's list branch), not at the top level. This
|
||||
# branch rebuilds the message from ordered_blocks and never reads
|
||||
# ``content``, so that marker would be dropped -- and because
|
||||
# _can_carry_marker already counted this message as a carrier, the
|
||||
# breakpoint is burned rather than relocated. #56195 covered the
|
||||
# complementary shape (blank content -> top-level marker); this is
|
||||
# the interleaved thinking + preamble-text + tool_use shape.
|
||||
_inline_cc = None
|
||||
_msg_content = m.get("content")
|
||||
if isinstance(_msg_content, list):
|
||||
for _blk in _msg_content:
|
||||
if isinstance(_blk, dict) and isinstance(
|
||||
_blk.get("cache_control"), dict
|
||||
):
|
||||
_inline_cc = _blk["cache_control"]
|
||||
break
|
||||
if _inline_cc is not None:
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
replayed, _inline_cc
|
||||
)
|
||||
return {"role": "assistant", "content": replayed}
|
||||
|
||||
blocks = _extract_preserved_thinking_blocks(m)
|
||||
# Cache markers dropped along with a blank block are relocated onto the
|
||||
# last surviving cacheable block below (via
|
||||
# _apply_assistant_cache_control_to_last_cacheable_block), rather than
|
||||
# lost -- prompt_caching.py's _apply_cache_marker() sets cache_control
|
||||
# directly on content[-1] for list content, so if that last part happens
|
||||
# to be blank text, dropping it silently would lose the breakpoint.
|
||||
_relocated_cache_control = None
|
||||
if content:
|
||||
if isinstance(content, list):
|
||||
converted_content = _convert_content_to_anthropic(content)
|
||||
if isinstance(converted_content, list):
|
||||
blocks.extend(converted_content)
|
||||
# Bedrock and strict Anthropic-compatible endpoints reject
|
||||
# text blocks where "text" is empty or whitespace-only. The
|
||||
# ordered-replay path enforces the same invariant via
|
||||
# _sanitize_replay_block(). Type-safe against ANY invalid
|
||||
# "text" value from an upstream payload -- None, or a
|
||||
# truthy non-string like an int -- not just None: checking
|
||||
# isinstance() first (rather than `blk.get("text") or ""`)
|
||||
# means a non-string value is treated as blank/invalid
|
||||
# instead of reaching .strip() and raising AttributeError.
|
||||
for blk in converted_content:
|
||||
_blk_text = blk.get("text") if isinstance(blk, dict) else None
|
||||
if (
|
||||
isinstance(blk, dict)
|
||||
and blk.get("type") == "text"
|
||||
and (not isinstance(_blk_text, str) or not _blk_text.strip())
|
||||
):
|
||||
if isinstance(blk.get("cache_control"), dict):
|
||||
_relocated_cache_control = blk["cache_control"]
|
||||
continue
|
||||
blocks.append(blk)
|
||||
else:
|
||||
blocks.append({"type": "text", "text": str(content)})
|
||||
# Scalar (non-list) content: a whitespace-only string is the
|
||||
# same invalid-payload case as an empty list block -- drop it
|
||||
# rather than emitting a blank text block.
|
||||
text_str = str(content)
|
||||
if text_str.strip():
|
||||
blocks.append({"type": "text", "text": text_str})
|
||||
for tc in m.get("tool_calls", []):
|
||||
if not tc or not isinstance(tc, dict):
|
||||
continue
|
||||
|
|
@ -2052,9 +2185,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
"name": fn.get("name", ""),
|
||||
"input": parsed_args,
|
||||
})
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
blocks, m.get("cache_control")
|
||||
)
|
||||
# Kimi's /coding endpoint (Anthropic protocol) requires assistant
|
||||
# tool-call messages to carry reasoning_content when thinking is
|
||||
# enabled server-side. Preserve it as a thinking block so Kimi
|
||||
|
|
@ -2080,19 +2210,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
)
|
||||
if isinstance(reasoning_content, str) and not _already_has_thinking:
|
||||
blocks.insert(0, {"type": "thinking", "thinking": reasoning_content})
|
||||
# Anthropic rejects empty assistant content
|
||||
effective = blocks or content
|
||||
if not effective or effective == "":
|
||||
effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
elif isinstance(effective, list):
|
||||
# The all-empty guard above misses a list that still contains a
|
||||
# whitespace-only text block (e.g. from a content array of blank parts,
|
||||
# or compression). Those also trip "text content blocks must contain
|
||||
# non-whitespace text" (#69512). Coerce text blocks in place; other
|
||||
# block types (thinking/tool_use/image) are left untouched.
|
||||
for blk in effective:
|
||||
if isinstance(blk, dict) and blk.get("type") == "text":
|
||||
blk["text"] = _safe_text(blk.get("text", ""))
|
||||
# Anthropic rejects empty assistant content. IMPORTANT: fall back only
|
||||
# to the placeholder, never to the raw `content` variable -- `content`
|
||||
# is the UNFILTERED original message content, and can itself be exactly
|
||||
# the blank/whitespace-only payload the filtering above just removed
|
||||
# (a sole blank text block, or scalar whitespace with no tool_calls).
|
||||
# `blocks or content` there would silently restore the invalid provider
|
||||
# payload this function exists to prevent (#69512).
|
||||
effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
# Applied here (after the empty-fallback resolution) rather than
|
||||
# earlier against `blocks` directly, so a cache_control relocated from
|
||||
# a dropped blank block that was the ONLY block still lands on the
|
||||
# (empty) placeholder instead of being silently lost when blocks was
|
||||
# empty at the point the marker would otherwise have been applied.
|
||||
if _relocated_cache_control is not None:
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
effective, _relocated_cache_control
|
||||
)
|
||||
_apply_assistant_cache_control_to_last_cacheable_block(
|
||||
effective, m.get("cache_control")
|
||||
)
|
||||
return {"role": "assistant", "content": effective}
|
||||
|
||||
|
||||
|
|
@ -2326,10 +2463,22 @@ def _manage_thinking_signatures(
|
|||
replayed assistant tool-call messages. See hermes-agent#13848 (Kimi) and
|
||||
hermes-agent#16748 (DeepSeek).
|
||||
|
||||
Nous Portal's ``/v1/messages`` route is the exception among third-party
|
||||
hosts: it proxies Claude to Anthropic/Vertex/Bedrock and validates the
|
||||
same signed thinking blocks. Sticky ``session_id`` keeps a conversation
|
||||
on one upstream instance so those signatures stay warm — stripping them
|
||||
here would 400 the first tool-loop turn ("thinking must be passed back").
|
||||
Portal therefore takes the native Anthropic replay path below.
|
||||
|
||||
Mutates ``result`` in place.
|
||||
"""
|
||||
_THINKING_TYPES = frozenset(("thinking", "redacted_thinking"))
|
||||
_is_third_party = _is_third_party_anthropic_endpoint(base_url)
|
||||
# Portal speaks Anthropic's thinking contract end-to-end; do not treat it
|
||||
# as a signature-blind proxy even though the host is not anthropic.com.
|
||||
_is_third_party = (
|
||||
_is_third_party_anthropic_endpoint(base_url)
|
||||
and not _is_nous_portal_endpoint(base_url)
|
||||
)
|
||||
|
||||
last_assistant_idx = None
|
||||
for i in range(len(result) - 1, -1, -1):
|
||||
|
|
@ -2586,7 +2735,12 @@ def build_anthropic_kwargs(
|
|||
)
|
||||
anthropic_tools = convert_tools_to_anthropic(tools) if tools else []
|
||||
|
||||
model = normalize_model_name(model, preserve_dots=preserve_dots)
|
||||
# Nous Portal routes on its own catalog ids (``anthropic/claude-opus-4.8``);
|
||||
# normalizing to the bare Anthropic slug would make the model unresolvable
|
||||
# there. Skipping the call preserves the prefix AND the dots, so
|
||||
# ``preserve_dots`` stays irrelevant for Portal.
|
||||
if not _is_nous_portal_endpoint(base_url):
|
||||
model = normalize_model_name(model, preserve_dots=preserve_dots)
|
||||
# effective_max_tokens = output cap for this call (≠ total context window)
|
||||
# Use the resolver helper so non-positive values (negative ints,
|
||||
# fractional floats, NaN, non-numeric) fail locally with a clear error
|
||||
|
|
@ -2825,6 +2979,8 @@ def create_anthropic_message(
|
|||
*,
|
||||
log_prefix: str = "",
|
||||
prefer_stream: bool = True,
|
||||
on_stream_event=None,
|
||||
on_response=None,
|
||||
) -> Any:
|
||||
"""Create an Anthropic message, aggregating via stream when available.
|
||||
|
||||
|
|
@ -2834,6 +2990,20 @@ def create_anthropic_message(
|
|||
crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to
|
||||
match the main turn path, falling back to ``create()`` only for providers
|
||||
that explicitly do not support streaming, such as restricted Bedrock roles.
|
||||
|
||||
``on_stream_event``: optional callable invoked once per streamed event
|
||||
(best-effort, exceptions swallowed). Lets callers report forward progress
|
||||
to liveness watchdogs — e.g. the auxiliary compression path ticking its
|
||||
progress hook so a slow-but-generating summary model isn't treated as
|
||||
hung. Only fires on the streaming path; the ``create()`` fallback has no
|
||||
events to report.
|
||||
|
||||
``on_response``: optional callable invoked once with the underlying httpx
|
||||
response before the message is aggregated (best-effort, exceptions
|
||||
swallowed). Response *headers* carry out-of-band provider state that the
|
||||
parsed ``Message`` drops — Nous Portal's ``x-nous-credits-*`` balance family
|
||||
in particular. Only fires on the streaming path, which is the one the main
|
||||
turn loop takes.
|
||||
"""
|
||||
sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix)
|
||||
|
||||
|
|
@ -2844,6 +3014,26 @@ def create_anthropic_message(
|
|||
stream_kwargs.pop("stream", None)
|
||||
try:
|
||||
with stream_fn(**stream_kwargs) as stream:
|
||||
if callable(on_response):
|
||||
try:
|
||||
on_response(getattr(stream, "response", None))
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"%son_response callback failed",
|
||||
log_prefix, exc_info=True,
|
||||
)
|
||||
if callable(on_stream_event):
|
||||
# Consume the event stream manually so each event can
|
||||
# tick the caller's progress callback; get_final_message
|
||||
# then returns the accumulated snapshot.
|
||||
for _event in stream:
|
||||
try:
|
||||
on_stream_event(_event)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"%son_stream_event callback failed",
|
||||
log_prefix, exc_info=True,
|
||||
)
|
||||
return stream.get_final_message()
|
||||
except Exception as exc:
|
||||
if not _is_stream_unavailable_error(exc):
|
||||
|
|
|
|||
|
|
@ -247,6 +247,50 @@ def aux_interrupt_protection(active: bool = True):
|
|||
_aux_interrupt_protection.active = prev
|
||||
|
||||
|
||||
# ── Forward-progress hook for streamed auxiliary calls ───────────────────
|
||||
# Long auxiliary calls (context compression is the prime case) are watched by
|
||||
# wall-clock deadlines in their hosts (gateway session hygiene). A fixed
|
||||
# deadline punishes SLOW summary models exactly as hard as HUNG ones: a
|
||||
# reasoning model happily streaming a large summary is killed mid-generation.
|
||||
# This thread-local hook lets the host observe liveness instead: the wire
|
||||
# consumers below tick it on every streamed token/SSE event, and the host
|
||||
# extends its deadline while tokens are moving (see gateway/run.py session
|
||||
# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the
|
||||
# call topology — the aux call and its stream consumption run synchronously
|
||||
# on the thread that installed the hook.
|
||||
_aux_progress = threading.local()
|
||||
|
||||
|
||||
def _notify_aux_progress() -> None:
|
||||
"""Tick the installed forward-progress hook, if any. Never raises."""
|
||||
hook = getattr(_aux_progress, "hook", None)
|
||||
if hook is None:
|
||||
return
|
||||
try:
|
||||
hook()
|
||||
except Exception:
|
||||
logger.debug("aux progress hook failed", exc_info=True)
|
||||
|
||||
|
||||
def _aux_progress_active() -> bool:
|
||||
return getattr(_aux_progress, "hook", None) is not None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def aux_progress_hook(hook):
|
||||
"""Install *hook* as the current thread's aux forward-progress callback.
|
||||
|
||||
``hook=None`` is a no-op passthrough so callers can wire it
|
||||
unconditionally. Re-entrant-safe: restores the previous hook on exit.
|
||||
"""
|
||||
prev = getattr(_aux_progress, "hook", None)
|
||||
_aux_progress.hook = hook if callable(hook) else prev
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_aux_progress.hook = prev
|
||||
|
||||
|
||||
def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
|
||||
"""Return False instead of raising when a patched symbol is not a type."""
|
||||
try:
|
||||
|
|
@ -1058,16 +1102,29 @@ class _CodexCompletionsAdapter:
|
|||
# key in extra_body (not top-level) and GitHub/Copilot Responses opts
|
||||
# out of cache-key routing entirely — for those hosts, skip it here.
|
||||
try:
|
||||
from agent.transports.codex import _content_cache_key
|
||||
from agent.transports.codex import (
|
||||
_content_cache_key,
|
||||
_default_prompt_cache_retention_for_request,
|
||||
)
|
||||
from utils import base_url_host_matches
|
||||
|
||||
_host_src = str(getattr(self._client, "base_url", "") or "")
|
||||
_is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai")
|
||||
_is_github = base_url_host_matches(_host_src, "githubcopilot.com")
|
||||
_is_github = (
|
||||
base_url_host_matches(_host_src, "githubcopilot.com")
|
||||
or base_url_host_matches(_host_src, "models.github.ai")
|
||||
)
|
||||
if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs:
|
||||
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"))
|
||||
if _cache_key:
|
||||
resp_kwargs["prompt_cache_key"] = _cache_key
|
||||
if "prompt_cache_retention" not in resp_kwargs:
|
||||
_cache_retention = _default_prompt_cache_retention_for_request(
|
||||
model,
|
||||
_host_src,
|
||||
)
|
||||
if _cache_retention:
|
||||
resp_kwargs["prompt_cache_retention"] = _cache_retention
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True
|
||||
|
|
@ -1149,6 +1206,10 @@ class _CodexCompletionsAdapter:
|
|||
def _on_each_event(_event: Any) -> None:
|
||||
# Re-check timeout/cancellation per event, matching the
|
||||
# cadence the old in-line ``_check_cancelled()`` used.
|
||||
# Each SSE event is also forward progress for hosts watching
|
||||
# a progress hook (gateway session hygiene): a reasoning
|
||||
# model streaming a long summary must not look hung.
|
||||
_notify_aux_progress()
|
||||
_check_cancelled()
|
||||
|
||||
event_stream = self._client.responses.create(**stream_kwargs)
|
||||
|
|
@ -1301,10 +1362,32 @@ class AsyncCodexAuxiliaryClient:
|
|||
class _AnthropicCompletionsAdapter:
|
||||
"""OpenAI-client-compatible adapter for Anthropic Messages API."""
|
||||
|
||||
def __init__(self, real_client: Any, model: str, is_oauth: bool = False):
|
||||
def __init__(
|
||||
self,
|
||||
real_client: Any,
|
||||
model: str,
|
||||
is_oauth: bool = False,
|
||||
base_url: str | None = None,
|
||||
):
|
||||
self._client = real_client
|
||||
self._model = model
|
||||
self._is_oauth = is_oauth
|
||||
# Prefer the caller-supplied URL (AnthropicAuxiliaryClient keeps the
|
||||
# pre-strip Portal ``.../v1`` form). Only fall back to the SDK
|
||||
# client's host for Nous Portal — a blanket fallback would flip
|
||||
# MiniMax/Zhipu/etc. aux adapters from "unknown host = native
|
||||
# Anthropic" to third-party (stripping thinking signatures).
|
||||
self._base_url = base_url or None
|
||||
if not self._base_url:
|
||||
candidate = str(getattr(real_client, "base_url", "") or "") or None
|
||||
if candidate:
|
||||
try:
|
||||
from agent.anthropic_adapter import _is_nous_portal_endpoint
|
||||
|
||||
if _is_nous_portal_endpoint(candidate):
|
||||
self._base_url = candidate
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def create(self, **kwargs) -> Any:
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message
|
||||
|
|
@ -1356,6 +1439,11 @@ class _AnthropicCompletionsAdapter:
|
|||
reasoning_config=_reasoning_cfg,
|
||||
tool_choice=normalized_tool_choice,
|
||||
is_oauth=self._is_oauth,
|
||||
# Portal routes on ``anthropic/<slug>`` catalog ids and replays
|
||||
# signed thinking like native Anthropic; both carve-outs key off
|
||||
# base_url. Omitting it normalizes the id to a bare Anthropic
|
||||
# slug and the Portal Messages route cannot resolve it.
|
||||
base_url=self._base_url,
|
||||
)
|
||||
# Opus 4.7+ rejects any non-default temperature/top_p/top_k; only set
|
||||
# temperature for models that still accept it. build_anthropic_kwargs
|
||||
|
|
@ -1390,7 +1478,18 @@ class _AnthropicCompletionsAdapter:
|
|||
existing = {}
|
||||
anthropic_kwargs["extra_body"] = {**existing, **passthrough}
|
||||
|
||||
response = create_anthropic_message(self._client, anthropic_kwargs)
|
||||
response = create_anthropic_message(
|
||||
self._client,
|
||||
anthropic_kwargs,
|
||||
# Tick the aux forward-progress hook per streamed event so hosts
|
||||
# watching liveness (gateway session hygiene) don't kill a
|
||||
# slow-but-generating summary model. No-op when no hook is
|
||||
# installed (None keeps the fast get_final_message path).
|
||||
on_stream_event=(
|
||||
(lambda _event: _notify_aux_progress())
|
||||
if _aux_progress_active() else None
|
||||
),
|
||||
)
|
||||
_transport = get_transport("anthropic_messages")
|
||||
_nr = _transport.normalize_response(
|
||||
response, strip_tool_prefix=self._is_oauth
|
||||
|
|
@ -1438,7 +1537,9 @@ class AnthropicAuxiliaryClient:
|
|||
|
||||
def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False):
|
||||
self._real_client = real_client
|
||||
adapter = _AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth)
|
||||
adapter = _AnthropicCompletionsAdapter(
|
||||
real_client, model, is_oauth=is_oauth, base_url=base_url,
|
||||
)
|
||||
self.chat = _AnthropicChatShim(adapter)
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
|
|
@ -1703,7 +1804,7 @@ def _read_nous_auth() -> Optional[dict]:
|
|||
try:
|
||||
if not _AUTH_JSON_PATH.is_file():
|
||||
return None
|
||||
data = json.loads(_AUTH_JSON_PATH.read_text())
|
||||
data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8"))
|
||||
if data.get("active_provider") != "nous":
|
||||
return None
|
||||
provider = data.get("providers", {}).get("nous", {})
|
||||
|
|
@ -4116,6 +4217,7 @@ def _try_main_agent_model_fallback(
|
|||
failed_provider: str,
|
||||
task: str = None,
|
||||
reason: str = "error",
|
||||
failed_model: Optional[str] = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Last-resort fallback to the user's main agent provider + model.
|
||||
|
||||
|
|
@ -4124,8 +4226,23 @@ def _try_main_agent_model_fallback(
|
|||
layer: if nothing the user asked for can serve the request, try the
|
||||
main chat model before giving up.
|
||||
|
||||
Skips when the failed provider already IS the main provider (no point
|
||||
retrying the same backend that just failed).
|
||||
``failed_model`` narrows the same-provider skip to the exact
|
||||
(provider, model) pair that just failed, mirroring
|
||||
:func:`_try_configured_fallback_chain`. This matters for self-hosted /
|
||||
custom endpoints serving several models behind one provider label: the
|
||||
aux compression model timing out says nothing about the health of the
|
||||
main agent model deployed on the same URL (real incident: aux
|
||||
``glm-5.2`` hung and timed out while main ``macaron-v1-venti`` on the
|
||||
identical endpoint was serving 448K-token turns fine — the
|
||||
provider-label skip discarded the one fallback that would have worked).
|
||||
|
||||
- Model-specific runtime failures (timeout, connection, rate limit,
|
||||
model-incompatible, invalid response) pass ``failed_model``: skip the
|
||||
main model only when it IS the exact model that failed.
|
||||
- Provider-wide failures (auth 401, payment 402) and legacy callers
|
||||
leave ``failed_model`` as None, keeping the whole-provider skip —
|
||||
the shared credentials/account are broken, so the main model on the
|
||||
same provider cannot help either.
|
||||
|
||||
Returns:
|
||||
(client, model, provider_label) or (None, None, "") if no fallback.
|
||||
|
|
@ -4142,9 +4259,23 @@ def _try_main_agent_model_fallback(
|
|||
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
|
||||
return None, None, ""
|
||||
|
||||
skip = (failed_provider or "").lower().strip()
|
||||
if main_provider.lower() == skip:
|
||||
# The thing that failed IS the main model — nothing to fall back to.
|
||||
# Identity + scope semantics owned by agent.backend_identity (#72468):
|
||||
# model-scoped failures skip only the exact deployment that failed;
|
||||
# provider-wide failures (no failed_model) skip the credential surface.
|
||||
from agent.backend_identity import (
|
||||
BackendIdentity,
|
||||
FailureScope,
|
||||
should_skip_candidate,
|
||||
)
|
||||
|
||||
skip_model = (failed_model or "").strip().lower() or None
|
||||
if should_skip_candidate(
|
||||
BackendIdentity.build(provider=main_provider, model=main_model),
|
||||
BackendIdentity.build(provider=failed_provider, model=skip_model),
|
||||
FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL,
|
||||
):
|
||||
# The thing that failed IS the main model (or the failure was
|
||||
# provider-wide) — nothing to fall back to.
|
||||
return None, None, ""
|
||||
if _is_provider_unhealthy(main_provider):
|
||||
_log_skip_unhealthy(main_provider, task)
|
||||
|
|
@ -4254,6 +4385,7 @@ def _try_configured_fallback_chain(
|
|||
task: str,
|
||||
failed_provider: str,
|
||||
reason: str = "error",
|
||||
failed_model: Optional[str] = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Try user-configured fallback_chain for a specific auxiliary task.
|
||||
|
||||
|
|
@ -4261,6 +4393,25 @@ def _try_configured_fallback_chain(
|
|||
entry in order. Each entry must have at least ``provider``; ``model``,
|
||||
``base_url``, and ``api_key`` are optional.
|
||||
|
||||
``failed_model`` narrows the skip check to the exact (provider, model)
|
||||
pair that just failed, rather than the whole provider. Without it every
|
||||
entry sharing the failed provider is skipped (the original behaviour).
|
||||
Callers pass it only when a sibling model on the same provider could
|
||||
plausibly recover:
|
||||
|
||||
- Model-specific runtime failures (timeout, connection, rate limit,
|
||||
model-incompatible, invalid response) pass ``failed_model`` so a
|
||||
chain that intentionally lists several models under the same provider
|
||||
— e.g. two more NVIDIA NIM models after the primary NIM model times
|
||||
out — is not skipped wholesale. Only the exact model that failed is
|
||||
skipped; the siblings still run instead of jumping straight to the
|
||||
main-agent-model safety net.
|
||||
- Provider-wide failures (auth 401, payment 402) and "no client could
|
||||
be built" callers leave ``failed_model`` as None, keeping the whole
|
||||
provider skipped — the shared credentials/account behind every model
|
||||
on that provider are broken, so a sibling can't help and the
|
||||
main-agent-model safety net should be reached instead.
|
||||
|
||||
Returns:
|
||||
(client, model, provider_label) or (None, None, "") if no fallback.
|
||||
"""
|
||||
|
|
@ -4272,7 +4423,24 @@ def _try_configured_fallback_chain(
|
|||
if not chain or not isinstance(chain, list):
|
||||
return None, None, ""
|
||||
|
||||
skip = failed_provider.lower().strip()
|
||||
skip_model = (failed_model or "").strip().lower() or None
|
||||
# Identity + scope semantics owned by agent.backend_identity (#59561,
|
||||
# #72468): a failed_model means the failure was model-scoped (timeout /
|
||||
# connection / rate limit) — only the exact deployment is skipped; no
|
||||
# failed_model means provider-wide (auth/payment) — the whole credential
|
||||
# surface is skipped.
|
||||
from agent.backend_identity import (
|
||||
BackendIdentity,
|
||||
FailureScope,
|
||||
should_skip_candidate,
|
||||
)
|
||||
|
||||
failed_ident = BackendIdentity.build(
|
||||
provider=failed_provider, model=skip_model,
|
||||
)
|
||||
failure_scope = (
|
||||
FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL
|
||||
)
|
||||
tried = []
|
||||
min_ctx = _task_minimum_context_length(task)
|
||||
|
||||
|
|
@ -4280,9 +4448,20 @@ def _try_configured_fallback_chain(
|
|||
if not isinstance(entry, dict):
|
||||
continue
|
||||
fb_provider = str(entry.get("provider", "")).strip()
|
||||
if not fb_provider or fb_provider.lower() == skip:
|
||||
if not fb_provider:
|
||||
continue
|
||||
fb_model = str(entry.get("model", "")).strip() or None
|
||||
fb_model_raw = str(entry.get("model", "")).strip()
|
||||
if should_skip_candidate(
|
||||
BackendIdentity.build(
|
||||
provider=fb_provider,
|
||||
model=fb_model_raw,
|
||||
base_url=str(entry.get("base_url") or ""),
|
||||
),
|
||||
failed_ident,
|
||||
failure_scope,
|
||||
):
|
||||
continue
|
||||
fb_model = fb_model_raw or None
|
||||
|
||||
label = f"fallback_chain[{i}]({fb_provider})"
|
||||
|
||||
|
|
@ -4949,10 +5128,11 @@ def resolve_provider_client(
|
|||
|
||||
# ── Nous Portal (OAuth) ──────────────────────────────────────────
|
||||
if provider == "nous":
|
||||
# Detect vision tasks: either explicit model override from
|
||||
# _PROVIDER_VISION_MODELS, or caller passed a known vision model.
|
||||
# Detect vision tasks: caller flag (strict vision backend), explicit
|
||||
# model override from _PROVIDER_VISION_MODELS, or a known vision id.
|
||||
_is_vision = (
|
||||
model in _PROVIDER_VISION_MODELS.values()
|
||||
is_vision
|
||||
or model in _PROVIDER_VISION_MODELS.values()
|
||||
or (model or "").strip().lower() == "mimo-v2-omni"
|
||||
)
|
||||
client, default = _try_nous(vision=_is_vision)
|
||||
|
|
@ -4961,6 +5141,17 @@ def resolve_provider_client(
|
|||
"but Nous Portal not configured (run: hermes auth)")
|
||||
return None, None
|
||||
final_model = _normalize_resolved_model(model or default, provider)
|
||||
# Dual-wire: anthropic/* → /v1/messages, everything else stays on
|
||||
# /chat/completions. Derive from the catalog id (not a stale
|
||||
# api_mode=chat_completions) so aux matches the main agent.
|
||||
from hermes_cli.providers import nous_api_mode
|
||||
|
||||
portal_mode = nous_api_mode(final_model)
|
||||
api_key_str = str(getattr(client, "api_key", "") or "")
|
||||
base_url_str = str(getattr(client, "base_url", "") or "")
|
||||
client = _maybe_wrap_anthropic(
|
||||
client, final_model, api_key_str, base_url_str, portal_mode,
|
||||
)
|
||||
return (_to_async_client(client, final_model, is_vision=is_vision) if async_mode
|
||||
else (client, final_model))
|
||||
|
||||
|
|
@ -5617,7 +5808,10 @@ def _resolve_strict_vision_backend(
|
|||
if provider == "openrouter":
|
||||
return _try_openrouter(model=model)
|
||||
if provider == "nous":
|
||||
return _try_nous(vision=True)
|
||||
# Must go through resolve_provider_client so anthropic/* vision
|
||||
# recommendations wrap onto /v1/messages — _try_nous alone returns
|
||||
# a bare OpenAI client and the call 404s.
|
||||
return resolve_provider_client("nous", model, is_vision=True)
|
||||
if provider == "openai-codex":
|
||||
# Route through resolve_provider_client so the caller's explicit
|
||||
# model is used. There is no safe default Codex model (shifting
|
||||
|
|
@ -6878,8 +7072,14 @@ def _build_call_kwargs(
|
|||
_is_gemini_native = is_native_gemini_base_url(_effective_base)
|
||||
except Exception:
|
||||
pass
|
||||
_nous_on_messages = False
|
||||
if _provider_norm in {"nous", "nous-portal", "nousresearch"}:
|
||||
from hermes_cli.providers import nous_api_mode
|
||||
|
||||
_nous_on_messages = nous_api_mode(model) == "anthropic_messages"
|
||||
if (
|
||||
_is_anthropic_compat_endpoint(provider, _effective_base)
|
||||
or _nous_on_messages
|
||||
or _is_nvidia_nim
|
||||
or _is_moa
|
||||
or _is_gemini_native
|
||||
|
|
@ -6974,21 +7174,43 @@ def _build_call_kwargs(
|
|||
else:
|
||||
effort = reasoning_config.get("effort") or "medium"
|
||||
merged_extra["reasoning"] = {"enabled": True, "effort": effort}
|
||||
if provider == "nous" and "tags" not in merged_extra:
|
||||
merged_extra["tags"] = _nous_portal_tags()
|
||||
# Portal product tags + sticky session_id. The provider profile usually
|
||||
# supplies both; this fallback covers profile-load failures and alias
|
||||
# spellings the profile lookup might miss. session_id keeps aux
|
||||
# compression/title/vision calls on the same upstream instance as the
|
||||
# main turn (cache warmth) — tags alone are not enough on /v1/messages.
|
||||
_provider_for_portal = str(provider or "").strip().lower()
|
||||
if _provider_for_portal in {"nous", "nous-portal", "nousresearch"}:
|
||||
if "tags" not in merged_extra:
|
||||
merged_extra["tags"] = _nous_portal_tags()
|
||||
if "session_id" not in merged_extra:
|
||||
try:
|
||||
from agent.portal_tags import get_conversation_context
|
||||
|
||||
sticky_key = get_conversation_context()
|
||||
except Exception:
|
||||
sticky_key = None
|
||||
if sticky_key:
|
||||
merged_extra["session_id"] = sticky_key
|
||||
if merged_extra:
|
||||
kwargs["extra_body"] = merged_extra
|
||||
|
||||
# Native Anthropic Messages adapters do not consume ``extra_body``. Carry
|
||||
# the normalized Hermes reasoning config through a private kwarg so the
|
||||
# adapter can pass it into build_anthropic_kwargs(), where provider-aware
|
||||
# thinking/output_config projection lives. Do not expose this private kwarg
|
||||
# to ordinary OpenAI-compatible SDK clients, which would reject it.
|
||||
# Anthropic Messages adapters translate Hermes reasoning into native
|
||||
# ``thinking`` via a private kwarg (and strip OpenAI-shaped
|
||||
# ``extra_body.reasoning``). Do not expose this private kwarg to ordinary
|
||||
# OpenAI-compatible SDK clients, which would reject it. Portal Claude is
|
||||
# dual-wire — include it when the catalog id selects /v1/messages.
|
||||
if reasoning_config and isinstance(reasoning_config, dict):
|
||||
provider_norm = str(provider or "").strip().lower()
|
||||
effective_base = base_url or ""
|
||||
_nous_on_messages = False
|
||||
if provider_norm in {"nous", "nous-portal", "nousresearch"}:
|
||||
from hermes_cli.providers import nous_api_mode
|
||||
|
||||
_nous_on_messages = nous_api_mode(model) == "anthropic_messages"
|
||||
if (
|
||||
provider_norm == "anthropic"
|
||||
or _nous_on_messages
|
||||
or _endpoint_speaks_anthropic_messages(effective_base)
|
||||
or _is_anthropic_compat_endpoint(provider_norm, effective_base)
|
||||
):
|
||||
|
|
@ -7104,6 +7326,346 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any:
|
|||
return value
|
||||
|
||||
|
||||
# ── Streamed aggregation for progress-hooked auxiliary calls ─────────────
|
||||
# When a forward-progress hook is installed (aux_progress_hook — today only
|
||||
# by context compression), the primary chat.completions attempt is upgraded
|
||||
# to a streamed request that is aggregated back into a complete response.
|
||||
# Two effects, both deliberate:
|
||||
# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead
|
||||
# of a total budget (httpx applies the read timeout per stream read), so
|
||||
# a slow-but-generating summary model is never killed mid-generation
|
||||
# while tokens are moving — only a genuinely silent connection dies.
|
||||
# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs
|
||||
# (gateway session hygiene) extend their deadlines on liveness instead
|
||||
# of guessing with a fixed wall clock.
|
||||
# A total ceiling still bounds the pathological 1-token-per-idle-window
|
||||
# stream; see _aux_stream_total_ceiling().
|
||||
|
||||
_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
|
||||
_AUX_STREAM_CEILING_MULTIPLIER = 4.0
|
||||
|
||||
|
||||
def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
|
||||
"""Absolute wall-clock bound for a progress-hooked streamed aux call.
|
||||
|
||||
Generous by design — the idle timeout is the real guard; this only stops
|
||||
a degenerate stream that trickles one token per idle window forever.
|
||||
"""
|
||||
try:
|
||||
timeout = float(effective_timeout) if effective_timeout is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
timeout = 0.0
|
||||
return max(_AUX_STREAM_CEILING_FLOOR_SECONDS,
|
||||
_AUX_STREAM_CEILING_MULTIPLIER * timeout)
|
||||
|
||||
|
||||
def _client_streams_internally(client: Any) -> bool:
|
||||
"""Wire adapters that consume a stream inside .create() already tick the
|
||||
progress hook themselves (Codex per SSE event, Anthropic per stream
|
||||
event); Bedrock's Converse shim cannot stream at all. None of them
|
||||
accept chat-completions ``stream=True`` semantics from us."""
|
||||
return isinstance(client, (
|
||||
CodexAuxiliaryClient,
|
||||
AnthropicAuxiliaryClient,
|
||||
BedrockAuxiliaryClient,
|
||||
))
|
||||
|
||||
|
||||
def _is_streaming_rejected_error(exc: Exception) -> bool:
|
||||
"""Provider explicitly refused a streamed chat.completions request."""
|
||||
err = str(exc).lower()
|
||||
if "stream_options" in err:
|
||||
return True
|
||||
return "stream" in err and (
|
||||
"not supported" in err
|
||||
or "unsupported" in err
|
||||
or "not allowed" in err
|
||||
or "disabled" in err
|
||||
)
|
||||
|
||||
|
||||
def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
|
||||
"""Detect providers that only accept streaming (non-stream = HTTP 400).
|
||||
|
||||
Some OpenAI-compatible endpoints reject non-streaming chat requests
|
||||
outright — e.g. Tencent Copilot returns
|
||||
``{"code": 11101, "msg": "Non-stream chat request is currently not
|
||||
supported"}``. The main conversation loop already streams, so interactive
|
||||
chat works; auxiliary tasks (title generation, compression, web extract)
|
||||
used the non-streaming path and failed on every call. When this returns
|
||||
True the auxiliary client sends ``stream=True`` and aggregates the chunks
|
||||
itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686).
|
||||
|
||||
Beyond the known-host list, users can mark ANY custom endpoint as
|
||||
stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml
|
||||
(list of substrings matched against the endpoint URL).
|
||||
"""
|
||||
_url = str(base_url or "").lower()
|
||||
if not _url:
|
||||
return False
|
||||
# Tencent Copilot — "Non-stream chat request is currently not supported"
|
||||
if base_url_host_matches(_url, "copilot.tencent.com"):
|
||||
return True
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
aux_cfg = (load_config() or {}).get("auxiliary", {})
|
||||
markers = aux_cfg.get("stream_only_base_urls") or []
|
||||
if isinstance(markers, (list, tuple)):
|
||||
for marker in markers:
|
||||
if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url:
|
||||
return True
|
||||
except Exception:
|
||||
# Config read is best-effort; never break an aux call over it.
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _create_with_progress(
|
||||
client: Any,
|
||||
kwargs: Dict[str, Any],
|
||||
task: Optional[str] = None,
|
||||
*,
|
||||
force_stream: bool = False,
|
||||
) -> Any:
|
||||
"""chat.completions.create() that streams when a progress hook is active
|
||||
or the provider only accepts streamed requests.
|
||||
|
||||
Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when
|
||||
neither trigger applies (every existing caller/task) or when the client's
|
||||
wire adapter streams internally. With a hook + a chunk-capable client,
|
||||
the request is sent with ``stream=True`` and aggregated, ticking the hook
|
||||
per chunk — so the configured ``timeout`` acts per stream read (idle)
|
||||
rather than as a total budget, and outer liveness watchdogs see tokens
|
||||
moving. ``force_stream=True`` (stream-only providers such as Tencent
|
||||
Copilot — credit @kudi88, PR #60686) takes the same streamed path even
|
||||
without a hook. Providers that reject the streamed request fall back to
|
||||
the plain non-streaming call — except under ``force_stream``, where a
|
||||
stream-only provider rejects the plain call by definition, so the
|
||||
original error is surfaced to the normal recovery chains instead.
|
||||
"""
|
||||
_notify_aux_progress() # request dispatched counts as progress
|
||||
if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
|
||||
return client.chat.completions.create(**kwargs)
|
||||
|
||||
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
|
||||
stream_kwargs = dict(kwargs)
|
||||
stream_kwargs["stream"] = True
|
||||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||||
try:
|
||||
chunks = client.chat.completions.create(**stream_kwargs)
|
||||
except Exception as exc:
|
||||
# Genuine provider failures (auth, credit, rate limit, network) are
|
||||
# not streaming's fault — surface them unchanged so the existing
|
||||
# recovery chains (credential refresh, pool rotation, provider
|
||||
# fallback) see the same error they would on a plain call.
|
||||
if (
|
||||
force_stream
|
||||
or _is_transient_transport_error(exc)
|
||||
or _is_auth_error(exc)
|
||||
or _is_payment_error(exc)
|
||||
or _is_rate_limit_error(exc)
|
||||
):
|
||||
raise
|
||||
# Anything else may be a streaming-specific rejection (explicit
|
||||
# "stream not supported", stream_options 400, or an idiosyncratic
|
||||
# 4xx). Retry non-streaming once; if the request itself is bad the
|
||||
# plain call reproduces the real error for the normal except-chains.
|
||||
logger.debug(
|
||||
"Auxiliary %s: streamed request failed (%s); retrying "
|
||||
"non-streaming", task or "call", exc,
|
||||
)
|
||||
return client.chat.completions.create(**kwargs)
|
||||
|
||||
# Some shims (MoA virtual provider under quiet mode, defensive adapters)
|
||||
# return a complete response even when stream=True was requested.
|
||||
if hasattr(chunks, "choices"):
|
||||
_notify_aux_progress()
|
||||
return chunks
|
||||
return _aggregate_chat_stream(
|
||||
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
|
||||
)
|
||||
|
||||
|
||||
def _aggregate_chat_stream(
|
||||
chunks: Any,
|
||||
*,
|
||||
model: str = "",
|
||||
total_ceiling: Optional[float] = None,
|
||||
) -> Any:
|
||||
"""Consume a chat.completions chunk stream into a complete response.
|
||||
|
||||
Ticks the thread-local aux progress hook on every chunk. Raises
|
||||
TimeoutError when *total_ceiling* seconds elapse before the stream
|
||||
finishes — phrased with "timed out" so existing timeout classification
|
||||
(``_is_timeout_error``) treats it exactly like a request timeout.
|
||||
Accumulation is shared with the async mirror via
|
||||
:class:`_ChatStreamAccumulator`.
|
||||
"""
|
||||
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
|
||||
try:
|
||||
for chunk in chunks:
|
||||
acc.feed(chunk)
|
||||
finally:
|
||||
close_fn = getattr(chunks, "close", None)
|
||||
if callable(close_fn):
|
||||
try:
|
||||
close_fn()
|
||||
except Exception:
|
||||
pass
|
||||
return acc.finish()
|
||||
|
||||
|
||||
class _ChatStreamAccumulator:
|
||||
"""Shared per-chunk accumulation for sync and async stream aggregation.
|
||||
|
||||
Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async
|
||||
consumer below cannot drift from the sync one (same content/reasoning/
|
||||
tool-call delta reassembly, same "timed out" ceiling phrasing).
|
||||
"""
|
||||
|
||||
def __init__(self, model: str = "", total_ceiling: Optional[float] = None):
|
||||
self._started = time.monotonic()
|
||||
self._total_ceiling = total_ceiling
|
||||
self.content_parts: List[str] = []
|
||||
self.reasoning_parts: List[str] = []
|
||||
self.tool_calls_acc: Dict[int, Dict[str, Any]] = {}
|
||||
self.finish_reason = None
|
||||
self.usage = None
|
||||
self.resp_id = ""
|
||||
self.resp_model = model or ""
|
||||
|
||||
def feed(self, chunk: Any) -> None:
|
||||
_notify_aux_progress()
|
||||
if (
|
||||
self._total_ceiling is not None
|
||||
and (time.monotonic() - self._started) >= self._total_ceiling
|
||||
):
|
||||
raise TimeoutError(
|
||||
f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
|
||||
"total ceiling (stream still open but over budget)"
|
||||
)
|
||||
self.resp_id = getattr(chunk, "id", None) or self.resp_id
|
||||
self.resp_model = getattr(chunk, "model", None) or self.resp_model
|
||||
chunk_usage = getattr(chunk, "usage", None)
|
||||
if chunk_usage:
|
||||
self.usage = chunk_usage
|
||||
choices = getattr(chunk, "choices", None) or []
|
||||
if not choices:
|
||||
return
|
||||
choice = choices[0]
|
||||
self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
|
||||
delta = getattr(choice, "delta", None)
|
||||
if delta is None:
|
||||
return
|
||||
piece = getattr(delta, "content", None)
|
||||
if piece:
|
||||
self.content_parts.append(piece)
|
||||
reasoning_piece = (
|
||||
getattr(delta, "reasoning", None)
|
||||
or getattr(delta, "reasoning_content", None)
|
||||
)
|
||||
if reasoning_piece and isinstance(reasoning_piece, str):
|
||||
self.reasoning_parts.append(reasoning_piece)
|
||||
for tc in (getattr(delta, "tool_calls", None) or []):
|
||||
idx = getattr(tc, "index", 0) or 0
|
||||
acc = self.tool_calls_acc.setdefault(
|
||||
idx, {"id": "", "name": "", "arguments": []}
|
||||
)
|
||||
if getattr(tc, "id", None):
|
||||
acc["id"] = tc.id
|
||||
fn = getattr(tc, "function", None)
|
||||
if fn is not None:
|
||||
if getattr(fn, "name", None):
|
||||
acc["name"] = fn.name
|
||||
if getattr(fn, "arguments", None):
|
||||
acc["arguments"].append(fn.arguments)
|
||||
|
||||
def finish(self) -> Any:
|
||||
tool_calls = None
|
||||
if self.tool_calls_acc:
|
||||
tool_calls = [
|
||||
SimpleNamespace(
|
||||
id=acc["id"],
|
||||
type="function",
|
||||
function=SimpleNamespace(
|
||||
name=acc["name"],
|
||||
arguments="".join(acc["arguments"]),
|
||||
),
|
||||
)
|
||||
for _idx, acc in sorted(self.tool_calls_acc.items())
|
||||
]
|
||||
message = SimpleNamespace(
|
||||
role="assistant",
|
||||
content="".join(self.content_parts),
|
||||
tool_calls=tool_calls,
|
||||
reasoning="".join(self.reasoning_parts) or None,
|
||||
)
|
||||
choice = SimpleNamespace(
|
||||
index=0,
|
||||
message=message,
|
||||
finish_reason=self.finish_reason or "stop",
|
||||
)
|
||||
return SimpleNamespace(
|
||||
id=self.resp_id,
|
||||
model=self.resp_model,
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=self.usage,
|
||||
)
|
||||
|
||||
|
||||
async def _aggregate_chat_stream_async(
|
||||
chunks: Any,
|
||||
*,
|
||||
model: str = "",
|
||||
total_ceiling: Optional[float] = None,
|
||||
) -> Any:
|
||||
"""Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer).
|
||||
|
||||
The AsyncOpenAI stream contract is an async iterator — consuming it with
|
||||
the sync helper raises. Same accumulation and ceiling semantics via
|
||||
:class:`_ChatStreamAccumulator`.
|
||||
"""
|
||||
acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling)
|
||||
try:
|
||||
async for chunk in chunks:
|
||||
acc.feed(chunk)
|
||||
finally:
|
||||
close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None)
|
||||
if callable(close_fn):
|
||||
try:
|
||||
result = close_fn()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
except Exception:
|
||||
pass
|
||||
return acc.finish()
|
||||
|
||||
|
||||
async def _acreate_with_stream(
|
||||
client: Any,
|
||||
kwargs: Dict[str, Any],
|
||||
task: Optional[str] = None,
|
||||
) -> Any:
|
||||
"""Async chat.completions.create() for stream-only providers.
|
||||
|
||||
Sends ``stream=True`` and aggregates the async chunk stream into a
|
||||
complete response (credit @kudi88, PR #60686 — async contract fixed to
|
||||
``async for`` and tool-call deltas preserved per sweeper review).
|
||||
"""
|
||||
total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout"))
|
||||
stream_kwargs = dict(kwargs)
|
||||
stream_kwargs["stream"] = True
|
||||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||||
chunks = await client.chat.completions.create(**stream_kwargs)
|
||||
# Defensive: shims may hand back a complete response despite stream=True.
|
||||
if hasattr(chunks, "choices"):
|
||||
return chunks
|
||||
return await _aggregate_chat_stream_async(
|
||||
chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling,
|
||||
)
|
||||
|
||||
|
||||
def call_llm(
|
||||
task: str = None,
|
||||
*,
|
||||
|
|
@ -7304,7 +7866,13 @@ def call_llm(
|
|||
# for the transient retry every auxiliary task shares. (PR #16587)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
client.chat.completions.create(**kwargs), task,
|
||||
_create_with_progress(
|
||||
client, kwargs, task,
|
||||
force_stream=_provider_requires_stream(
|
||||
resolved_provider, _base_info or resolved_base_url,
|
||||
),
|
||||
),
|
||||
task,
|
||||
provider=resolved_provider, base_url=_base_info)
|
||||
except Exception as transient_err:
|
||||
if not _is_transient_transport_error(transient_err):
|
||||
|
|
@ -7337,7 +7905,13 @@ def call_llm(
|
|||
time.sleep(_backoff)
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
client.chat.completions.create(**kwargs), task)
|
||||
_create_with_progress(
|
||||
client, kwargs, task,
|
||||
force_stream=_provider_requires_stream(
|
||||
resolved_provider, _base_info or resolved_base_url,
|
||||
),
|
||||
),
|
||||
task)
|
||||
except Exception as retry_transient:
|
||||
if not _is_transient_transport_error(retry_transient):
|
||||
raise
|
||||
|
|
@ -7654,6 +8228,15 @@ def call_llm(
|
|||
logger.info("Auxiliary %s: %s on %s (%s), trying fallback",
|
||||
task or "call", reason, resolved_provider, first_err)
|
||||
|
||||
# Narrow the configured-chain skip to the exact model that
|
||||
# failed ONLY for model-specific failures. Auth (401) and
|
||||
# payment (402) errors are provider-wide — the credentials or
|
||||
# account behind every model on that provider are the same — so
|
||||
# a sibling model can't recover; keep skipping the whole
|
||||
# provider so the main-agent-model safety net is still reached.
|
||||
_chain_failed_model = (
|
||||
None if reason in ("auth error", "payment error") else final_model
|
||||
)
|
||||
# Fallback order (#26882, #26803):
|
||||
# 1. User-configured fallback_chain (per-task) if set
|
||||
# 2. For auto: top-level main fallback_providers/fallback_model
|
||||
|
|
@ -7662,7 +8245,8 @@ def call_llm(
|
|||
fb_client, fb_model, fb_label = (None, None, "")
|
||||
if is_auto:
|
||||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
task, resolved_provider or "auto", reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
if fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_main_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
|
|
@ -7671,10 +8255,12 @@ def call_llm(
|
|||
resolved_provider, task, reason=reason)
|
||||
else:
|
||||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
task, resolved_provider or "auto", reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
if fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
|
||||
resolved_provider, task, reason=reason)
|
||||
resolved_provider, task, reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
|
||||
if fb_client is not None:
|
||||
fb_resp = _call_fallback_candidate_sync(
|
||||
|
|
@ -7890,9 +8476,25 @@ async def async_call_llm(
|
|||
# Retry ONCE on the same provider for a transient transport blip
|
||||
# before the except-chain escalates to fallback — see call_llm()
|
||||
# for the rationale. (PR #16587)
|
||||
_force_stream_async = (
|
||||
_provider_requires_stream(
|
||||
resolved_provider, _client_base or resolved_base_url,
|
||||
)
|
||||
and not isinstance(client, (
|
||||
AsyncCodexAuxiliaryClient,
|
||||
AsyncAnthropicAuxiliaryClient,
|
||||
AsyncBedrockAuxiliaryClient,
|
||||
))
|
||||
)
|
||||
|
||||
async def _acreate(_kwargs: Dict[str, Any]) -> Any:
|
||||
if _force_stream_async:
|
||||
return await _acreate_with_stream(client, _kwargs, task)
|
||||
return await client.chat.completions.create(**_kwargs)
|
||||
|
||||
try:
|
||||
return _validate_llm_response(
|
||||
await client.chat.completions.create(**kwargs), task,
|
||||
await _acreate(kwargs), task,
|
||||
provider=resolved_provider, base_url=_client_base)
|
||||
except Exception as transient_err:
|
||||
if not _is_transient_transport_error(transient_err):
|
||||
|
|
@ -7913,7 +8515,7 @@ async def async_call_llm(
|
|||
task or "call", transient_err,
|
||||
)
|
||||
return _validate_llm_response(
|
||||
await client.chat.completions.create(**kwargs), task)
|
||||
await _acreate(kwargs), task)
|
||||
except Exception as first_err:
|
||||
if "temperature" in kwargs and _is_unsupported_temperature_error(first_err):
|
||||
retry_kwargs = dict(kwargs)
|
||||
|
|
@ -8172,6 +8774,15 @@ async def async_call_llm(
|
|||
logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback",
|
||||
task or "call", reason, resolved_provider, first_err)
|
||||
|
||||
# Narrow the configured-chain skip to the exact model that
|
||||
# failed ONLY for model-specific failures. Auth (401) and
|
||||
# payment (402) errors are provider-wide — the credentials or
|
||||
# account behind every model on that provider are the same — so
|
||||
# a sibling model can't recover; keep skipping the whole
|
||||
# provider so the main-agent-model safety net is still reached.
|
||||
_chain_failed_model = (
|
||||
None if reason in ("auth error", "payment error") else final_model
|
||||
)
|
||||
# Fallback order (#26882, #26803):
|
||||
# 1. User-configured fallback_chain (per-task) if set
|
||||
# 2. For auto: top-level main fallback_providers/fallback_model
|
||||
|
|
@ -8180,7 +8791,8 @@ async def async_call_llm(
|
|||
fb_client, fb_model, fb_label = (None, None, "")
|
||||
if is_auto:
|
||||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
task, resolved_provider or "auto", reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
if fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_main_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
|
|
@ -8189,10 +8801,12 @@ async def async_call_llm(
|
|||
resolved_provider, task, reason=reason)
|
||||
else:
|
||||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
task, resolved_provider or "auto", reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
if fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
|
||||
resolved_provider, task, reason=reason)
|
||||
resolved_provider, task, reason=reason,
|
||||
failed_model=_chain_failed_model)
|
||||
|
||||
if fb_client is not None:
|
||||
# Convert sync fallback client to async
|
||||
|
|
|
|||
204
agent/backend_identity.py
Normal file
204
agent/backend_identity.py
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
"""Single owner for backend identity and failure-scoped skip decisions.
|
||||
|
||||
Every fallback / dedup / skip / quarantine decision in Hermes ultimately asks
|
||||
one question: **"is this candidate the same backend as the one that failed,
|
||||
along the axis that failure invalidated?"** Before this module, that
|
||||
question was re-implemented inline at six call sites across four subsystems,
|
||||
each comparing whatever string was locally convenient (provider label,
|
||||
provider+model, base_url+model, ...). Each incident fixed one site while the
|
||||
others kept the bug: #22548 (same-shim aliases), #70893 (xai-oauth vs xai —
|
||||
same host, distinct credential), #59561 (aux chain skipped sibling models),
|
||||
#72468 (aux main-model safety net, same bug three weeks later), #62984 /
|
||||
#54250 / #57584 (dedup ignoring base_url strands multi-endpoint pools).
|
||||
|
||||
The root insight: "provider" conflates three independent identity axes, and
|
||||
each failure class invalidates a different one:
|
||||
|
||||
* **credential surface** — auth 401 / payment 402 kill everything sharing the
|
||||
credential (every model, every host reached with that key/token).
|
||||
* **endpoint** — DNS failure / connection refused kill everything behind the
|
||||
URL, regardless of model or credential.
|
||||
* **model deployment** — timeout / overload / rate limit / model-incompatible
|
||||
kill ONE model's deployment. A sibling model behind the same URL is an
|
||||
independent deployment (real incident: aux ``glm-5.2`` hung and timed out
|
||||
while main ``macaron-v1-venti`` on the identical endpoint was serving
|
||||
448K-token turns).
|
||||
|
||||
Call sites should build :class:`BackendIdentity` values, classify the failure
|
||||
with :func:`classify_failure_scope`, and ask :func:`should_skip_candidate`.
|
||||
Do not re-implement any comparison inline — extend THIS module instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FailureScope(Enum):
|
||||
"""Which identity axis a failure invalidates."""
|
||||
|
||||
#: Timeout, overload/429, connection blip, model-incompatible, invalid
|
||||
#: response: evidence against ONE model deployment only.
|
||||
MODEL = "model"
|
||||
#: Auth 401 / payment 402: evidence against the shared credential —
|
||||
#: every model reached with it is equally dead.
|
||||
CREDENTIAL = "credential"
|
||||
#: DNS / connection-refused / unreachable host: evidence against the
|
||||
#: endpoint — every model behind the URL is equally dead.
|
||||
ENDPOINT = "endpoint"
|
||||
|
||||
|
||||
#: Reason strings already used by auxiliary_client's except-chain, mapped to
|
||||
#: scopes. Unknown reasons default to MODEL — the least-invalidating scope —
|
||||
#: so an unrecognized failure never over-skips viable candidates.
|
||||
_REASON_SCOPES = {
|
||||
"auth error": FailureScope.CREDENTIAL,
|
||||
"payment error": FailureScope.CREDENTIAL,
|
||||
"rate limit": FailureScope.MODEL,
|
||||
"model incompatible with route": FailureScope.MODEL,
|
||||
"invalid provider response": FailureScope.MODEL,
|
||||
"connection error": FailureScope.MODEL,
|
||||
"timeout": FailureScope.MODEL,
|
||||
}
|
||||
|
||||
|
||||
def classify_failure_scope(reason: Optional[str]) -> FailureScope:
|
||||
"""Map a human-readable failure reason to the identity axis it kills."""
|
||||
return _REASON_SCOPES.get((reason or "").strip().lower(), FailureScope.MODEL)
|
||||
|
||||
|
||||
def _norm_provider(value: Optional[str]) -> str:
|
||||
return (value or "").strip().lower()
|
||||
|
||||
|
||||
def _norm_model(value: Optional[str]) -> str:
|
||||
return (value or "").strip().lower()
|
||||
|
||||
|
||||
def _norm_base_url(value: Optional[str]) -> str:
|
||||
return (value or "").strip().rstrip("/").lower()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendIdentity:
|
||||
"""Normalized identity of one (provider, model, endpoint) deployment.
|
||||
|
||||
Empty fields mean "unknown" — comparisons treat an unknown axis as
|
||||
non-distinguishing (it can neither prove sameness nor difference on its
|
||||
own; the remaining axes decide).
|
||||
"""
|
||||
|
||||
provider: str = ""
|
||||
model: str = ""
|
||||
base_url: str = ""
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
provider: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
) -> "BackendIdentity":
|
||||
return cls(
|
||||
provider=_norm_provider(provider),
|
||||
model=_norm_model(model),
|
||||
base_url=_norm_base_url(base_url),
|
||||
)
|
||||
|
||||
|
||||
def _both_first_class(a: BackendIdentity, b: BackendIdentity) -> bool:
|
||||
"""True when both providers are distinct registered first-class providers.
|
||||
|
||||
Two different registry providers have distinct credential surfaces even
|
||||
when they share an inference host (xai-oauth vs xai, openai-codex vs
|
||||
openai-api) — #70893. Custom/shim aliases are NOT in the registry, so
|
||||
two aliases pointing at one URL still count as the same backend (#22548).
|
||||
"""
|
||||
if not a.provider or not b.provider or a.provider == b.provider:
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
|
||||
return a.provider in PROVIDER_REGISTRY and b.provider in PROVIDER_REGISTRY
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def same_credential_surface(a: BackendIdentity, b: BackendIdentity) -> bool:
|
||||
"""Do two identities share the credential a 401/402 just invalidated?
|
||||
|
||||
Conservative on purpose: an unprovable axis must answer "different"
|
||||
(try the candidate — worst case one wasted RTT) rather than "same"
|
||||
(skip — worst case stranded failover). Two distinct custom labels at
|
||||
one URL may carry different per-entry api_keys, so a shared URL alone
|
||||
never proves a shared credential; it is only used as a weak signal
|
||||
when a provider label is missing entirely.
|
||||
"""
|
||||
if a.provider and b.provider:
|
||||
# Same label = same configured credential. Different labels =
|
||||
# different credential config (first-class registry providers
|
||||
# explicitly so — #70893; custom entries can each carry their own
|
||||
# api_key, so sameness is unprovable and we must not skip).
|
||||
return a.provider == b.provider
|
||||
# Provider unknown on a side: same explicit URL is the best signal left.
|
||||
return bool(a.base_url and a.base_url == b.base_url)
|
||||
|
||||
|
||||
def same_endpoint(a: BackendIdentity, b: BackendIdentity) -> bool:
|
||||
"""Do two identities sit behind the endpoint that just went unreachable?"""
|
||||
if a.base_url and b.base_url:
|
||||
return a.base_url == b.base_url
|
||||
# An unknown base_url inherits the provider default → same provider
|
||||
# label implies the same default endpoint.
|
||||
return bool(a.provider and a.provider == b.provider)
|
||||
|
||||
|
||||
def same_deployment(a: BackendIdentity, b: BackendIdentity) -> bool:
|
||||
"""Are these the exact same model deployment (the thing a timeout kills)?
|
||||
|
||||
Provider+model must match; the base_url axis distinguishes only when BOTH
|
||||
sides carry an explicit URL (#62984: same provider+model on two different
|
||||
explicit URLs is two deployments — a pool). A side with an unknown URL
|
||||
inherits the provider default and cannot prove difference.
|
||||
"""
|
||||
if not (a.provider and b.provider and a.provider == b.provider):
|
||||
# Same-host different-label shims: same URL + same model IS the same
|
||||
# deployment even when the alias labels differ (#22548) — unless both
|
||||
# labels are first-class registry providers (#70893).
|
||||
if (
|
||||
a.base_url
|
||||
and a.base_url == b.base_url
|
||||
and a.model
|
||||
and a.model == b.model
|
||||
and not _both_first_class(a, b)
|
||||
):
|
||||
return True
|
||||
return False
|
||||
if not (a.model and b.model and a.model == b.model):
|
||||
return False
|
||||
if a.base_url and b.base_url and a.base_url != b.base_url:
|
||||
return False # distinct explicit endpoints — a pool, not a dup
|
||||
return True
|
||||
|
||||
|
||||
def should_skip_candidate(
|
||||
candidate: BackendIdentity,
|
||||
failed: BackendIdentity,
|
||||
scope: FailureScope = FailureScope.MODEL,
|
||||
) -> bool:
|
||||
"""THE skip predicate: would trying ``candidate`` just repeat the failure?
|
||||
|
||||
True when the candidate is the same backend as ``failed`` along the axis
|
||||
``scope`` says the failure invalidated. Every fallback/dedup/skip site
|
||||
must call this instead of comparing labels inline.
|
||||
"""
|
||||
if scope is FailureScope.CREDENTIAL:
|
||||
return same_credential_surface(candidate, failed)
|
||||
if scope is FailureScope.ENDPOINT:
|
||||
return same_endpoint(candidate, failed)
|
||||
return same_deployment(candidate, failed)
|
||||
|
|
@ -209,7 +209,10 @@ _SKILL_REVIEW_PROMPT = (
|
|||
"conversation for skills the user loaded via /skill-name or you "
|
||||
"read via skill_view. If any of them covers the territory of the "
|
||||
"new learning, PATCH that one first. It is the skill that was in "
|
||||
"play, so it's the right one to extend.\n"
|
||||
"play, so it's the right one to extend — but only if it is "
|
||||
"curator-managed. Bundled, hub, pinned, and user-owned skills are "
|
||||
"off-limits to you no matter how relevant (see Protected skills "
|
||||
"below); for those, fall through to the next option.\n"
|
||||
" 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). "
|
||||
"If no loaded skill fits but an existing class-level skill does, "
|
||||
"patch it. Add a subsection, a pitfall, or broaden a trigger.\n"
|
||||
|
|
@ -251,10 +254,18 @@ _SKILL_REVIEW_PROMPT = (
|
|||
"Protected skills (DO NOT edit these):\n"
|
||||
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
|
||||
" • Hub-installed skills (installed via 'hermes skills install').\n"
|
||||
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
|
||||
"pin only blocks deletion/archive/consolidation by the curator, not "
|
||||
"content updates. Patch them when a pitfall or missing step turns up, "
|
||||
"same as any other agent-created skill.\n"
|
||||
" • Skills in skills.external_dirs (externally owned).\n"
|
||||
" • PINNED skills (marked via 'hermes curator pin'). You are an "
|
||||
"autonomous no-user-present actor, so pin blocks your writes too — "
|
||||
"content updates included. Only the user, in a foreground session, "
|
||||
"can change a pinned skill.\n"
|
||||
" • USER-OWNED skills — anything not curator-managed. A skill the "
|
||||
"user hand-wrote, installed by URL, or asked a foreground agent to "
|
||||
"create is theirs, not yours; your writes to it WILL be refused. "
|
||||
"This includes skills that were loaded or consulted this session: "
|
||||
"being in play does not make one yours to edit. If such a skill is "
|
||||
"wrong or outdated, say so in your reply and recommend "
|
||||
"'hermes curator adopt <name>' — do not try to patch it.\n"
|
||||
"If the only skills that need updating are protected, say\n"
|
||||
"'Nothing to save.' and stop.\n\n"
|
||||
"Do NOT capture (these become persistent self-imposed constraints "
|
||||
|
|
@ -309,7 +320,9 @@ _COMBINED_REVIEW_PROMPT = (
|
|||
" 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were "
|
||||
"loaded via /skill-name or skill_view in the conversation. If one "
|
||||
"of them covers the learning, PATCH it first. It was in play; "
|
||||
"it's the right place.\n"
|
||||
"it's the right place — provided it is curator-managed. Protected "
|
||||
"and user-owned skills are off-limits however relevant; fall "
|
||||
"through when one of those is the best fit.\n"
|
||||
" 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to "
|
||||
"find the right one). Patch it.\n"
|
||||
" 3. ADD A SUPPORT FILE under an existing umbrella via "
|
||||
|
|
@ -337,10 +350,15 @@ _COMBINED_REVIEW_PROMPT = (
|
|||
"Protected skills (DO NOT edit these):\n"
|
||||
" • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n"
|
||||
" • Hub-installed skills (installed via 'hermes skills install').\n"
|
||||
"Pinned skills (marked via 'hermes curator pin') CAN be improved — "
|
||||
"pin only blocks deletion/archive/consolidation by the curator, not "
|
||||
"content updates. Patch them when a pitfall or missing step turns up, "
|
||||
"same as any other agent-created skill.\n"
|
||||
" • Skills in skills.external_dirs (externally owned).\n"
|
||||
" • PINNED skills (marked via 'hermes curator pin'). Pin blocks "
|
||||
"autonomous writes entirely — content updates included — because no "
|
||||
"user is present to consent. Only a foreground session can change one.\n"
|
||||
" • USER-OWNED skills — anything not curator-managed (hand-written, "
|
||||
"URL-installed, or created by a foreground agent at the user's "
|
||||
"request). Your writes to these WILL be refused, including to skills "
|
||||
"loaded or consulted this session. If one is wrong, say so in your "
|
||||
"reply and recommend 'hermes curator adopt <name>' instead.\n"
|
||||
"If the only skills that need updating are protected, say\n"
|
||||
"'Nothing to save.' and stop.\n\n"
|
||||
"Do NOT capture as skills (these become persistent self-imposed "
|
||||
|
|
|
|||
|
|
@ -107,6 +107,22 @@ class CardInfo:
|
|||
return f"{self.masked} — {label}" if label else self.masked
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PaymentMethodInfo:
|
||||
"""The payment method on file. `kind` is "card", "link", or "unknown"
|
||||
— anything else is normalised to "unknown" at parse time, so consumers
|
||||
only ever see fields that belong to the kind they are looking at."""
|
||||
|
||||
kind: str
|
||||
brand: Optional[str] = None
|
||||
last4: Optional[str] = None
|
||||
wallet: Optional[str] = None
|
||||
email: Optional[str] = None
|
||||
resolved_via: Optional[str] = None
|
||||
#: What the server called it, when we did not recognise the kind.
|
||||
raw_kind: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MonthlyCap:
|
||||
limit_usd: Optional[Decimal] = None
|
||||
|
|
@ -150,6 +166,7 @@ class BillingState:
|
|||
min_usd: Optional[Decimal] = None
|
||||
max_usd: Optional[Decimal] = None
|
||||
card: Optional[CardInfo] = None
|
||||
payment_method: Optional[PaymentMethodInfo] = None
|
||||
monthly_cap: Optional[MonthlyCap] = None
|
||||
auto_reload: Optional[AutoReload] = None
|
||||
portal_url: Optional[str] = None
|
||||
|
|
@ -201,6 +218,41 @@ def _parse_card(raw: Any) -> Optional[CardInfo]:
|
|||
return CardInfo(brand=brand, last4=last4, resolved_via=resolved_via)
|
||||
|
||||
|
||||
def _parse_payment_method(raw: Any) -> Optional[PaymentMethodInfo]:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
kind = raw.get("kind")
|
||||
if not isinstance(kind, str):
|
||||
return None
|
||||
|
||||
def _optional_string(key: str) -> Optional[str]:
|
||||
value = raw.get(key)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
resolved_via = _optional_string("resolvedVia")
|
||||
brand = _optional_string("brand")
|
||||
last4 = _optional_string("last4")
|
||||
# Settle the kind here, the way _parse_card settles a card, so nothing
|
||||
# downstream has to re-check which fields this kind is allowed to have.
|
||||
if kind == "card" and brand and last4:
|
||||
return PaymentMethodInfo(
|
||||
kind="card",
|
||||
brand=brand,
|
||||
last4=last4,
|
||||
wallet=_optional_string("wallet"),
|
||||
resolved_via=resolved_via,
|
||||
)
|
||||
if kind == "link":
|
||||
return PaymentMethodInfo(
|
||||
kind="link",
|
||||
email=_optional_string("email"),
|
||||
resolved_via=resolved_via,
|
||||
)
|
||||
return PaymentMethodInfo(
|
||||
kind="unknown", raw_kind=kind, resolved_via=resolved_via
|
||||
)
|
||||
|
||||
|
||||
def _parse_monthly_cap(raw: Any) -> Optional[MonthlyCap]:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
|
|
@ -274,6 +326,7 @@ def billing_state_from_payload(
|
|||
min_usd=parse_money(bounds.get("minUsd")),
|
||||
max_usd=parse_money(bounds.get("maxUsd")),
|
||||
card=_parse_card(payload.get("card")),
|
||||
payment_method=_parse_payment_method(payload.get("paymentMethod")),
|
||||
monthly_cap=_parse_monthly_cap(payload.get("monthlyCap")),
|
||||
auto_reload=_parse_auto_reload(payload.get("autoReload")),
|
||||
portal_url=portal_url,
|
||||
|
|
|
|||
|
|
@ -188,6 +188,31 @@ def _provider_preferences_for_agent(agent) -> Dict[str, Any]:
|
|||
return preferences
|
||||
|
||||
|
||||
def _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs: dict) -> dict:
|
||||
"""Merge Portal ``tags`` / ``session_id`` onto an Anthropic Messages kwargs dict.
|
||||
|
||||
The Nous provider profile is only consulted by the OpenAI-wire transport;
|
||||
anthropic_messages callers must merge it themselves. Passes ``session_id``
|
||||
only — not ``provider_preferences`` (those become a top-level ``provider``
|
||||
routing object on the OpenAI wire). Never blocks a turn on tagging.
|
||||
"""
|
||||
if getattr(agent, "provider", None) not in {"nous", "nous-portal", "nousresearch"}:
|
||||
return anthropic_kwargs
|
||||
try:
|
||||
from providers import get_provider_profile
|
||||
|
||||
nous_profile = get_provider_profile("nous")
|
||||
if nous_profile is not None:
|
||||
anthropic_kwargs.setdefault("extra_body", {}).update(
|
||||
nous_profile.build_extra_body(
|
||||
session_id=getattr(agent, "session_id", None)
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — never block a turn on tagging
|
||||
logger.debug("Nous Portal extra_body merge failed: %s", exc)
|
||||
return anthropic_kwargs
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.getenv(name, str(default)))
|
||||
|
|
@ -433,26 +458,56 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client):
|
|||
|
||||
|
||||
def should_use_direct_api_call(agent) -> bool:
|
||||
"""Whether a cron OpenAI-wire request should skip the interrupt worker.
|
||||
"""Whether an OpenAI-wire request should skip the interrupt worker.
|
||||
|
||||
Issue #62151 is specific to OpenRouter's chat-completions path inside the
|
||||
gateway cron thread stack. Keep native/Codex/Bedrock/MoA transports on their
|
||||
established workers: their cancellation and client ownership differ, and
|
||||
the report provides no evidence that those paths share the pre-HTTP wedge.
|
||||
Two nested-pool contexts wedge before the socket opens when the request
|
||||
is pushed onto yet another daemon worker thread:
|
||||
|
||||
- Gateway cron turns (#62151): gateway asyncio loop → cron thread →
|
||||
interrupt worker. Fixed by running inline.
|
||||
- Delegated children (#60203): gateway loop → async-delegation executor
|
||||
(module-lifetime daemon pool) → per-child timeout executor → interrupt
|
||||
worker. Same fingerprint after multi-day gateway uptime — children hang
|
||||
at their FIRST API call with zero stale-detector output (the worker
|
||||
never reaches dispatch), all providers, restart cures it. The cron fix
|
||||
originally excluded delegation "for lack of evidence"; #60203 is that
|
||||
evidence.
|
||||
|
||||
Running inline drops the deepest thread layer (whose only job is
|
||||
interactive-interrupt responsiveness). Interrupts still work: the inline
|
||||
path registers ``agent._active_request_abort``, which ``interrupt()``
|
||||
invokes cross-thread to shut the active sockets — the same mechanism the
|
||||
async-delegation stall monitor (#72227) relies on.
|
||||
|
||||
Keep native/Codex/Bedrock/MoA transports on their established workers:
|
||||
their cancellation and client ownership differ.
|
||||
"""
|
||||
return (
|
||||
getattr(agent, "platform", None) == "cron"
|
||||
and getattr(agent, "api_mode", None) == "chat_completions"
|
||||
and getattr(agent, "provider", None) != "moa"
|
||||
)
|
||||
if getattr(agent, "api_mode", None) != "chat_completions":
|
||||
return False
|
||||
if getattr(agent, "provider", None) == "moa":
|
||||
return False
|
||||
if getattr(agent, "platform", None) == "cron":
|
||||
return True
|
||||
# Delegated child (delegate_task sync or background) — detected via the
|
||||
# execution ContextVar set by _run_single_child, with the agent's own
|
||||
# platform stamp as a fallback for callers that bypass the runner.
|
||||
try:
|
||||
from agent.delegation_context import is_delegated_child_context
|
||||
|
||||
if is_delegated_child_context():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(agent, "platform", None) == "subagent"
|
||||
|
||||
|
||||
def direct_api_call(agent, api_kwargs: dict):
|
||||
"""Run a non-streaming LLM call inline on the conversation thread.
|
||||
|
||||
Used when ``should_use_direct_api_call`` is True. Skips the interrupt worker
|
||||
(whose only job is interactive-interrupt responsiveness, which this context
|
||||
does not have) so the nested-pool deadlock (#62151) cannot occur. Because the
|
||||
Used when ``should_use_direct_api_call`` is True (cron turns and
|
||||
delegated children). Skips the interrupt worker (whose only job is
|
||||
interactive-interrupt responsiveness, which these contexts do not have)
|
||||
so the nested-pool deadlock (#62151, #60203) cannot occur. Because the
|
||||
request runs in-flight normally, the per-request OpenAI client's own httpx
|
||||
timeout (provider ``request_timeout_seconds`` / ``HERMES_API_TIMEOUT``) bounds
|
||||
a genuinely hung provider — the same bound interactive calls already rely on.
|
||||
|
|
@ -463,7 +518,7 @@ def direct_api_call(agent, api_kwargs: dict):
|
|||
request_client_lock = threading.Lock()
|
||||
|
||||
def _abort_active_request(reason: str) -> None:
|
||||
"""Abort the inline request from cron's watchdog/interrupt thread."""
|
||||
"""Abort the inline request from a watchdog/interrupt thread."""
|
||||
with request_client_lock:
|
||||
request_client = request_client_holder["client"]
|
||||
if request_client is not None:
|
||||
|
|
@ -993,7 +1048,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None)
|
||||
if ephemeral_out is not None:
|
||||
agent._ephemeral_max_output_tokens = None # consume immediately
|
||||
return _transport.build_kwargs(
|
||||
anthropic_kwargs = _transport.build_kwargs(
|
||||
model=agent.model,
|
||||
messages=anthropic_messages,
|
||||
tools=tools_for_api,
|
||||
|
|
@ -1006,6 +1061,12 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
fast_mode=(agent.request_overrides or {}).get("speed") == "fast",
|
||||
drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)),
|
||||
)
|
||||
# Nous Portal reads ``tags`` and ``session_id`` as top-level body fields
|
||||
# on its Messages route the same way it does on /chat/completions, but
|
||||
# the profile hook that produces them is only consulted by the
|
||||
# OpenAI-wire transport. Merge them here so Messages traffic keeps
|
||||
# product attribution and sticky routing.
|
||||
return _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs)
|
||||
|
||||
# AWS Bedrock native Converse API — bypasses the OpenAI client entirely.
|
||||
# The adapter handles message/tool conversion and boto3 calls directly.
|
||||
|
|
@ -1076,6 +1137,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
tools=tools_for_api,
|
||||
reasoning_config=agent.reasoning_config,
|
||||
session_id=getattr(agent, "session_id", None),
|
||||
base_url=agent.base_url,
|
||||
max_tokens=agent.max_tokens,
|
||||
timeout=agent._resolved_api_call_timeout(),
|
||||
request_overrides=agent.request_overrides,
|
||||
|
|
@ -1598,29 +1660,28 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
)
|
||||
return agent._try_activate_fallback(reason)
|
||||
|
||||
# Skip entries that resolve to the current (provider, model) — falling
|
||||
# back to the same backend that just failed loops the failure. Compare
|
||||
# base_url too so two distinct custom_providers entries pointing at the
|
||||
# same shim/proxy URL also dedup. See issue #22548.
|
||||
current_provider = (getattr(agent, "provider", "") or "").strip().lower()
|
||||
current_model = (getattr(agent, "model", "") or "").strip()
|
||||
current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower()
|
||||
fb_base_url_for_dedup = (fb.get("base_url") or "").strip().rstrip("/").lower()
|
||||
if fb_provider == current_provider and fb_model == current_model:
|
||||
# Skip entries that resolve to the same backend that just failed —
|
||||
# falling back to it loops the failure. Identity semantics (which axes
|
||||
# distinguish two backends, shim aliases, first-class credential
|
||||
# surfaces, multi-endpoint pools) are owned by agent.backend_identity —
|
||||
# see #22548, #70893, #62984. Do not re-implement comparisons here.
|
||||
from agent.backend_identity import BackendIdentity, should_skip_candidate
|
||||
|
||||
current_ident = BackendIdentity.build(
|
||||
provider=getattr(agent, "provider", ""),
|
||||
model=getattr(agent, "model", ""),
|
||||
base_url=str(getattr(agent, "base_url", "") or ""),
|
||||
)
|
||||
fb_ident = BackendIdentity.build(
|
||||
provider=fb_provider,
|
||||
model=fb_model,
|
||||
base_url=(fb.get("base_url") or ""),
|
||||
)
|
||||
if should_skip_candidate(fb_ident, current_ident):
|
||||
logger.warning(
|
||||
"Fallback skip: chain entry %s/%s matches current provider/model",
|
||||
fb_provider, fb_model,
|
||||
)
|
||||
return agent._try_activate_fallback(reason)
|
||||
if (
|
||||
fb_base_url_for_dedup
|
||||
and current_base_url
|
||||
and fb_base_url_for_dedup == current_base_url
|
||||
and fb_model == current_model
|
||||
):
|
||||
logger.warning(
|
||||
"Fallback skip: chain entry base_url %s matches current backend",
|
||||
fb_base_url_for_dedup,
|
||||
"Fallback skip: chain entry %s/%s resolves to the same backend "
|
||||
"as the current one (%s)",
|
||||
fb_provider, fb_model, current_ident.base_url or current_ident.provider,
|
||||
)
|
||||
return agent._try_activate_fallback(reason)
|
||||
|
||||
|
|
@ -1671,6 +1732,14 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
_fb_is_azure = agent._is_azure_openai_url(fb_base_url)
|
||||
if fb_provider == "openai-codex":
|
||||
fb_api_mode = "codex_responses"
|
||||
elif fb_provider in {"nous", "nous-portal", "nousresearch"}:
|
||||
# Portal is dual-wire: anthropic/* must land on /v1/messages.
|
||||
# resolve_provider_client still returns an OpenAI client for
|
||||
# Nous; the anthropic_messages branch below rebuilds the native
|
||||
# client from that credential + base_url.
|
||||
from hermes_cli.providers import nous_api_mode
|
||||
|
||||
fb_api_mode = nous_api_mode(fb_model)
|
||||
elif (
|
||||
fb_provider == "anthropic"
|
||||
or fb_base_url.rstrip("/").lower().endswith("/anthropic")
|
||||
|
|
@ -1739,6 +1808,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
fb_provider, fb_model, _pool_provider,
|
||||
)
|
||||
agent._credential_pool = None
|
||||
agent._credential_pool_entry_id = None
|
||||
if getattr(agent, "_credential_pool", None) is None:
|
||||
try:
|
||||
from agent.credential_pool import load_pool
|
||||
|
|
@ -1801,6 +1871,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
# not only after a later credential-rotation rebuild.
|
||||
agent._replace_primary_openai_client(reason="fallback_timeout_apply")
|
||||
|
||||
from agent.agent_runtime_helpers import sync_credential_pool_entry_id
|
||||
sync_credential_pool_entry_id(agent)
|
||||
|
||||
# Re-evaluate prompt caching for the new provider/model
|
||||
agent._use_prompt_caching, agent._use_native_cache_layout = (
|
||||
agent._anthropic_prompt_cache_policy(
|
||||
|
|
@ -2093,7 +2166,9 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str:
|
|||
_ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None,
|
||||
max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config,
|
||||
is_oauth=agent._is_anthropic_oauth,
|
||||
preserve_dots=agent._anthropic_preserve_dots())
|
||||
preserve_dots=agent._anthropic_preserve_dots(),
|
||||
base_url=getattr(agent, "_anthropic_base_url", None))
|
||||
_ant_kw = _merge_nous_portal_messages_extra_body(agent, _ant_kw)
|
||||
summary_response = agent._anthropic_messages_create(_ant_kw)
|
||||
_summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth)
|
||||
final_response = (_summary_result.content or "").strip()
|
||||
|
|
@ -2123,7 +2198,9 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str:
|
|||
_ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None,
|
||||
is_oauth=agent._is_anthropic_oauth,
|
||||
max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config,
|
||||
preserve_dots=agent._anthropic_preserve_dots())
|
||||
preserve_dots=agent._anthropic_preserve_dots(),
|
||||
base_url=getattr(agent, "_anthropic_base_url", None))
|
||||
_ant_kw2 = _merge_nous_portal_messages_extra_body(agent, _ant_kw2)
|
||||
retry_response = agent._anthropic_messages_create(_ant_kw2)
|
||||
_retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth)
|
||||
final_response = (_retry_result.content or "").strip()
|
||||
|
|
@ -3444,13 +3521,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# already worker-owned-closed by _close_request_client_once
|
||||
# above; the next attempt builds a fresh one. The shared
|
||||
# _anthropic_client is never closed from inside a request.
|
||||
if agent.api_mode != "anthropic_messages":
|
||||
try:
|
||||
agent._replace_primary_openai_client(
|
||||
reason="stream_mid_tool_retry_pool_cleanup"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector for OpenAI.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next attempt.
|
||||
continue
|
||||
|
||||
# SSE error events from proxies (e.g. OpenRouter sends
|
||||
|
|
@ -3509,13 +3582,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# above; next attempt builds fresh), so the shared
|
||||
# _anthropic_client is never closed from inside a
|
||||
# request — only the OpenAI-wire primary is refreshed.
|
||||
if agent.api_mode != "anthropic_messages":
|
||||
try:
|
||||
agent._replace_primary_openai_client(
|
||||
reason="stream_retry_pool_cleanup"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector for OpenAI.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next attempt.
|
||||
continue
|
||||
# Retries exhausted. Log the final failure with
|
||||
# full diagnostic detail (chain, headers,
|
||||
|
|
@ -3758,10 +3827,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# FD-recycle corruption vector. Nothing further is needed.
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup")
|
||||
except Exception:
|
||||
pass
|
||||
# #70773: same FD-recycle corruption vector as #67142.
|
||||
# The shared OpenAI client's connection pool must NOT be
|
||||
# closed from this watchdog/poll thread — worker threads
|
||||
# from previous stale-killed attempts may still be
|
||||
# unwinding their SSL BIOs. The request-local client is
|
||||
# already closed above via _close_request_client_once.
|
||||
# The shared client will be replaced lazily by
|
||||
# _ensure_primary_openai_client on the next request.
|
||||
pass
|
||||
# Reset the timer so we don't kill repeatedly while
|
||||
# the inner thread processes the closure.
|
||||
last_chunk_time["t"] = time.time()
|
||||
|
|
|
|||
|
|
@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs(
|
|||
allowed_keys = {
|
||||
"model", "instructions", "input", "tools", "store",
|
||||
"reasoning", "include", "max_output_tokens", "temperature",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key",
|
||||
"prompt_cache_retention", "service_tier",
|
||||
"extra_headers", "extra_body", "timeout",
|
||||
}
|
||||
normalized: Dict[str, Any] = {
|
||||
|
|
@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs(
|
|||
if isinstance(temperature, (int, float)):
|
||||
normalized["temperature"] = float(temperature)
|
||||
|
||||
# Pass through tool_choice, parallel_tool_calls, prompt_cache_key
|
||||
for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"):
|
||||
# Pass through cache routing/retention and tool-dispatch hints.
|
||||
for passthrough_key in (
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"prompt_cache_key",
|
||||
"prompt_cache_retention",
|
||||
):
|
||||
val = api_kwargs.get(passthrough_key)
|
||||
if val is not None:
|
||||
normalized[passthrough_key] = val
|
||||
|
|
|
|||
|
|
@ -778,12 +778,27 @@ def run_codex_app_server_turn(
|
|||
# the already-flushed user turn). See gateway/run.py agent_persisted.
|
||||
if getattr(agent, "_session_db", None) is not None:
|
||||
try:
|
||||
agent._flush_messages_to_session_db(messages)
|
||||
_codex_flush_ok = agent._flush_messages_to_session_db(messages)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
_codex_flush_ok = False
|
||||
logger.warning(
|
||||
"codex app-server projected-message flush failed",
|
||||
exc_info=True,
|
||||
)
|
||||
if _codex_flush_ok is False:
|
||||
# Unlike the chat-completions loop (which fails closed BEFORE
|
||||
# projection — see conversation_loop session_persistence_failed),
|
||||
# codex output has already streamed to the user by the time this
|
||||
# flush runs, so there is nothing left to withhold. We cannot
|
||||
# flip agent_persisted=False either: the gateway fallback write
|
||||
# would re-INSERT the already-flushed user turn (#860/#42039).
|
||||
# Surface the durability gap loudly instead of a silent debug.
|
||||
logger.warning(
|
||||
"codex app-server turn was delivered but could NOT be "
|
||||
"persisted to the session DB (session=%s) — this turn "
|
||||
"will be missing after restart/resume",
|
||||
getattr(agent, "session_id", None),
|
||||
)
|
||||
|
||||
|
||||
# Counter ticks for the agent-improvement loop.
|
||||
|
|
|
|||
|
|
@ -520,30 +520,46 @@ class RuntimeMode:
|
|||
return None
|
||||
return [self.profile.toolset, *_enabled_mcp_servers(config)]
|
||||
|
||||
def system_blocks(self) -> list[str]:
|
||||
"""Stable system-prompt blocks for this posture (brief + workspace).
|
||||
def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Return prefix, workspace, and trailing posture blocks separately.
|
||||
|
||||
The operating brief carries a model-family edit-format nudge appended
|
||||
to it (one cached string, not a separate block) so the model is steered
|
||||
toward the `patch` mode it handles best — see ``_edit_format_line``.
|
||||
|
||||
The three lists preserve the historical flat prompt order: the brief,
|
||||
the live workspace snapshot, then configured operator instructions.
|
||||
Prompt assembly can therefore put a cache boundary before the snapshot
|
||||
without changing the persisted system-prompt bytes.
|
||||
"""
|
||||
if not self.is_coding:
|
||||
return []
|
||||
blocks: list[str] = []
|
||||
return [], [], []
|
||||
prefix: list[str] = []
|
||||
workspace_parts: list[str] = []
|
||||
trailing: list[str] = []
|
||||
if self.profile.guidance:
|
||||
brief = self.profile.guidance
|
||||
edit_line = _edit_format_line(self.model)
|
||||
if edit_line:
|
||||
brief = f"{brief}\n{edit_line}"
|
||||
blocks.append(brief)
|
||||
prefix.append(brief)
|
||||
workspace = build_coding_workspace_block(self.cwd)
|
||||
if workspace:
|
||||
blocks.append(workspace)
|
||||
workspace_parts.append(workspace)
|
||||
# Operator instructions ride their own block so the brief (block 0) stays
|
||||
# byte-stable and cache-keyed independently of user config.
|
||||
if self.instructions:
|
||||
blocks.append(f"Operator instructions (from config):\n{self.instructions}")
|
||||
return blocks
|
||||
trailing.append(f"Operator instructions (from config):\n{self.instructions}")
|
||||
return prefix, workspace_parts, trailing
|
||||
|
||||
def system_blocks(self) -> list[str]:
|
||||
"""Return posture blocks in their historical display order.
|
||||
|
||||
``system_prompt_parts`` is the cache-aware API. This compatibility
|
||||
helper retains the public flat list for callers outside prompt assembly.
|
||||
"""
|
||||
prefix, workspace, trailing = self.system_prompt_parts()
|
||||
return [*prefix, *workspace, *trailing]
|
||||
|
||||
def compact_skill_categories(self) -> frozenset[str]:
|
||||
"""Skill categories to demote to names-only in the prompt's skill index.
|
||||
|
|
@ -644,6 +660,19 @@ def coding_system_blocks(
|
|||
).system_blocks()
|
||||
|
||||
|
||||
def coding_system_prompt_parts(
|
||||
*,
|
||||
platform: Optional[str] = None,
|
||||
cwd: Optional[str | Path] = None,
|
||||
config: Optional[dict[str, Any]] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> tuple[list[str], list[str], list[str]]:
|
||||
"""Return coding prefix, workspace snapshot, and trailing guidance."""
|
||||
return resolve_runtime_mode(
|
||||
platform=platform, cwd=cwd, config=config, model=model
|
||||
).system_prompt_parts()
|
||||
|
||||
|
||||
def coding_compact_skill_categories(
|
||||
*,
|
||||
platform: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -154,3 +154,207 @@ def compute_session_context_breakdown(
|
|||
"estimated_total": estimated_total,
|
||||
"model": getattr(agent, "model", "") or "",
|
||||
}
|
||||
|
||||
|
||||
# ── /context rendering (CLI + gateway) ──────────────────────────────────────
|
||||
#
|
||||
# Pure text renderers over the payload above. The CLI shows a glyph block-grid
|
||||
# plus a category table; the gateway uses the same table without the grid
|
||||
# (proportional monospace is not guaranteed on messaging platforms).
|
||||
|
||||
_CATEGORY_GLYPHS = {
|
||||
"system_prompt": "■",
|
||||
"tool_definitions": "▣",
|
||||
"rules": "▩",
|
||||
"skills": "▤",
|
||||
"mcp": "▥",
|
||||
"subagent_definitions": "▦",
|
||||
"memory": "▧",
|
||||
"conversation": "▨",
|
||||
}
|
||||
_FREE_GLYPH = "·"
|
||||
_GRID_COLUMNS = 20
|
||||
_GRID_ROWS = 5 # 100 cells → 1 cell per percent of the context window
|
||||
|
||||
# Human-readable tables cap the expanded listings; nothing is dropped from
|
||||
# the underlying data.
|
||||
_DETAILS_TABLE_LIMIT = 15
|
||||
|
||||
|
||||
def _bytes_to_tokens(size: Optional[int]) -> Optional[int]:
|
||||
if size is None:
|
||||
return None
|
||||
return (int(size) + 3) // 4
|
||||
|
||||
|
||||
def compute_context_details(agent: Any) -> Dict[str, Any]:
|
||||
"""Expanded per-skill / per-toolset cost listing for ``/context all``.
|
||||
|
||||
Reuses the ``hermes prompt-size`` attribution mechanism (PR #66656):
|
||||
per-skill index-line bytes parsed from the live ``<available_skills>``
|
||||
block, and per-toolset schema bytes attributed via the tool registry's
|
||||
canonical tool→toolset map. Byte figures are converted to the same
|
||||
chars/4 token heuristic the categories above use.
|
||||
"""
|
||||
from hermes_cli.prompt_size import (
|
||||
_compute_skills_breakdown,
|
||||
_compute_toolsets_breakdown,
|
||||
)
|
||||
from agent.system_prompt import build_system_prompt_parts
|
||||
|
||||
parts = build_system_prompt_parts(agent)
|
||||
stable = parts.get("stable", "") or ""
|
||||
skills_match = _SKILLS_BLOCK_RE.search(stable)
|
||||
skills_block = skills_match.group(0) if skills_match else ""
|
||||
|
||||
skills: List[Dict[str, Any]] = []
|
||||
if skills_block:
|
||||
for entry in _compute_skills_breakdown(skills_block):
|
||||
skills.append({
|
||||
"name": entry.get("name", ""),
|
||||
"index_tokens": _bytes_to_tokens(entry.get("index_line_bytes")) or 0,
|
||||
"skill_md_tokens": _bytes_to_tokens(entry.get("skill_md_bytes")),
|
||||
})
|
||||
|
||||
toolsets: List[Dict[str, Any]] = []
|
||||
tools = list(getattr(agent, "tools", None) or [])
|
||||
if tools:
|
||||
for group in _compute_toolsets_breakdown(tools):
|
||||
toolsets.append({
|
||||
"toolset": group.get("toolset", ""),
|
||||
"tool_count": int(group.get("tool_count", 0) or 0),
|
||||
"schema_tokens": _bytes_to_tokens(group.get("json_bytes")) or 0,
|
||||
})
|
||||
|
||||
return {"skills": skills, "toolsets": toolsets}
|
||||
|
||||
|
||||
def render_context_grid(payload: Dict[str, Any]) -> List[str]:
|
||||
"""Render the payload as a Claude Code-style glyph block grid.
|
||||
|
||||
100 cells (5×20), each one percent of the model context window. Categories
|
||||
fill in declaration order; the remainder renders as free space.
|
||||
"""
|
||||
context_max = int(payload.get("context_max") or 0)
|
||||
categories = payload.get("categories") or []
|
||||
total_cells = _GRID_COLUMNS * _GRID_ROWS
|
||||
|
||||
cells: List[str] = []
|
||||
if context_max > 0:
|
||||
for cat in categories:
|
||||
tokens = int(cat.get("tokens") or 0)
|
||||
n = round(tokens / context_max * total_cells)
|
||||
if tokens > 0 and n == 0:
|
||||
n = 1 # never render a nonzero category as invisible
|
||||
glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪")
|
||||
cells.extend([glyph] * n)
|
||||
cells = cells[:total_cells]
|
||||
cells.extend([_FREE_GLYPH] * (total_cells - len(cells)))
|
||||
|
||||
return [
|
||||
" ".join(cells[row * _GRID_COLUMNS:(row + 1) * _GRID_COLUMNS])
|
||||
for row in range(_GRID_ROWS)
|
||||
]
|
||||
|
||||
|
||||
def render_context_category_lines(payload: Dict[str, Any]) -> List[str]:
|
||||
"""Render the 'Estimated usage by category' table as plain-text lines."""
|
||||
categories = payload.get("categories") or []
|
||||
context_max = int(payload.get("context_max") or 0)
|
||||
estimated_total = int(payload.get("estimated_total") or 0)
|
||||
denom = context_max or estimated_total
|
||||
|
||||
lines = ["Estimated usage by category"]
|
||||
if not categories:
|
||||
lines.append(" (no data yet — send a message first)")
|
||||
return lines
|
||||
|
||||
width = max(len(str(cat.get("label") or "")) for cat in categories)
|
||||
width = max(width, len("Free space"))
|
||||
for cat in categories:
|
||||
tokens = int(cat.get("tokens") or 0)
|
||||
glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪")
|
||||
pct = tokens / denom * 100 if denom else 0.0
|
||||
label = str(cat.get("label") or cat.get("id") or "")
|
||||
lines.append(f"{glyph} {label:<{width}} {tokens:>9,} tokens {pct:>5.1f}%")
|
||||
if context_max > 0:
|
||||
free = max(0, context_max - estimated_total)
|
||||
pct = free / context_max * 100
|
||||
lines.append(f"{_FREE_GLYPH} {'Free space':<{width}} {free:>9,} tokens {pct:>5.1f}%")
|
||||
return lines
|
||||
|
||||
|
||||
def render_context_details_lines(details: Dict[str, Any]) -> List[str]:
|
||||
"""Render the expanded ``/context all`` per-skill / per-toolset tables."""
|
||||
lines: List[str] = []
|
||||
|
||||
toolsets = details.get("toolsets") or []
|
||||
if toolsets:
|
||||
lines.append("Toolsets by schema cost (largest first)")
|
||||
for group in toolsets[:_DETAILS_TABLE_LIMIT]:
|
||||
lines.append(
|
||||
f" {group['toolset']:<24} {group['tool_count']:>3} tools"
|
||||
f" {group['schema_tokens']:>8,} tokens"
|
||||
)
|
||||
remaining = len(toolsets) - _DETAILS_TABLE_LIMIT
|
||||
if remaining > 0:
|
||||
lines.append(f" … and {remaining} more")
|
||||
|
||||
skills = details.get("skills") or []
|
||||
if skills:
|
||||
if lines:
|
||||
lines.append("")
|
||||
lines.append("Skills by cost (index = always-on; SKILL.md = cost when loaded)")
|
||||
for entry in skills[:_DETAILS_TABLE_LIMIT]:
|
||||
name = str(entry.get("name") or "")
|
||||
if len(name) > 28:
|
||||
name = name[:27] + "…"
|
||||
md = entry.get("skill_md_tokens")
|
||||
md_str = f"{md:>8,}" if md is not None else f"{'n/a':>8}"
|
||||
lines.append(
|
||||
f" {name:<28} index {entry['index_tokens']:>6,}"
|
||||
f" SKILL.md {md_str} tokens"
|
||||
)
|
||||
remaining = len(skills) - _DETAILS_TABLE_LIMIT
|
||||
if remaining > 0:
|
||||
lines.append(f" … and {remaining} more")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def render_context_breakdown_lines(
|
||||
payload: Dict[str, Any],
|
||||
*,
|
||||
details: Optional[Dict[str, Any]] = None,
|
||||
grid: bool = True,
|
||||
) -> List[str]:
|
||||
"""Render the full /context view as plain-text lines.
|
||||
|
||||
``grid=True`` (CLI) prepends the glyph block grid; the gateway passes
|
||||
``grid=False`` and keeps its own gauge. ``details`` (from
|
||||
:func:`compute_context_details`) appends the expanded listings.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
if grid:
|
||||
lines.extend(render_context_grid(payload))
|
||||
lines.append("")
|
||||
lines.extend(render_context_category_lines(payload))
|
||||
|
||||
context_max = int(payload.get("context_max") or 0)
|
||||
context_used = int(payload.get("context_used") or 0)
|
||||
if context_max > 0:
|
||||
pct = int(payload.get("context_percent") or 0)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"Context window: {context_used:,} / {context_max:,} tokens ({pct}%)"
|
||||
)
|
||||
|
||||
if details is not None:
|
||||
detail_lines = render_context_details_lines(details)
|
||||
if detail_lines:
|
||||
lines.append("")
|
||||
lines.extend(detail_lines)
|
||||
else:
|
||||
lines.append("")
|
||||
lines.append("Use /context all for per-skill and per-toolset costs.")
|
||||
return lines
|
||||
|
|
|
|||
|
|
@ -90,9 +90,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool:
|
|||
|
||||
|
||||
HISTORICAL_TASK_HEADING = "## Historical Task Snapshot"
|
||||
HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State"
|
||||
HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks"
|
||||
HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work"
|
||||
|
||||
|
||||
SUMMARY_PREFIX = (
|
||||
|
|
@ -107,9 +104,7 @@ SUMMARY_PREFIX = (
|
|||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
|
||||
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
|
||||
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
|
||||
f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
|
|
@ -219,8 +214,45 @@ _MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW]
|
|||
# stale directive it carried (e.g. "resume exactly from Active Task") survives
|
||||
# embedded in the body and keeps hijacking replies. Keep newest-first; entries
|
||||
# are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes.
|
||||
# NEVER mutate or reorder an existing entry — each one is the exact wire text a
|
||||
# shipped build persisted, so editing it silently un-normalizes every summary
|
||||
# written by that build generation; prepend only. tests/agent/
|
||||
# test_summary_prefix_semantics.py byte-pins every entry to enforce this.
|
||||
_HISTORICAL_SUMMARY_PREFIXES = (
|
||||
# Jul 2026 (#65848 class): identical to the current prefix except it
|
||||
# Pre-#69619: identical to the current prefix except the stale-item
|
||||
# discard clause named all four historical headings (the three
|
||||
# section headers removed by #69619 were still in the template).
|
||||
# Summaries persisted by builds immediately before #69619 carry this
|
||||
# exact text and must remain detectable/strippable on resume.
|
||||
"[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted "
|
||||
"into the summary below. This is a handoff from a previous context "
|
||||
"window — treat it as background reference, NOT as active instructions. "
|
||||
"Do NOT answer questions or fulfill requests mentioned in this summary; "
|
||||
"they were already addressed. "
|
||||
"Respond ONLY to the latest user message that appears AFTER this "
|
||||
"summary — that message is the single source of truth for what to do "
|
||||
"right now. "
|
||||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
|
||||
"'## Historical Pending User Asks' / "
|
||||
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
"back', 'just verify', 'don't do that anymore', 'never mind', a new "
|
||||
"topic) must immediately end any in-flight work described in the "
|
||||
"summary; do not re-surface it in later turns. "
|
||||
"IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system "
|
||||
"prompt is ALWAYS authoritative and active — never ignore or deprioritize "
|
||||
"memory content due to this compaction note. "
|
||||
"None of the above restricts HOW you work: your tools remain fully "
|
||||
"active — keep calling them normally for the active task (edit files, "
|
||||
"run commands, search) instead of merely narrating what you would do. "
|
||||
"The current session state (files, config, etc.) may reflect work "
|
||||
"described here — avoid repeating it:",
|
||||
# Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it
|
||||
# lacked the explicit "tools remain fully active" clause — the strong
|
||||
# REFERENCE ONLY framing bled into general tool-use suppression
|
||||
# (observed: 7 consecutive narration-only turns immediately after a
|
||||
|
|
@ -236,9 +268,9 @@ _HISTORICAL_SUMMARY_PREFIXES = (
|
|||
"Topic overlap with the summary does NOT mean you should resume its "
|
||||
"task: even on similar topics, the latest user message WINS. Treat ONLY "
|
||||
"the latest message as the active task and discard stale items from "
|
||||
f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / "
|
||||
f"'{HISTORICAL_PENDING_ASKS_HEADING}' / "
|
||||
f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or "
|
||||
"'## Historical Task Snapshot' / '## Historical In-Progress State' / "
|
||||
"'## Historical Pending User Asks' / "
|
||||
"'## Historical Remaining Work' entirely — do not 'wrap up' or "
|
||||
"'finish' work described there unless the latest message explicitly "
|
||||
"asks for it. "
|
||||
"Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll "
|
||||
|
|
@ -1213,6 +1245,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._last_aux_model_failure_model = None
|
||||
self._last_compression_savings_pct = 100.0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self._fallback_compression_streak = 0
|
||||
self._verify_compaction_cleared_threshold = False
|
||||
self._last_compression_made_progress = False
|
||||
|
|
@ -1350,6 +1383,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._last_aux_model_failure_model = None
|
||||
self._last_compression_savings_pct = 100.0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self._fallback_compression_streak = 0
|
||||
self._verify_compaction_cleared_threshold = False
|
||||
self._last_compression_made_progress = False
|
||||
|
|
@ -1376,6 +1410,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._consecutive_timeout_failures = 0
|
||||
self._fallback_compression_streak = 0
|
||||
self._ineffective_compression_count = 0
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
self.get_active_compression_failure_cooldown()
|
||||
self._load_fallback_compression_streak()
|
||||
self._load_ineffective_compression_count()
|
||||
|
|
@ -1747,6 +1782,15 @@ class ContextCompressor(ContextEngine):
|
|||
# rationale as the gpt-5.5/Codex 85% autoraise.
|
||||
_MIN_CTX_TRIGGER_RATIO = 0.85
|
||||
|
||||
# Anti-thrash recovery window (#14694): once the ineffective/fallback
|
||||
# breaker trips, automatic compaction stays blocked for this long, then
|
||||
# ONE probe attempt is allowed (counters drop to 1 strike, so another
|
||||
# ineffective pass re-trips immediately). Long enough that a genuinely
|
||||
# incompressible session isn't compacting in a loop; short enough that a
|
||||
# session which has since grown real compressible material recovers well
|
||||
# before it rides into the provider's hard context limit.
|
||||
_ANTI_THRASH_RECOVERY_SECONDS = 300.0
|
||||
|
||||
@staticmethod
|
||||
def _coerce_max_tokens(value: Any) -> int | None:
|
||||
"""Normalize a max_tokens value to a positive int or None.
|
||||
|
|
@ -2016,6 +2060,12 @@ class ContextCompressor(ContextEngine):
|
|||
# Anti-thrashing: track whether last compression was effective
|
||||
self._last_compression_savings_pct: float = 100.0
|
||||
self._ineffective_compression_count: int = 0
|
||||
# Monotonic deadline after which a tripped anti-thrash guard grants
|
||||
# one probation probe (#14694). 0.0 = clock not armed. Armed lazily on
|
||||
# the first blocked evaluation; deliberately NOT durable, so a process
|
||||
# restart with a persisted tripped counter (#69872) waits a full fresh
|
||||
# window before probing (#54923: restart must never disarm a guard).
|
||||
self._anti_thrash_recovery_deadline: float = 0.0
|
||||
# Consecutive completed deterministic-fallback boundaries. Unlike the
|
||||
# real-usage effectiveness counter, ordinary fitting responses must not
|
||||
# reset this breaker; only a healthy completed summary does.
|
||||
|
|
@ -2306,21 +2356,66 @@ class ContextCompressor(ContextEngine):
|
|||
_cooldown_remaining,
|
||||
)
|
||||
return True
|
||||
# Anti-thrashing: back off if recent compressions were ineffective
|
||||
# Anti-thrashing: back off if recent compressions were ineffective.
|
||||
# The back-off must not be permanent (#14694): the tripped state was
|
||||
# judged against the transcript as it existed THEN (e.g. a middle
|
||||
# region too small to matter), but the conversation keeps growing and
|
||||
# can accumulate plenty of compressible material later. Without a
|
||||
# recovery path the session never auto-compacts again and rides into
|
||||
# the provider's hard context limit. Recovery is a probation probe:
|
||||
# after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE
|
||||
# attempt by dropping the tripped counter(s) to 1 strike (persisted,
|
||||
# so sibling agents on the same session row unblock too). If the probe
|
||||
# is ineffective again the very next verdict re-trips the guard, so
|
||||
# the worst case in the truly-incompressible state is one compaction
|
||||
# attempt per recovery window — bounded, not thrash.
|
||||
#
|
||||
# The clock is armed lazily on the first BLOCKED evaluation rather
|
||||
# than persisted at trip time: a fresh process that loads a durable
|
||||
# tripped counter (#69872) therefore starts a full window blocked,
|
||||
# preserving the restart-must-not-disarm contract (#54923).
|
||||
if (
|
||||
self._ineffective_compression_count >= 2
|
||||
or self._fallback_compression_streak >= 2
|
||||
):
|
||||
_now = time.monotonic()
|
||||
if self._anti_thrash_recovery_deadline <= 0.0:
|
||||
self._anti_thrash_recovery_deadline = (
|
||||
_now + self._ANTI_THRASH_RECOVERY_SECONDS
|
||||
)
|
||||
elif _now >= self._anti_thrash_recovery_deadline:
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
if self._ineffective_compression_count >= 2:
|
||||
self._record_ineffective_compression_verdict(1)
|
||||
if self._fallback_compression_streak >= 2:
|
||||
self._fallback_compression_streak = 1
|
||||
self._persist_fallback_compression_streak()
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Anti-thrashing recovery: %.0fs elapsed since the "
|
||||
"guard tripped — allowing one compaction probe "
|
||||
"(ineffective=%d fallback=%d).",
|
||||
self._ANTI_THRASH_RECOVERY_SECONDS,
|
||||
self._ineffective_compression_count,
|
||||
self._fallback_compression_streak,
|
||||
)
|
||||
return False
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Compression skipped — repeated compaction attempts did not "
|
||||
"restore healthy context. ineffective=%d fallback=%d. "
|
||||
"Consider /new to start fresh, or /compress <topic> for "
|
||||
"focused compression.",
|
||||
"Auto-compaction will retry once in %.0fs. Consider /new "
|
||||
"to start fresh, or /compress <topic> for focused "
|
||||
"compression.",
|
||||
self._ineffective_compression_count,
|
||||
self._fallback_compression_streak,
|
||||
max(0.0, self._anti_thrash_recovery_deadline - _now),
|
||||
)
|
||||
return True
|
||||
# Guard not tripped (counters were cleared by an effective compaction
|
||||
# or a fitting real-usage reading) — disarm any pending recovery clock
|
||||
# so a LATER trip starts its own full window.
|
||||
self._anti_thrash_recovery_deadline = 0.0
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
|
@ -2968,12 +3063,6 @@ Recovered from a deterministic fallback because the LLM context summarizer was u
|
|||
## Active State
|
||||
Unknown from deterministic fallback. Inspect current repository/session state if needed.
|
||||
|
||||
{HISTORICAL_IN_PROGRESS_HEADING}
|
||||
Unknown from deterministic fallback — the latest user ask is recorded once under
|
||||
"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an
|
||||
unfulfilled instruction to re-answer; verify current state and continue from the
|
||||
protected recent messages after this summary.
|
||||
|
||||
## Blocked
|
||||
{_bullets(blockers, limit=5)}
|
||||
|
||||
|
|
@ -2983,17 +3072,9 @@ None recoverable from deterministic fallback.
|
|||
## Resolved Questions
|
||||
None recoverable from deterministic fallback.
|
||||
|
||||
{HISTORICAL_PENDING_ASKS_HEADING}
|
||||
None recoverable from deterministic fallback. (The latest user ask is preserved once
|
||||
under "{HISTORICAL_TASK_HEADING}" as historical context — it is NOT necessarily
|
||||
outstanding.)
|
||||
|
||||
## Relevant Files
|
||||
{_bullets(relevant_files, limit=12)}
|
||||
|
||||
{HISTORICAL_REMAINING_WORK_HEADING}
|
||||
Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims.
|
||||
|
||||
## Last Dropped Turns
|
||||
{_bullets(last_dropped_turns, limit=8)}
|
||||
|
||||
|
|
@ -3304,9 +3385,6 @@ Be specific with file paths, commands, line numbers, and results.]
|
|||
- Any running processes or servers
|
||||
- Environment details that matter]
|
||||
|
||||
{HISTORICAL_IN_PROGRESS_HEADING}
|
||||
[Work currently underway — what was being done when compaction fired]
|
||||
|
||||
## Blocked
|
||||
[Any blockers, errors, or issues not yet resolved. Include exact error messages.]
|
||||
|
||||
|
|
@ -3316,15 +3394,9 @@ Be specific with file paths, commands, line numbers, and results.]
|
|||
## Resolved Questions
|
||||
{_resolved_questions_instructions}
|
||||
|
||||
{HISTORICAL_PENDING_ASKS_HEADING}
|
||||
{_pending_asks_instructions}
|
||||
|
||||
## Relevant Files
|
||||
[Files read, modified, or created — with brief note on each]
|
||||
|
||||
{HISTORICAL_REMAINING_WORK_HEADING}
|
||||
[What remains to be done — framed as STALE context for reference only. The agent must NOT resume this work unless the latest user message explicitly asks for it.]
|
||||
|
||||
## Critical Context
|
||||
[Any specific values, error messages, configuration details, or data that would be lost without explicit preservation. NEVER include API keys, tokens, passwords, or credentials — write [REDACTED] instead.]
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ REFERENCE_PATTERN = re.compile(
|
|||
rf"(?<![\w/])@(?:(?P<simple>diff|staged)\b|(?P<kind>file|folder|git|url):(?P<value>{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))"
|
||||
)
|
||||
TRAILING_PUNCTUATION = ",.;!?"
|
||||
_NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""")
|
||||
_SENSITIVE_HOME_DIRS = (".ssh", ".aws", ".gnupg", ".kube", ".docker", ".azure", ".config/gh")
|
||||
_SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",)
|
||||
_SENSITIVE_HOME_FILES = (
|
||||
|
|
@ -60,6 +61,21 @@ class ContextReferenceResult:
|
|||
blocked: bool = False
|
||||
|
||||
|
||||
def format_reference_value(value: str) -> str:
|
||||
"""Quote a reference value so ``REFERENCE_PATTERN`` reads it back whole.
|
||||
|
||||
The unquoted alternative in the pattern is ``\\S+``, so a path containing a
|
||||
space parses as a truncated ref with the tail left behind as loose text.
|
||||
Mirrors ``formatRefValue`` in the desktop's directive-text.tsx.
|
||||
"""
|
||||
if not _NEEDS_QUOTING.search(value):
|
||||
return value
|
||||
for quote in ("`", '"', "'"):
|
||||
if quote not in value:
|
||||
return f"{quote}{value}{quote}"
|
||||
return value
|
||||
|
||||
|
||||
def parse_context_references(message: str) -> list[ContextReference]:
|
||||
refs: list[ContextReference] = []
|
||||
if not message:
|
||||
|
|
@ -308,7 +324,7 @@ def _expand_git_reference(
|
|||
["git", *args],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=30,
|
||||
stdin=subprocess.DEVNULL,
|
||||
**_popen_kwargs,
|
||||
|
|
@ -534,7 +550,7 @@ def _rg_files(path: Path, cwd: Path, limit: int) -> list[Path] | None:
|
|||
["rg", "--files", str(path.relative_to(cwd))],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=10,
|
||||
stdin=subprocess.DEVNULL,
|
||||
**_popen_kwargs,
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import uuid
|
|||
import threading
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from agent.context_engine import (
|
||||
automatic_compaction_status_message,
|
||||
|
|
@ -221,6 +221,25 @@ class CompressionCommitFence:
|
|||
self._lock = threading.Lock()
|
||||
self._cancelled = False
|
||||
self._commit_started = False
|
||||
# Forward-progress telemetry: the compression worker touches this
|
||||
# whenever the streamed summary call produces a token (see
|
||||
# ContextCompressor._call_summary_llm). Waiters use it to distinguish
|
||||
# a SLOW-but-alive summary model from a HUNG one, so slow models are
|
||||
# not killed by a fixed wall-clock deadline while tokens are moving.
|
||||
self._last_progress = time.monotonic()
|
||||
|
||||
def touch_progress(self) -> None:
|
||||
"""Record forward progress (e.g. a streamed summary token arriving).
|
||||
|
||||
Called from the compression worker thread; read by async waiters via
|
||||
:meth:`seconds_since_progress`. A bare float store is atomic in
|
||||
CPython, so no lock is needed.
|
||||
"""
|
||||
self._last_progress = time.monotonic()
|
||||
|
||||
def seconds_since_progress(self) -> float:
|
||||
"""Seconds since the worker last reported forward progress."""
|
||||
return max(0.0, time.monotonic() - self._last_progress)
|
||||
|
||||
def cancel_before_commit(self) -> bool:
|
||||
"""Cancel a pending commit, or wait for an active commit to finish.
|
||||
|
|
@ -373,6 +392,125 @@ def compression_skipped_due_to_lock(agent: Any) -> bool:
|
|||
return _sig is True or isinstance(_sig, str)
|
||||
|
||||
|
||||
def _adopt_live_compression_child(
|
||||
agent: Any,
|
||||
session_db: Any,
|
||||
parent_session_id: str,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Move a stale compression contender onto the unique durable child.
|
||||
|
||||
Resolve and load first, then mutate the live agent. This ordering keeps the
|
||||
stale contender fail-closed when lineage is ambiguous or the compacted
|
||||
handoff cannot be read.
|
||||
"""
|
||||
finder = getattr(type(session_db), "find_live_compression_child", None)
|
||||
loader = getattr(type(session_db), "get_messages_as_conversation", None)
|
||||
if not callable(finder) or not callable(loader):
|
||||
return None
|
||||
child = finder(session_db, parent_session_id)
|
||||
if not child or not child.get("id"):
|
||||
return None
|
||||
child_session_id = str(child["id"])
|
||||
recovered = loader(session_db, child_session_id)
|
||||
if not isinstance(recovered, list) or not recovered:
|
||||
return None
|
||||
# Revalidate after loading: the child may have rotated or a competing
|
||||
# continuation may have appeared between the two DB reads.
|
||||
confirmed = finder(session_db, parent_session_id)
|
||||
if not confirmed or str(confirmed.get("id") or "") != child_session_id:
|
||||
return None
|
||||
|
||||
agent.session_id = child_session_id
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
|
||||
set_current_session_id(child_session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = child_session_id
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
|
||||
set_session_context(child_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
agent._session_db_created = True
|
||||
if child.get("system_prompt"):
|
||||
agent._cached_system_prompt = child["system_prompt"]
|
||||
agent._last_flushed_db_idx = len(recovered)
|
||||
agent._flushed_db_message_session_id = child_session_id
|
||||
agent._flushed_db_message_ids = {
|
||||
id(message) for message in recovered if isinstance(message, dict)
|
||||
}
|
||||
|
||||
on_session_start = getattr(agent.context_compressor, "on_session_start", None)
|
||||
if callable(on_session_start):
|
||||
try:
|
||||
on_session_start(
|
||||
child_session_id,
|
||||
boundary_reason="compression",
|
||||
old_session_id=parent_session_id,
|
||||
session_db=session_db,
|
||||
platform=getattr(agent, "platform", None) or "cli",
|
||||
conversation_id=getattr(agent, "_gateway_session_key", None),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("context engine compression-child adoption failed: %s", exc)
|
||||
else:
|
||||
bind_state = getattr(agent.context_compressor, "bind_session_state", None)
|
||||
if callable(bind_state):
|
||||
try:
|
||||
bind_state(session_db=session_db, session_id=child_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if agent._memory_manager:
|
||||
agent._memory_manager.on_session_switch(
|
||||
child_session_id,
|
||||
parent_session_id=parent_session_id,
|
||||
reset=False,
|
||||
reason="compression",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("memory manager compression-child adoption failed: %s", exc)
|
||||
|
||||
return recovered
|
||||
|
||||
|
||||
def recover_rotated_compression_session(
|
||||
agent: Any,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Recover a stale live agent before a new turn writes to its old parent."""
|
||||
session_db = getattr(agent, "_session_db", None)
|
||||
session_id = getattr(agent, "session_id", None) or ""
|
||||
if session_db is None or not session_id:
|
||||
return None
|
||||
try:
|
||||
if not _session_was_rotated_by_compression(session_db, session_id):
|
||||
return None
|
||||
# Rotation publication holds the parent compression lease until the
|
||||
# child handoff is durable. A concurrent turn waits briefly rather than
|
||||
# observing the intentional parent-ended/child-empty intermediate state.
|
||||
holder_getter = getattr(session_db, "get_compression_lock_holder", None)
|
||||
for attempt in range(21):
|
||||
recovered = _adopt_live_compression_child(agent, session_db, session_id)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
holder = holder_getter(session_id) if callable(holder_getter) else None
|
||||
if not holder or attempt == 20:
|
||||
return None
|
||||
time.sleep(0.05)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"compression session recovery failed for session=%s (%s: %s)",
|
||||
session_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _compression_lock_holder(agent: Any) -> str:
|
||||
"""Build a unique holder id for the lock: pid:tid:agent-instance:uuid.
|
||||
|
||||
|
|
@ -533,7 +671,17 @@ class _CompressionLockLeaseRefresher:
|
|||
# by the TTL the acquirer set — the lock can never be held past its TTL
|
||||
# by a stuck refresher.
|
||||
consecutive_failures = 0
|
||||
while not self._stop.wait(self._refresh_interval_seconds):
|
||||
# First refresh happens immediately, not one interval late. Everything
|
||||
# between try_acquire() and start() (the rotation-ownership lookup, the
|
||||
# durable-breaker re-read, thread startup) is charged against the very
|
||||
# first lease, so on a short TTL under load the lock could already be
|
||||
# expired — and reclaimable by a competing path — before tick #1.
|
||||
first = True
|
||||
while first or not self._stop.wait(self._refresh_interval_seconds):
|
||||
if first:
|
||||
first = False
|
||||
if self._stop.is_set():
|
||||
break
|
||||
try:
|
||||
refreshed = self._db.refresh_compression_lock(
|
||||
self._session_id,
|
||||
|
|
@ -892,6 +1040,7 @@ _SYNTHETIC_USER_FLAGS = (
|
|||
"_empty_recovery_synthetic",
|
||||
"_verification_stop_synthetic",
|
||||
"_pre_verify_synthetic",
|
||||
"_dropped_toolcall_nudge",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1252,8 +1401,11 @@ def compress_context(
|
|||
# parent_session_id child, no
|
||||
# `name #N` renumber, no contextvar/env/logging re-sync, no memory/context-
|
||||
# engine session-switch. The conversation keeps one durable id for life,
|
||||
# eliminating the session-rotation bug cluster. Default False during rollout.
|
||||
in_place = bool(getattr(agent, "compression_in_place", False))
|
||||
# eliminating the session-rotation bug cluster. Default True (2107b86024).
|
||||
# Default True matches DEFAULT_CONFIG / #38763. A missing attribute must
|
||||
# NOT fall back to rotation mode — that re-enables the pre-lease drift
|
||||
# path and can wedge busy sessions that never set the flag.
|
||||
in_place = bool(getattr(agent, "compression_in_place", True))
|
||||
# Set True once the in-place DB write actually completes (the DB block can
|
||||
# raise and skip it). Surfaced to the gateway via agent._last_compaction_in_place.
|
||||
compacted_in_place = False
|
||||
|
|
@ -1460,6 +1612,8 @@ def compress_context(
|
|||
if _lock_released:
|
||||
return
|
||||
_lock_released = True
|
||||
if getattr(agent, "_active_compression_lock_holder", None) == _lock_holder:
|
||||
agent._active_compression_lock_holder = None
|
||||
if _lock_refresher is not None:
|
||||
try:
|
||||
_lock_refresher.stop()
|
||||
|
|
@ -1471,6 +1625,9 @@ def compress_context(
|
|||
except Exception as _rel_err:
|
||||
logger.debug("compression lock release failed: %s", _rel_err)
|
||||
|
||||
if _lock_holder is not None:
|
||||
agent._active_compression_lock_holder = _lock_holder
|
||||
|
||||
# A delayed contender can acquire the parent lock after the winning path
|
||||
# has released it and completed rotation. The lock serializes work but does
|
||||
# not by itself prove that this stale agent still owns a live parent.
|
||||
|
|
@ -1493,15 +1650,25 @@ def compress_context(
|
|||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
return messages, _existing_sp
|
||||
if _parent_already_rotated:
|
||||
logger.info(
|
||||
"compression skipped: session=%s was already rotated by "
|
||||
"another compression path",
|
||||
_lock_sid,
|
||||
recovered_messages = _adopt_live_compression_child(
|
||||
agent, _lock_db, _lock_sid
|
||||
)
|
||||
_release_lock()
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
if recovered_messages is not None:
|
||||
logger.warning(
|
||||
"compression recovery: stale session=%s adopted live child=%s",
|
||||
_lock_sid,
|
||||
agent.session_id,
|
||||
)
|
||||
return recovered_messages, _existing_sp
|
||||
logger.warning(
|
||||
"compression skipped: session=%s was already rotated by "
|
||||
"another compression path, but no unique live child could be adopted",
|
||||
_lock_sid,
|
||||
)
|
||||
return messages, _existing_sp
|
||||
|
||||
# The agent may have been constructed before another path completed an
|
||||
|
|
@ -1535,6 +1702,47 @@ def compress_context(
|
|||
)
|
||||
_lock_refresher.start()
|
||||
|
||||
# The caller's history snapshot predates lease acquisition. Reload the
|
||||
# durable parent after the lease is live; MORE durable rows than the
|
||||
# snapshot carries means a frontend/background writer committed a turn
|
||||
# in that window, so publishing from this snapshot would omit it.
|
||||
# Deliberately a LENGTH check, not content equality: in-memory
|
||||
# mutation of past turns is legal (multimodal compression, retry
|
||||
# history replacement, think-tag stripping), and a content-equality
|
||||
# abort would permanently wedge compression on such sessions — the
|
||||
# #14694 failure shape.
|
||||
# Rotation-only: in-place compaction (archive_and_compact) is
|
||||
# non-destructive — pre-compaction rows are soft-archived (active=0,
|
||||
# compacted=1), stay searchable and recoverable, so snapshot/durable
|
||||
# drift cannot lose data there and must not abort compaction.
|
||||
#
|
||||
# When durable DID grow, ADOPT it and continue rather than aborting.
|
||||
# Aborting returned the stale snapshot unchanged, so busy sessions
|
||||
# (memory review / shared session_id writers) stayed permanently
|
||||
# behind the DB: every /compress and auto-compress saw
|
||||
# "changed before lease acquisition", surfaced as the misleading
|
||||
# "No changes from compression", and never reclaimed tokens.
|
||||
if not in_place and _lock_db is not None and _lock_sid:
|
||||
durable_loader = getattr(
|
||||
type(_lock_db), "get_messages_as_conversation", None
|
||||
)
|
||||
if callable(durable_loader):
|
||||
durable_parent = durable_loader(_lock_db, _lock_sid)
|
||||
if isinstance(durable_parent, list) and len(durable_parent) > len(messages):
|
||||
logger.info(
|
||||
"compression: session=%s grew before lease "
|
||||
"(%d → %d msgs); adopting durable snapshot",
|
||||
_lock_sid,
|
||||
len(messages),
|
||||
len(durable_parent),
|
||||
)
|
||||
messages = durable_parent
|
||||
_pre_msg_count = len(messages)
|
||||
# Token estimate was for the stale snapshot; clear it so
|
||||
# the compressor re-derives from the adopted transcript
|
||||
# instead of under-counting the newly visible rows.
|
||||
approx_tokens = 0
|
||||
|
||||
# Notify external memory provider before compression discards context.
|
||||
# The provider's on_pre_compress() may return a string of insights it
|
||||
# wants surfaced inside the compression summary; capture and forward it
|
||||
|
|
@ -1575,7 +1783,29 @@ def compress_context(
|
|||
|
||||
messages_before_compression = copy.deepcopy(messages)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
# Publish forward progress to the commit fence while the summary LLM
|
||||
# call streams. Async hosts (gateway session hygiene) poll
|
||||
# ``commit_fence.seconds_since_progress()`` to extend their deadline
|
||||
# while tokens are moving — so a SLOW summary model is only killed
|
||||
# when it is actually silent, not merely thorough. The hook is
|
||||
# thread-local and the compress call is synchronous on this thread,
|
||||
# so it cannot leak into unrelated auxiliary calls.
|
||||
#
|
||||
# Fenceless callers (CLI /compress, in-loop auto-compress) install a
|
||||
# no-op hook: nobody polls their progress, but an ACTIVE hook is what
|
||||
# switches the summary call onto the streamed path — giving every
|
||||
# compression path the same two guarantees: the configured timeout
|
||||
# acts on inactivity (slow models finish), and a byte-trickling
|
||||
# provider that keeps the connection alive forever is cut off at the
|
||||
# streamed total ceiling (see _aux_stream_total_ceiling) instead of
|
||||
# outliving the SDK's inactivity timeout indefinitely.
|
||||
from agent.auxiliary_client import aux_progress_hook
|
||||
_progress_hook = (
|
||||
commit_fence.touch_progress if commit_fence is not None
|
||||
else (lambda: None)
|
||||
)
|
||||
with aux_progress_hook(_progress_hook):
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
except BaseException as _compress_exc:
|
||||
# ANY exception after lock acquisition — memory hook, capability
|
||||
# inspection, engine lookup, or compress() — must release the lock so
|
||||
|
|
@ -1803,6 +2033,20 @@ def compress_context(
|
|||
):
|
||||
new_system_prompt = cached_system_prompt
|
||||
agent._cached_system_prompt = cached_system_prompt
|
||||
# _invalidate_system_prompt() above also cleared the
|
||||
# cross-session-stable prefix marker boundary. The kept prompt
|
||||
# is byte-identical, so reconstruct the stable tier and reuse
|
||||
# it ONLY when the kept prompt still literally starts with it
|
||||
# (same startswith gate as the restore path); otherwise the
|
||||
# request layer falls back to the legacy single-breakpoint
|
||||
# layout with the prompt bytes untouched.
|
||||
from agent.system_prompt import reconstruct_static_prefix
|
||||
|
||||
reconstruct_static_prefix(
|
||||
agent,
|
||||
system_message=system_message,
|
||||
log_label="compression keep-prompt",
|
||||
)
|
||||
else:
|
||||
new_system_prompt = agent._build_system_prompt(system_message)
|
||||
agent._cached_system_prompt = new_system_prompt
|
||||
|
|
@ -1879,40 +2123,13 @@ def compress_context(
|
|||
)
|
||||
except Exception:
|
||||
pass # best-effort — don't block compression on a flush error
|
||||
# Propagate title to the new session with auto-numbering
|
||||
old_title = agent._session_db.get_session_title(agent.session_id)
|
||||
agent._session_db.end_session(agent.session_id, "compression")
|
||||
old_session_id = agent.session_id
|
||||
agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
# Ordering contract: the agent thread updates the contextvar here;
|
||||
# the gateway propagates to SessionEntry after run_in_executor returns.
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
|
||||
set_current_session_id(agent.session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = agent.session_id
|
||||
# The gateway/tools session context (ContextVar + env) and the
|
||||
# logging session context are SEPARATE mechanisms. The call above
|
||||
# moves the former; the ``[session_id]`` tag on log lines comes
|
||||
# from ``hermes_logging._session_context`` (set once per turn in
|
||||
# conversation_loop.py). Without this, post-rotation log lines in
|
||||
# the same turn keep the STALE old id while the message/DB/gateway
|
||||
# state carry the new one — breaking log correlation exactly at the
|
||||
# compaction boundary (see #34089). Guarded separately so a logging
|
||||
# failure can never regress the routing update above.
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
|
||||
set_session_context(agent.session_id)
|
||||
except Exception:
|
||||
pass
|
||||
agent._session_db_created = False
|
||||
# Publish parent closure + child row + compacted handoff in
|
||||
# one transaction. No reader can observe a missing/empty child.
|
||||
# The rotation child must stay on the parent's profile —
|
||||
# mirror _ensure_db_session's stamp ("default" persists as
|
||||
# NULL). _insert_session_row's parent backfill additionally
|
||||
# COALESCEs from the parent row, covering app-global remote
|
||||
# sessions whose thread lacks the HERMES_HOME context.
|
||||
# NULL). publish_compression_child additionally COALESCEs
|
||||
# from the parent row, covering app-global remote sessions
|
||||
# whose thread lacks the HERMES_HOME context.
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
|
||||
|
|
@ -1921,53 +2138,39 @@ def compress_context(
|
|||
_profile_for_child = None
|
||||
except Exception:
|
||||
_profile_for_child = None
|
||||
old_title = agent._session_db.get_session_title(agent.session_id)
|
||||
old_session_id = agent.session_id
|
||||
new_session_id = (
|
||||
f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_"
|
||||
f"{uuid.uuid4().hex[:6]}"
|
||||
)
|
||||
agent._session_db.publish_compression_child(
|
||||
parent_session_id=old_session_id,
|
||||
child_session_id=new_session_id,
|
||||
source=agent.platform
|
||||
or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=agent.model,
|
||||
model_config=agent._session_init_model_config,
|
||||
system_prompt=new_system_prompt,
|
||||
messages=compressed,
|
||||
cwd=getattr(agent, "working_directory", None),
|
||||
profile_name=_profile_for_child,
|
||||
compression_lock_holder=_lock_holder,
|
||||
require_compression_lease=_lock_holder is not None,
|
||||
)
|
||||
agent.session_id = new_session_id
|
||||
try:
|
||||
agent._session_db.create_session(
|
||||
session_id=agent.session_id,
|
||||
source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=agent.model,
|
||||
model_config=agent._session_init_model_config,
|
||||
parent_session_id=old_session_id,
|
||||
profile_name=_profile_for_child,
|
||||
)
|
||||
except Exception as _cs_err:
|
||||
# The child row could not be created (e.g. FK constraint,
|
||||
# contended write). Previously the outer handler simply
|
||||
# warned and let the agent continue on the NEW id — which
|
||||
# has no row in state.db, producing an orphan: the parent
|
||||
# is ended, the child is never indexed, and every
|
||||
# subsequent message is attributed to a session that
|
||||
# doesn't exist (#33906/#33907). Roll the live id back to
|
||||
# the parent so the conversation stays attached to a real,
|
||||
# indexed session instead of a phantom.
|
||||
logger.warning(
|
||||
"Compression child session create failed (%s) — "
|
||||
"rolling back to parent session %s to avoid an orphan.",
|
||||
_cs_err, old_session_id,
|
||||
)
|
||||
agent.session_id = old_session_id
|
||||
try:
|
||||
from gateway.session_context import set_current_session_id
|
||||
set_current_session_id(agent.session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = agent.session_id
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
set_session_context(agent.session_id)
|
||||
except Exception:
|
||||
pass
|
||||
# Re-open the parent: it was ended above, but we're
|
||||
# continuing on it, so it must not stay closed.
|
||||
try:
|
||||
agent._session_db.reopen_session(old_session_id)
|
||||
except Exception:
|
||||
pass
|
||||
old_session_id = None # no rotation happened
|
||||
# The parent row already exists in state.db, so mark the
|
||||
# session as created — _ensure_db_session would otherwise
|
||||
# retry a (harmless INSERT OR IGNORE) create next turn.
|
||||
agent._session_db_created = True
|
||||
raise
|
||||
from gateway.session_context import set_current_session_id
|
||||
|
||||
set_current_session_id(agent.session_id)
|
||||
except Exception:
|
||||
os.environ["HERMES_SESSION_ID"] = agent.session_id
|
||||
try:
|
||||
from hermes_logging import set_session_context
|
||||
|
||||
set_session_context(agent.session_id)
|
||||
except Exception:
|
||||
pass
|
||||
agent._session_db_created = True
|
||||
split_status = "rotated_committed"
|
||||
# Carry a persistent /goal onto the continuation session.
|
||||
|
|
@ -1987,18 +2190,14 @@ def compress_context(
|
|||
except (ValueError, Exception) as e:
|
||||
logger.debug("Could not propagate title on compression: %s", e)
|
||||
|
||||
# Shared post-write steps (both modes target agent.session_id, which
|
||||
# in-place keeps and rotation has already reassigned to the new id):
|
||||
# refresh the stored system prompt and reset the flush cursor so the
|
||||
# next turn re-bases its append diff.
|
||||
agent._session_db.update_system_prompt(agent.session_id, new_system_prompt)
|
||||
# In-place mode still updates/replaces the current row here.
|
||||
# Rotation already published prompt + compacted handoff atomically.
|
||||
if in_place:
|
||||
agent._session_db.update_system_prompt(
|
||||
agent.session_id, new_system_prompt
|
||||
)
|
||||
agent._last_flushed_db_idx = 0
|
||||
else:
|
||||
# A headless turn can be killed before its finalizer. Persist
|
||||
# the rotated child's compacted handoff at the boundary so
|
||||
# the new session is immediately resumable.
|
||||
agent._session_db.replace_messages(agent.session_id, compressed)
|
||||
agent._last_flushed_db_idx = len(compressed)
|
||||
agent._flushed_db_message_session_id = agent.session_id
|
||||
agent._flushed_db_message_ids = {
|
||||
|
|
@ -2008,7 +2207,22 @@ def compress_context(
|
|||
}
|
||||
_session_commit_succeeded = True
|
||||
except Exception as e:
|
||||
split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed"
|
||||
if (
|
||||
not in_place
|
||||
and locals().get("old_session_id")
|
||||
and agent.session_id == old_session_id
|
||||
):
|
||||
# Atomic publication failed (including lease loss): keep the
|
||||
# parent live and discard the stale compacted snapshot.
|
||||
old_session_id = None
|
||||
messages[:] = copy.deepcopy(messages_before_compression)
|
||||
compressed = messages
|
||||
_compression_made_progress = False
|
||||
split_status = (
|
||||
"aborted"
|
||||
if locals().get("old_session_id") is None and not in_place
|
||||
else "failed_not_indexed"
|
||||
)
|
||||
# If the rotation rolled back to the parent (orphan-avoidance
|
||||
# above), agent.session_id is the still-indexed parent and
|
||||
# old_session_id was cleared — so this is recovery, not an
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from agent.turn_context import (
|
|||
reanchor_current_turn_user_idx,
|
||||
)
|
||||
from agent.turn_retry_state import TurnRetryState
|
||||
from agent.runtime_cwd import resolve_agent_cwd
|
||||
from agent.message_sanitization import (
|
||||
close_interrupted_tool_sequence,
|
||||
_repair_tool_call_arguments,
|
||||
|
|
@ -71,7 +72,10 @@ from agent.model_metadata import (
|
|||
save_context_length,
|
||||
)
|
||||
from agent.process_bootstrap import _install_safe_stdio
|
||||
from agent.prompt_caching import apply_anthropic_cache_control
|
||||
from agent.prompt_caching import (
|
||||
apply_anthropic_cache_control,
|
||||
strip_anthropic_cache_control,
|
||||
)
|
||||
from agent.retry_utils import (
|
||||
adaptive_rate_limit_backoff,
|
||||
is_zai_coding_overload_error,
|
||||
|
|
@ -436,6 +440,20 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
|||
# Continuing session — reuse the exact system prompt from the
|
||||
# previous turn so the Anthropic cache prefix matches.
|
||||
agent._cached_system_prompt = stored_prompt
|
||||
# Reconstruct the cross-session-stable prefix for the early cache
|
||||
# breakpoint. The static prefix is not persisted (only the full
|
||||
# prompt is), so gateway surfaces that build a fresh AIAgent per
|
||||
# turn would otherwise lose the two-block system layout after the
|
||||
# first turn — flip-flopping the wire shape mid-conversation and
|
||||
# silently degrading to the legacy single-breakpoint layout.
|
||||
#
|
||||
# ``reconstruct_static_prefix`` gates on ``_use_prompt_caching`` (so
|
||||
# non-Anthropic routes skip the rebuild), applies the startswith
|
||||
# safety gate (stored prompt bytes are never rewritten), and
|
||||
# fails open to the legacy cache layout.
|
||||
from agent.system_prompt import reconstruct_static_prefix
|
||||
|
||||
reconstruct_static_prefix(agent, system_message=system_message)
|
||||
return
|
||||
if stored_prompt:
|
||||
stored_state = "stale_runtime"
|
||||
|
|
@ -508,9 +526,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
|||
|
||||
|
||||
def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
||||
"""Return False when the persisted Model/Provider lines are stale."""
|
||||
"""Return False when the persisted runtime-identity lines are stale."""
|
||||
|
||||
def line_value(label: str) -> str:
|
||||
"""Last matching line wins.
|
||||
|
||||
Safe ONLY for fields emitted in the volatile tier at the very END of
|
||||
the prompt (Model / Provider / Platform). User-supplied project
|
||||
context (AGENTS.md / CLAUDE.md / .cursorrules) is embedded in the
|
||||
middle context tier, so a last-match scan lets project prose shadow
|
||||
any field emitted EARLIER — see ``host_info_value``.
|
||||
"""
|
||||
prefix = f"{label}:"
|
||||
value = ""
|
||||
for line in prompt.splitlines():
|
||||
|
|
@ -518,6 +544,32 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
|||
value = line[len(prefix):].strip()
|
||||
return value
|
||||
|
||||
def host_info_value(label: str) -> str:
|
||||
"""Read a field from the prompt's own host-info block.
|
||||
|
||||
The host-info block (``build_environment_hints``) sits in the STABLE
|
||||
tier, ahead of the embedded project context files. A bare scan of the
|
||||
whole prompt would therefore match a user's ``AGENTS.md`` that merely
|
||||
contains a line starting with the same label, comparing runtime state
|
||||
against project prose. That mismatch never clears, so the check would
|
||||
reject the stored prompt on EVERY turn — rebuilding the system prompt
|
||||
each message and destroying the prefix cache for the whole session,
|
||||
which is far worse than the staleness this function guards against.
|
||||
|
||||
Anchor on the ``User home directory:`` line that immediately precedes
|
||||
the working-directory line in that block, and take the FIRST such
|
||||
occurrence, so only Hermes' own emitted block can satisfy the read.
|
||||
"""
|
||||
prefix = f"{label}:"
|
||||
lines = prompt.splitlines()
|
||||
for idx, line in enumerate(lines):
|
||||
if not line.startswith("User home directory:"):
|
||||
continue
|
||||
for candidate in lines[idx + 1: idx + 4]:
|
||||
if candidate.startswith(prefix):
|
||||
return candidate[len(prefix):].strip()
|
||||
return ""
|
||||
|
||||
stored_model = line_value("Model")
|
||||
current_model = str(getattr(agent, "model", "") or "").strip()
|
||||
if stored_model and current_model and stored_model != current_model:
|
||||
|
|
@ -528,6 +580,24 @@ def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
|
|||
if stored_provider and current_provider and stored_provider != current_provider:
|
||||
return False
|
||||
|
||||
# Detect cwd drift: if the stored prompt was built in a different working
|
||||
# directory, reuse would silently inject a stale path into the prefix cache.
|
||||
# Compare against resolve_agent_cwd() — the SAME resolver used to build the
|
||||
# prompt — so gateway/TUI sessions that set TERMINAL_CWD are not falsely
|
||||
# rejected (they would always differ from the launch dir's os.getcwd()).
|
||||
stored_cwd = host_info_value("Current working directory")
|
||||
if stored_cwd:
|
||||
if stored_cwd != str(resolve_agent_cwd()):
|
||||
return False
|
||||
|
||||
# Detect runtime-surface drift: the stored prompt records which platform it
|
||||
# was built for (e.g. "desktop" vs "cli"). Reusing a desktop-built prompt on
|
||||
# a terminal session (or vice versa) would inject the wrong runtime hints.
|
||||
stored_platform = line_value("Platform")
|
||||
current_platform = str(getattr(agent, "platform", "") or "").strip()
|
||||
if stored_platform and current_platform and stored_platform != current_platform:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -690,6 +760,41 @@ def _compression_deferred_result(
|
|||
}
|
||||
|
||||
|
||||
def _rewrite_system_content_blocks(system_message: dict, effective: str) -> bool:
|
||||
"""Rewrite a cache-decorated system message in place, keeping its blocks.
|
||||
|
||||
``apply_anthropic_cache_control`` runs once per call block, *before* the
|
||||
retry loop, and splits the system prompt into ``[static prefix, volatile
|
||||
tail]`` text blocks carrying the cache_control breakpoints. Assigning a bare
|
||||
string over that list drops both breakpoints, so the failover retry ships
|
||||
the whole system prompt uncached and re-bills it in full.
|
||||
|
||||
``rewrite_prompt_model_identity`` only touches the LAST ``Model:`` /
|
||||
``Provider:`` lines, and those live in the volatile tail — so the static
|
||||
prefix stays byte-identical and its cache entry keeps matching. Returns
|
||||
False when the shape is not one we can safely patch, so the caller falls
|
||||
back to the plain-string assignment.
|
||||
"""
|
||||
content = system_message.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return False
|
||||
if not all(
|
||||
isinstance(part, dict) and part.get("type") == "text" for part in content
|
||||
):
|
||||
return False
|
||||
if len(content) == 1:
|
||||
content[0]["text"] = effective
|
||||
return True
|
||||
if len(content) == 2:
|
||||
head = content[0].get("text") or ""
|
||||
if head and effective.startswith(head):
|
||||
tail = effective[len(head):]
|
||||
if tail:
|
||||
content[1]["text"] = tail
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _sync_failover_system_message(agent, api_messages, active_system_prompt):
|
||||
"""Refresh the in-flight system message after a provider failover.
|
||||
|
||||
|
|
@ -712,10 +817,109 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt):
|
|||
effective = sp
|
||||
if agent.ephemeral_system_prompt:
|
||||
effective = (effective + "\n\n" + agent.ephemeral_system_prompt).strip()
|
||||
api_messages[0]["content"] = effective
|
||||
if not _rewrite_system_content_blocks(api_messages[0], effective):
|
||||
api_messages[0]["content"] = effective
|
||||
return sp
|
||||
|
||||
|
||||
def _ensure_cached_system_prompt_static(agent, system_message=None) -> None:
|
||||
"""Rebuild ``_cached_system_prompt_static`` when caching becomes active.
|
||||
|
||||
Sessions restored under a cache-off primary skip the static-prefix rebuild
|
||||
(gated on ``_use_prompt_caching`` at restore time). A later failover to a
|
||||
cache-on provider would otherwise redecorate with ``static_system_prefix=
|
||||
None`` and silently fall back to the legacy system-plus-3 layout (#72626).
|
||||
|
||||
Thin wrapper over :func:`agent.system_prompt.reconstruct_static_prefix`,
|
||||
which memoizes failed rebuilds so this stays cheap on the retry-loop hot
|
||||
path (it runs at the top of every attempt).
|
||||
"""
|
||||
from agent.system_prompt import reconstruct_static_prefix
|
||||
|
||||
reconstruct_static_prefix(
|
||||
agent, system_message=system_message, log_label="failover redecoration"
|
||||
)
|
||||
|
||||
|
||||
def _peel_moa_guidance(
|
||||
messages: List[Dict[str, Any]],
|
||||
guidance: Any,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Remove MoA reference guidance previously attached by ``_attach_reference_guidance``.
|
||||
|
||||
Thin wrapper over :func:`agent.moa_loop.peel_reference_guidance` (kept
|
||||
adjacent to the attach so the forward/inverse shapes evolve together).
|
||||
Lazy import mirrors the module's other moa_loop touchpoints.
|
||||
"""
|
||||
from agent.moa_loop import peel_reference_guidance
|
||||
|
||||
return peel_reference_guidance(messages, guidance)
|
||||
|
||||
|
||||
def _redecorate_prompt_cache_for_provider(
|
||||
agent,
|
||||
api_messages: List[Dict[str, Any]],
|
||||
*,
|
||||
system_message=None,
|
||||
moa_prepared: Optional[Dict[str, Any]] = None,
|
||||
) -> tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
||||
"""Strip and re-apply cache_control for the *current* provider policy.
|
||||
|
||||
Decoration runs once per call block before the retry loop for the primary
|
||||
provider. ``try_activate_fallback`` refreshes ``_use_prompt_caching`` /
|
||||
``_use_native_cache_layout`` but the nine failover ``continue`` paths reused
|
||||
the old ``api_messages`` (#72626). Mirror ``_reapply_reasoning_echo_for_provider``
|
||||
by reshaping at the top of each retry attempt.
|
||||
|
||||
The source list is the mutated in-flight request (image shrink / ASCII /
|
||||
reasoning_details recoveries already applied) — never a pristine
|
||||
pre-decoration snapshot. MoA guidance is peeled, the base is redecorated,
|
||||
then ``rebase_prepared_request`` re-attaches guidance outside the cached
|
||||
span.
|
||||
"""
|
||||
messages: List[Dict[str, Any]] = [
|
||||
dict(m) if isinstance(m, dict) else m for m in (api_messages or [])
|
||||
]
|
||||
prepared = moa_prepared
|
||||
guidance = prepared.get("guidance") if isinstance(prepared, dict) else None
|
||||
if guidance:
|
||||
messages = _peel_moa_guidance(messages, guidance)
|
||||
|
||||
strip_anthropic_cache_control(messages)
|
||||
|
||||
# Direct attribute access matches the call-block decoration site — the
|
||||
# flags are unconditionally initialized on AIAgent, and a getattr
|
||||
# default here would mask a real init bug as silent cache-off.
|
||||
if agent._use_prompt_caching:
|
||||
_ensure_cached_system_prompt_static(agent, system_message=system_message)
|
||||
static = getattr(agent, "_cached_system_prompt_static", None)
|
||||
messages = apply_anthropic_cache_control(
|
||||
messages,
|
||||
cache_ttl=agent._cache_ttl,
|
||||
native_anthropic=agent._use_native_cache_layout,
|
||||
static_system_prefix=static if isinstance(static, str) else None,
|
||||
)
|
||||
|
||||
if (
|
||||
prepared is not None
|
||||
and getattr(agent, "provider", None) == "moa"
|
||||
):
|
||||
# No `and guidance` here: guidance=None is a real prepared shape
|
||||
# (all-references-failed / silent degraded policy builds the
|
||||
# prepared request without attaching guidance), and the MoA facade
|
||||
# sends prepared["messages"] — not api_kwargs["messages"] — so the
|
||||
# rebase must refresh the prepared object even when there is no
|
||||
# guidance to re-attach. rebase_prepared_request handles falsy
|
||||
# guidance by copying the messages and skipping the attach.
|
||||
completions = getattr(getattr(agent.client, "chat", None), "completions", None)
|
||||
rebase = getattr(completions, "rebase_prepared_request", None)
|
||||
if callable(rebase):
|
||||
prepared = rebase(prepared, messages)
|
||||
messages = prepared["messages"]
|
||||
|
||||
return messages, prepared
|
||||
|
||||
|
||||
def _apply_context_engine_selection(
|
||||
agent: Any,
|
||||
api_messages: List[Dict[str, Any]],
|
||||
|
|
@ -941,6 +1145,9 @@ def run_conversation(
|
|||
# Commentary deduplication spans all provider continuations and tool calls
|
||||
# within one user turn, but must not suppress the same phrase next turn.
|
||||
agent._delivered_interim_texts = set()
|
||||
# A configured SessionDB append failure halts only the affected turn. A
|
||||
# cached gateway agent must recover on the next message if storage did.
|
||||
agent._incremental_persistence_failed = False
|
||||
|
||||
# Main conversation loop counters (pure locals consumed by the loop below).
|
||||
api_call_count = 0
|
||||
|
|
@ -1278,9 +1485,9 @@ def run_conversation(
|
|||
#
|
||||
# Hermes invariant: the system prompt is built ONCE per session
|
||||
# (cached on ``_cached_system_prompt``) and replayed verbatim on
|
||||
# every turn. We send it as a single content string so the
|
||||
# bytes are byte-stable across turns and upstream prompt caches
|
||||
# stay warm.
|
||||
# every turn. ``apply_anthropic_cache_control`` may split its stable
|
||||
# prefix into content blocks on the wire, but the stored string and
|
||||
# its byte-stability remain unchanged.
|
||||
effective_system = active_system_prompt or ""
|
||||
if agent.ephemeral_system_prompt:
|
||||
effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip()
|
||||
|
|
@ -1365,19 +1572,6 @@ def run_conversation(
|
|||
logger=request_logger,
|
||||
)
|
||||
|
||||
# Apply Anthropic prompt caching for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party Anthropic-compatible
|
||||
# gateways. Auto-detected: if ``_use_prompt_caching`` is set,
|
||||
# inject cache_control breakpoints (system + last 3 messages)
|
||||
# to reduce input token costs by ~75% on multi-turn
|
||||
# conversations.
|
||||
if agent._use_prompt_caching:
|
||||
api_messages = apply_anthropic_cache_control(
|
||||
api_messages,
|
||||
cache_ttl=agent._cache_ttl,
|
||||
native_anthropic=agent._use_native_cache_layout,
|
||||
)
|
||||
|
||||
# Safety net: strip orphaned tool results / add stubs for missing
|
||||
# results before sending to the API. Runs unconditionally — not
|
||||
# gated on context_compressor — so orphans from session loading or
|
||||
|
|
@ -1436,6 +1630,39 @@ def run_conversation(
|
|||
# the OpenAI SDK. Sanitizing here prevents the 3-retry cycle.
|
||||
_sanitize_messages_surrogates(api_messages)
|
||||
|
||||
# Apply Anthropic prompt caching for Claude models on native
|
||||
# Anthropic, OpenRouter, and third-party Anthropic-compatible
|
||||
# gateways. Auto-detected: if ``_use_prompt_caching`` is set, inject
|
||||
# cache_control breakpoints for the static system prefix, full system
|
||||
# prompt, and last two messages (or the legacy system-and-3 layout
|
||||
# when no static prefix is available).
|
||||
#
|
||||
# Runs LAST, after every message mutation above. Marking earlier
|
||||
# defeats the prefix stability the mutations exist to create:
|
||||
# ``_apply_cache_marker`` rewrites ``content`` from a plain string
|
||||
# into a ``[{"type": "text", ...}]`` block, so the marked messages
|
||||
# no longer match the ``isinstance(content, str)`` test in the
|
||||
# whitespace-normalization pass and silently keep their raw
|
||||
# leading/trailing whitespace. A tool result ending in "\n" is
|
||||
# therefore sent unstripped while it sits in the last-3 window and
|
||||
# stripped once it rolls out of it — the same message, different
|
||||
# bytes on consecutive turns, which breaks the prefix match at
|
||||
# exactly the point the breakpoints were meant to protect. Marking
|
||||
# last also keeps breakpoints off messages that the orphan sweep or
|
||||
# the thinking-only drop is about to remove or merge away.
|
||||
if agent._use_prompt_caching:
|
||||
_static_system_prefix = getattr(agent, "_cached_system_prompt_static", None)
|
||||
api_messages = apply_anthropic_cache_control(
|
||||
api_messages,
|
||||
cache_ttl=agent._cache_ttl,
|
||||
native_anthropic=agent._use_native_cache_layout,
|
||||
static_system_prefix=(
|
||||
_static_system_prefix
|
||||
if isinstance(_static_system_prefix, str)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
# Build a persistent-MoA request before measuring compression pressure.
|
||||
# MoA reference output is injected into the aggregator prompt, but it
|
||||
# is deliberately ephemeral and therefore absent from ``messages``.
|
||||
|
|
@ -1766,6 +1993,18 @@ def run_conversation(
|
|||
# unless the active provider needs it) so the fallback request
|
||||
# isn't sent with stale, primary-shaped reasoning fields.
|
||||
agent._reapply_reasoning_echo_for_provider(api_messages)
|
||||
# Same story for prompt-cache decoration (#72626): try_activate_
|
||||
# fallback refreshes the policy flags, but the decorated list
|
||||
# still carries the primary's breakpoints (or none). Strip and
|
||||
# re-render for the current provider before building kwargs.
|
||||
api_messages, _moa_prepared_request = (
|
||||
_redecorate_prompt_cache_for_provider(
|
||||
agent,
|
||||
api_messages,
|
||||
system_message=system_message,
|
||||
moa_prepared=_moa_prepared_request,
|
||||
)
|
||||
)
|
||||
api_kwargs = agent._build_api_kwargs(api_messages)
|
||||
if agent._force_ascii_payload:
|
||||
_sanitize_structure_non_ascii(api_kwargs)
|
||||
|
|
@ -3455,7 +3694,7 @@ def run_conversation(
|
|||
agent._buffer_vprint("🔐 Vertex AI token refreshed after 401. Retrying request...")
|
||||
continue
|
||||
if (
|
||||
agent.api_mode == "chat_completions"
|
||||
agent.api_mode in ("chat_completions", "anthropic_messages")
|
||||
and agent.provider == "nous"
|
||||
and status_code == 401
|
||||
and not _retry.nous_auth_retry_attempted
|
||||
|
|
@ -5329,6 +5568,14 @@ def run_conversation(
|
|||
args_preview = raw_args[:200] if isinstance(raw_args, str) else repr(raw_args)[:200]
|
||||
logging.debug("Tool call: %s with args: %s...", tc.function.name, args_preview)
|
||||
|
||||
# Uniquify duplicate tool-call ids BEFORE any downstream
|
||||
# consumer (validation error paths, dispatch, history build,
|
||||
# Responses item-id derivation). Models that reuse one id for
|
||||
# different calls in a batch otherwise lose the later call's
|
||||
# result: the pre-API sanitizer keeps only the first
|
||||
# call/result pair per id. See _uniquify_tool_call_ids.
|
||||
agent._uniquify_tool_call_ids(assistant_message.tool_calls)
|
||||
|
||||
# Validate tool call names - detect model hallucinations
|
||||
# Repair mismatched tool names before validating
|
||||
for tc in assistant_message.tool_calls:
|
||||
|
|
@ -5617,6 +5864,10 @@ def run_conversation(
|
|||
# flag so it can fire again if the model goes empty on
|
||||
# a LATER tool round.
|
||||
agent._post_tool_empty_retried = False
|
||||
# A landed tool call means any earlier dropped-tool-call stall
|
||||
# was recovered — refresh that budget too so it guards each
|
||||
# stall independently rather than capping the whole run.
|
||||
agent._dropped_toolcall_retries = 0
|
||||
|
||||
previous_msg = messages[-1] if messages else None
|
||||
current_interim_visible = agent._interim_assistant_visible_text(assistant_msg)
|
||||
|
|
@ -5633,8 +5884,6 @@ def run_conversation(
|
|||
and previous_interim_visible == current_interim_visible
|
||||
)
|
||||
messages.append(assistant_msg)
|
||||
if not duplicate_previous_interim:
|
||||
agent._emit_interim_assistant_message(assistant_msg)
|
||||
|
||||
# Mixed batch: error-result the invalid calls and strip them
|
||||
# from the execution set. The assistant message above keeps
|
||||
|
|
@ -5656,13 +5905,17 @@ def run_conversation(
|
|||
if tc.function.name in agent.valid_tool_names
|
||||
]
|
||||
|
||||
_tool_turn_persisted = None
|
||||
try:
|
||||
# Persist the assistant tool-call turn before any tool
|
||||
# side effects run. If a destructive tool restarts or
|
||||
# terminates Hermes mid-turn, resume logic still sees the
|
||||
# exact tool-call block that already executed.
|
||||
agent._flush_messages_to_session_db(messages, conversation_history)
|
||||
_tool_turn_persisted = agent._flush_messages_to_session_db(
|
||||
messages, conversation_history
|
||||
)
|
||||
except Exception as exc:
|
||||
_tool_turn_persisted = False
|
||||
logger.warning(
|
||||
"Incremental tool-call persistence failed before execution "
|
||||
"(session=%s): %s",
|
||||
|
|
@ -5670,6 +5923,22 @@ def run_conversation(
|
|||
exc,
|
||||
)
|
||||
|
||||
if _tool_turn_persisted is False:
|
||||
# The canonical append failed. Do not project the row or
|
||||
# run side-effecting tools from state that exists only in
|
||||
# this process. Breaking also avoids retrying the same
|
||||
# unpersisted turn until the iteration budget is exhausted.
|
||||
_turn_exit_reason = "session_persistence_failed"
|
||||
final_response = ""
|
||||
failed = True
|
||||
break
|
||||
|
||||
# A UI must never observe an assistant/tool-call row that is
|
||||
# still only an ephemeral in-memory projection. Emit interim
|
||||
# commentary only after the canonical SessionDB append above.
|
||||
if not duplicate_previous_interim:
|
||||
agent._emit_interim_assistant_message(assistant_msg)
|
||||
|
||||
# Close any open streaming display (response box, reasoning
|
||||
# box) before tool execution begins. Intermediate turns may
|
||||
# have streamed early content that opened the response box;
|
||||
|
|
@ -5684,6 +5953,15 @@ def run_conversation(
|
|||
|
||||
agent._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count)
|
||||
|
||||
if getattr(agent, "_incremental_persistence_failed", False):
|
||||
# A tool result could not be made canonical. Do not send
|
||||
# the in-memory result back to the model or project any
|
||||
# later events from this turn.
|
||||
_turn_exit_reason = "session_persistence_failed"
|
||||
final_response = ""
|
||||
failed = True
|
||||
break
|
||||
|
||||
if agent._tool_guardrail_halt_decision is not None:
|
||||
decision = agent._tool_guardrail_halt_decision
|
||||
_turn_exit_reason = "guardrail_halt"
|
||||
|
|
@ -5865,11 +6143,26 @@ def run_conversation(
|
|||
# Save session log incrementally (so progress is visible even if interrupted)
|
||||
agent._session_messages = messages
|
||||
|
||||
# Touch activity before continuing so the gateway's
|
||||
# inactivity monitor never sees a stale timestamp
|
||||
# between tool completion and the start of the next
|
||||
# API call. Without this, a tool-call result (which
|
||||
# takes ~0s to process) followed by slow post-tool
|
||||
# processing (compression, persist) and a slow
|
||||
# follow-up API call can exceed the gateway inactivity
|
||||
# timeout (HERMES_AGENT_TIMEOUT, default 1800s) and the
|
||||
# gateway kills the session before the next activity
|
||||
# touch fires (#69559, #69131).
|
||||
agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}")
|
||||
# Continue loop for next response
|
||||
continue
|
||||
|
||||
else:
|
||||
# No tool calls - this is the final response
|
||||
# No tool calls - this is the final response.
|
||||
# (Dropped tool-call recovery — finish_reason=="tool_calls" with
|
||||
# an empty tool_calls array — is handled at the finalization
|
||||
# chokepoint below, after final_msg is built, so it catches
|
||||
# every path that reaches turn finalization, not just this one.)
|
||||
final_response = assistant_message.content or ""
|
||||
|
||||
# Fix: unmute output when entering the no-tool-call branch
|
||||
|
|
@ -6141,7 +6434,28 @@ def run_conversation(
|
|||
". No fallback providers configured.")
|
||||
)
|
||||
|
||||
final_response = "(empty)"
|
||||
# Deliver a labeled reasoning excerpt instead of a bare
|
||||
# "(empty)" when the model DID think but never produced
|
||||
# visible text. This is delivery-only: the persisted
|
||||
# assistant message above keeps the "(empty)" sentinel
|
||||
# (its replay semantics prevent empty-response loops),
|
||||
# and raw chain-of-thought is never promoted to a normal
|
||||
# answer earlier in the ladder — prefill continuation,
|
||||
# empty-content retries, and provider fallback all run
|
||||
# first. Only at this terminal, where the alternative is
|
||||
# returning nothing, is showing the model's own reasoning
|
||||
# (clearly labeled as such) strictly more useful.
|
||||
# Idea credit: PR #48795 (@ligl0325).
|
||||
if reasoning_text:
|
||||
final_response = (
|
||||
"⚠️ The model produced only internal reasoning and "
|
||||
"no final answer, despite retries"
|
||||
+ (" and fallback" if agent._fallback_chain else "")
|
||||
+ ". Its last reasoning, which may contain the "
|
||||
"answer:\n\n" + reasoning_preview
|
||||
)
|
||||
else:
|
||||
final_response = "(empty)"
|
||||
break
|
||||
|
||||
# Reset retry counter/signature on successful content
|
||||
|
|
@ -6201,6 +6515,64 @@ def run_conversation(
|
|||
|
||||
final_msg = agent._build_assistant_message(assistant_message, finish_reason)
|
||||
|
||||
# ── Dropped tool-call recovery (copilot/Claude) ────────
|
||||
# Some providers (observed: claude-opus-4.8 / claude-sonnet-4.5
|
||||
# on GitHub Copilot, ~2026-07) return finish_reason="tool_calls"
|
||||
# while the parsed tool_calls array is empty — the model
|
||||
# signalled it wanted to act but the payload shipped no call.
|
||||
# Reaching finalization with that mismatch means the turn is
|
||||
# about to end with the task unstarted (the narration, which may
|
||||
# be in content or only in the reasoning field, gets treated as
|
||||
# the final answer). Re-prompt (bounded to 3 CONSECUTIVE stalls;
|
||||
# the budget resets after any successful tool round) to make the
|
||||
# model emit the call instead of exiting. finish_reason="stop"
|
||||
# text finishes never enter this guard.
|
||||
if (
|
||||
finish_reason == "tool_calls"
|
||||
and not assistant_message.tool_calls
|
||||
and getattr(agent, "_dropped_toolcall_retries", 0) < 3
|
||||
):
|
||||
agent._dropped_toolcall_retries = getattr(agent, "_dropped_toolcall_retries", 0) + 1
|
||||
logger.warning(
|
||||
"finish_reason=tool_calls with empty tool_calls array "
|
||||
"(narration only) — re-prompting to emit the call "
|
||||
"(retry %d/3, model=%s provider=%s)",
|
||||
agent._dropped_toolcall_retries, agent.model, agent.provider,
|
||||
)
|
||||
agent._emit_status(
|
||||
"↻ Model signaled a tool call but sent none — "
|
||||
f"re-prompting ({agent._dropped_toolcall_retries}/3)"
|
||||
)
|
||||
# Both halves of the re-prompt pair are ephemeral recovery
|
||||
# scaffolding (mirrors the empty-response nudge pattern):
|
||||
# the interim narration-only assistant turn exists solely to
|
||||
# keep role alternation valid for the nudge, and the nudge
|
||||
# exists solely to drive the retry. Flag both so the
|
||||
# persistence layer never writes them to the durable
|
||||
# transcript and the finalization pop below can strip an
|
||||
# unanswered tail pair. A recovered (answered) pair stays
|
||||
# buried mid-list in live memory but is skipped by the
|
||||
# flush regardless of position.
|
||||
final_msg["_dropped_toolcall_nudge"] = True
|
||||
messages.append(final_msg)
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Your previous turn indicated a tool call but none was "
|
||||
"included. Do not narrate a plan or restate intent — issue "
|
||||
"the actual tool call now to continue the task."
|
||||
),
|
||||
"_dropped_toolcall_nudge": True,
|
||||
})
|
||||
agent._session_messages = messages
|
||||
final_response = None
|
||||
continue
|
||||
|
||||
# Reached finalization without the dropped-tool-call mismatch —
|
||||
# a genuine turn end. Clear the consecutive-stall budget so the
|
||||
# next turn starts fresh.
|
||||
agent._dropped_toolcall_retries = 0
|
||||
|
||||
# Pop thinking-only prefill and empty-response retry
|
||||
# scaffolding before appending either a final response or a
|
||||
# verification-stop follow-up. These internal turns are only
|
||||
|
|
@ -6213,6 +6585,7 @@ def run_conversation(
|
|||
messages[-1].get("_thinking_prefill")
|
||||
or messages[-1].get("_empty_recovery_synthetic")
|
||||
or messages[-1].get("_empty_terminal_sentinel")
|
||||
or messages[-1].get("_dropped_toolcall_nudge")
|
||||
)
|
||||
):
|
||||
messages.pop()
|
||||
|
|
|
|||
|
|
@ -512,7 +512,7 @@ class CopilotACPClient:
|
|||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
bufsize=1,
|
||||
cwd=self._acp_cwd,
|
||||
env=_build_subprocess_env(),
|
||||
|
|
@ -708,7 +708,7 @@ class CopilotACPClient:
|
|||
if block_error:
|
||||
raise PermissionError(block_error)
|
||||
try:
|
||||
content = path.read_text()
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
content = ""
|
||||
line = params.get("line")
|
||||
|
|
@ -736,7 +736,7 @@ class CopilotACPClient:
|
|||
if denied:
|
||||
raise PermissionError(denied)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(str(params.get("content") or ""))
|
||||
path.write_text(str(params.get("content") or ""), encoding="utf-8")
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message_id,
|
||||
|
|
|
|||
|
|
@ -594,6 +594,14 @@ class CredentialPool:
|
|||
# Re-armed to None on every successful selection so a recover→re-exhaust
|
||||
# transition logs promptly instead of being swallowed by a stale window.
|
||||
self._last_no_entries_log_at: Optional[float] = None
|
||||
# #70401: consecutive mark_exhausted_and_rotate() calls whose supplied
|
||||
# credential identity matched no pool entry (OAuth wrappers whose
|
||||
# runtime key rotates, entries pruned by another process, ...). These
|
||||
# rotations mark nothing exhausted, so without a cap the pool can
|
||||
# never converge to "no available entries" and the caller's 401 retry
|
||||
# loop runs unbounded and non-interruptible. Reset whenever a real
|
||||
# entry is identified or an escape path returns None.
|
||||
self._unmatched_rotation_streak: int = 0
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
with self._lock:
|
||||
|
|
@ -622,6 +630,28 @@ class CredentialPool:
|
|||
with self._lock:
|
||||
return self._current_unlocked()
|
||||
|
||||
def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]:
|
||||
"""Return the stable id for the runtime credential in use.
|
||||
|
||||
Prefer the current selection when it still supplies ``api_key_hint``.
|
||||
If the cursor was cleared, fall back to an unambiguous key match.
|
||||
"""
|
||||
with self._lock:
|
||||
current = self._current_unlocked()
|
||||
if current is not None and (
|
||||
api_key_hint is None
|
||||
or current.runtime_api_key == api_key_hint
|
||||
):
|
||||
return current.id
|
||||
if api_key_hint is None:
|
||||
return None
|
||||
matches = [
|
||||
entry
|
||||
for entry in self._entries
|
||||
if entry.runtime_api_key == api_key_hint
|
||||
]
|
||||
return matches[0].id if len(matches) == 1 else None
|
||||
|
||||
def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
|
||||
"""Swap an entry in-place by id, preserving sort order."""
|
||||
for idx, entry in enumerate(self._entries):
|
||||
|
|
@ -1562,7 +1592,13 @@ class CredentialPool:
|
|||
|
||||
def select(self) -> Optional[PooledCredential]:
|
||||
with self._lock:
|
||||
return self._select_unlocked()
|
||||
entry = self._select_unlocked()
|
||||
if entry is not None:
|
||||
# A normal (non-recovery) selection starts a fresh episode —
|
||||
# don't let a leftover unmatched-rotation streak from an old
|
||||
# failure trip the #70401 bound early next time.
|
||||
self._unmatched_rotation_streak = 0
|
||||
return entry
|
||||
|
||||
def _available_entries(self, *, clear_expired: bool = False, refresh: bool = False) -> List[PooledCredential]:
|
||||
"""Return entries not currently in exhaustion cooldown.
|
||||
|
|
@ -1763,10 +1799,17 @@ class CredentialPool:
|
|||
status_code: Optional[int],
|
||||
error_context: Optional[Dict[str, Any]] = None,
|
||||
api_key_hint: Optional[str] = None,
|
||||
credential_id: Optional[str] = None,
|
||||
) -> Optional[PooledCredential]:
|
||||
with self._lock:
|
||||
entry = None
|
||||
if api_key_hint:
|
||||
identity_supplied = bool(credential_id or api_key_hint)
|
||||
if credential_id:
|
||||
entry = next(
|
||||
(e for e in self._entries if e.id == credential_id),
|
||||
None,
|
||||
)
|
||||
if entry is None and api_key_hint:
|
||||
# Prefer the specific entry whose API key matches the one that
|
||||
# actually failed. When this pool was freshly loaded from disk
|
||||
# (another process already rotated), current() is None and
|
||||
|
|
@ -1775,20 +1818,59 @@ class CredentialPool:
|
|||
(e for e in self._entries if e.runtime_api_key == api_key_hint),
|
||||
None,
|
||||
)
|
||||
if entry is None:
|
||||
# The failed key is identifiable but matches no entry
|
||||
# (rotated away, or a wrapper whose runtime key differs).
|
||||
# Falling through to current()/_select_unlocked() would
|
||||
# mark an INNOCENT healthy key exhausted for the full
|
||||
# cooldown TTL. Don't guess — just hand back a fresh
|
||||
# selection so the caller can retry.
|
||||
logger.info(
|
||||
"credential pool: failed key hint matched no %s entry; "
|
||||
"rotating without marking any credential exhausted",
|
||||
if entry is None and identity_supplied:
|
||||
# The failed credential is identifiable but matches no entry
|
||||
# (rotated away, or a wrapper whose runtime key differs).
|
||||
# Falling through to current()/_select_unlocked() would mark an
|
||||
# innocent healthy key exhausted for the full cooldown TTL.
|
||||
#
|
||||
# #70401: this branch must still be BOUNDED. With OAuth-token
|
||||
# auth the upstream 401's key hint never matches any entry's
|
||||
# ``runtime_api_key``, so every retry lands here, nothing is
|
||||
# ever marked exhausted, and the pool can never reach the
|
||||
# "no available entries" state — the caller retries the same
|
||||
# dead token forever (~6/sec, starving the event loop so chat
|
||||
# interrupts are never processed). The single-entry case
|
||||
# below already escapes; multi-entry pools could still
|
||||
# ping-pong A→B→A indefinitely without marking anything.
|
||||
# Cap consecutive no-mark rotations at one full lap of the
|
||||
# available entries: past that, every candidate has been
|
||||
# handed back at least once without recovery, so stop
|
||||
# guessing and surface the error (no cooldown is written for
|
||||
# anybody — healthy keys stay available for the next turn).
|
||||
self._unmatched_rotation_streak += 1
|
||||
available_count = len(self._available_entries())
|
||||
if self._unmatched_rotation_streak > max(available_count, 1):
|
||||
logger.warning(
|
||||
"credential pool: failed credential identity matched no "
|
||||
"%s entry for %d consecutive rotations (pool size %d) — "
|
||||
"surfacing the error instead of rotating again",
|
||||
self.provider,
|
||||
self._unmatched_rotation_streak,
|
||||
available_count,
|
||||
)
|
||||
self._unmatched_rotation_streak = 0
|
||||
self._current_id = None
|
||||
return self._select_unlocked()
|
||||
return None
|
||||
logger.info(
|
||||
"credential pool: failed credential identity matched no %s "
|
||||
"entry; rotating without marking any credential exhausted",
|
||||
self.provider,
|
||||
)
|
||||
self._current_id = None
|
||||
next_entry = self._select_unlocked()
|
||||
if next_entry is not None and len(self._available_entries()) == 1:
|
||||
# A single-entry pool cannot rotate. Returning its only
|
||||
# entry reports a successful recovery without changing
|
||||
# the credential, so the caller retries the same 401
|
||||
# indefinitely. Let fallback/error propagation proceed.
|
||||
self._unmatched_rotation_streak = 0
|
||||
self._current_id = None
|
||||
return None
|
||||
return next_entry
|
||||
# A real entry was identified — any prior unmatched-rotation
|
||||
# streak is stale (this mark WILL advance pool state).
|
||||
self._unmatched_rotation_streak = 0
|
||||
if entry is None:
|
||||
entry = self._current_unlocked() or self._select_unlocked()
|
||||
if entry is None:
|
||||
|
|
@ -1806,12 +1888,13 @@ class CredentialPool:
|
|||
# disconnects (a ~2.5min hang with no error surfaced to the user).
|
||||
# Mark every entry sharing the failed key so the pool can reach the
|
||||
# "no available entries" state and let the error propagate.
|
||||
if api_key_hint:
|
||||
failed_runtime_key = getattr(entry, "runtime_api_key", None)
|
||||
if identity_supplied and failed_runtime_key:
|
||||
siblings_marked = False
|
||||
for sibling in self._entries:
|
||||
if sibling.id == entry.id:
|
||||
continue
|
||||
if sibling.runtime_api_key == api_key_hint:
|
||||
if sibling.runtime_api_key == failed_runtime_key:
|
||||
self._mark_exhausted(
|
||||
sibling, status_code, error_context, persist=False
|
||||
)
|
||||
|
|
@ -1885,9 +1968,11 @@ class CredentialPool:
|
|||
return self._try_refresh_current_unlocked()
|
||||
|
||||
def try_refresh_matching(
|
||||
self, api_key_hint: Optional[str] = None
|
||||
self,
|
||||
api_key_hint: Optional[str] = None,
|
||||
credential_id: Optional[str] = None,
|
||||
) -> Optional[PooledCredential]:
|
||||
"""Force-refresh the entry that supplied ``api_key_hint``.
|
||||
"""Force-refresh the entry that supplied the failed request.
|
||||
|
||||
Direct provider integrations may reload the pool after a request has
|
||||
already failed, so they cannot rely on ``current_id`` identifying the
|
||||
|
|
@ -1897,17 +1982,29 @@ class CredentialPool:
|
|||
"""
|
||||
with self._lock:
|
||||
entry = None
|
||||
if api_key_hint:
|
||||
if credential_id:
|
||||
entry = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in self._entries
|
||||
if candidate.runtime_api_key == api_key_hint
|
||||
if candidate.id == credential_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
entry = self._current_unlocked() or self._select_unlocked(refresh=False)
|
||||
if entry is None:
|
||||
if api_key_hint:
|
||||
entry = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in self._entries
|
||||
if candidate.runtime_api_key == api_key_hint
|
||||
),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
entry = self._current_unlocked() or self._select_unlocked(
|
||||
refresh=False
|
||||
)
|
||||
if entry is None:
|
||||
return None
|
||||
self._current_id = entry.id
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ def _remove_env_source(provider: str, removed) -> RemovalResult:
|
|||
if env_path.exists():
|
||||
env_in_dotenv = any(
|
||||
line.strip().startswith(f"{env_var}=")
|
||||
for line in env_path.read_text(errors="replace").splitlines()
|
||||
for line in env_path.read_text(errors="replace", encoding="utf-8").splitlines()
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ def apply_automatic_transitions(now: Optional[datetime] = None) -> Dict[str, int
|
|||
|
||||
counts = {"marked_stale": 0, "archived": 0, "reactivated": 0, "checked": 0, "seeded": 0}
|
||||
|
||||
for row in _u.agent_created_report():
|
||||
for row in _u.curated_report():
|
||||
counts["checked"] += 1
|
||||
name = row["name"]
|
||||
if row.get("pinned"):
|
||||
|
|
@ -1472,15 +1472,16 @@ def _render_report_markdown(p: Dict[str, Any]) -> str:
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _render_candidate_list() -> str:
|
||||
"""Human/agent-readable list of agent-created skills with usage stats."""
|
||||
rows = skill_usage.agent_created_report()
|
||||
"""Human/agent-readable list of curator-managed skills with usage stats."""
|
||||
rows = skill_usage.curated_report()
|
||||
if not rows:
|
||||
return "No agent-created skills to review."
|
||||
return "No curator-managed skills to review."
|
||||
cron_referenced = _cron_referenced_skills()
|
||||
lines = [f"Agent-created skills ({len(rows)}):\n"]
|
||||
lines = [f"Curator-managed skills ({len(rows)}):\n"]
|
||||
for r in rows:
|
||||
lines.append(
|
||||
f"- {r['name']} "
|
||||
f"provenance={r.get('provenance', 'agent')} "
|
||||
f"state={r['state']} "
|
||||
f"pinned={'yes' if r.get('pinned') else 'no'} "
|
||||
f"cron={'yes' if r['name'] in cron_referenced else 'no'} "
|
||||
|
|
@ -1533,7 +1534,7 @@ def run_curator_review(
|
|||
if dry_run:
|
||||
# Count candidates without mutating state.
|
||||
try:
|
||||
report = skill_usage.agent_created_report()
|
||||
report = skill_usage.curated_report()
|
||||
counts = {
|
||||
"checked": len(report),
|
||||
"marked_stale": 0,
|
||||
|
|
@ -1586,7 +1587,7 @@ def run_curator_review(
|
|||
nonlocal auto_summary
|
||||
# Snapshot skill state BEFORE the LLM pass so the report can diff.
|
||||
try:
|
||||
before_report = skill_usage.agent_created_report()
|
||||
before_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
before_report = []
|
||||
before_names = {r.get("name") for r in before_report if isinstance(r, dict)}
|
||||
|
|
@ -1612,7 +1613,7 @@ def run_curator_review(
|
|||
state2["last_run_duration_seconds"] = elapsed
|
||||
state2["last_run_summary"] = final_summary
|
||||
try:
|
||||
after_report = skill_usage.agent_created_report()
|
||||
after_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
after_report = []
|
||||
try:
|
||||
|
|
@ -1699,7 +1700,7 @@ def run_curator_review(
|
|||
try:
|
||||
rename_lines = _build_rename_summary(
|
||||
before_names=before_names,
|
||||
after_report=skill_usage.agent_created_report(),
|
||||
after_report=skill_usage.curated_report(),
|
||||
tool_calls=llm_meta.get("tool_calls", []) or [],
|
||||
model_final=llm_meta.get("final", "") or "",
|
||||
)
|
||||
|
|
@ -1717,7 +1718,7 @@ def run_curator_review(
|
|||
# reporting bug never breaks the curator itself. Report path is
|
||||
# recorded in state so `hermes curator status` can point at it.
|
||||
try:
|
||||
after_report = skill_usage.agent_created_report()
|
||||
after_report = skill_usage.curated_report()
|
||||
except Exception:
|
||||
after_report = []
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -159,6 +159,14 @@ _RATE_LIMIT_PATTERNS = [
|
|||
"throttlingexception",
|
||||
"too many concurrent requests",
|
||||
"servicequotaexceededexception",
|
||||
# Generic throttle prefix — Bedrock (and some proxies) surface throttling
|
||||
# as "Throttling error: Too many tokens, please wait before trying
|
||||
# again." Without this entry the message falls through to the
|
||||
# context-overflow list (which contains "too many tokens") and the retry
|
||||
# loop compresses a healthy session instead of backing off. Matched
|
||||
# BEFORE _CONTEXT_OVERFLOW_PATTERNS in the message-only path, so the
|
||||
# throttle wins. (port of anomalyco/opencode#37848's exclusion guard)
|
||||
"throttling",
|
||||
]
|
||||
|
||||
# Patterns that indicate provider-side overload, NOT a per-credential rate
|
||||
|
|
@ -212,6 +220,12 @@ _PAYLOAD_TOO_LARGE_PATTERNS = [
|
|||
"request entity too large",
|
||||
"payload too large",
|
||||
"error code: 413",
|
||||
# Anthropic's structured 413 error type. Normally arrives with an HTTP
|
||||
# 413 status (handled by the status path), but aggregators/proxies can
|
||||
# re-wrap it into a plain message with no status attribute — route it to
|
||||
# the same compression recovery. (port of anomalyco/opencode#37848)
|
||||
"request_too_large",
|
||||
"request exceeds the maximum size",
|
||||
]
|
||||
|
||||
# Image-size patterns. Matched against 400 bodies (not 413) because most
|
||||
|
|
@ -298,6 +312,10 @@ _CONTEXT_OVERFLOW_PATTERNS = [
|
|||
"max input token",
|
||||
"input token",
|
||||
"exceeds the maximum number of input tokens",
|
||||
# Together/Fireworks-style: "Input length 131393 exceeds the maximum
|
||||
# allowed input length of 131040 tokens." No other pattern in this list
|
||||
# matches that wording. (port of anomalyco/opencode#37848)
|
||||
"maximum allowed input length",
|
||||
]
|
||||
|
||||
# Model not found patterns
|
||||
|
|
|
|||
|
|
@ -163,6 +163,42 @@ _FREE_TIER_GUIDANCE = (
|
|||
)
|
||||
|
||||
|
||||
def is_standard_key_auth_error(
|
||||
status: int, error_message: str, reason: str = ""
|
||||
) -> bool:
|
||||
"""Return True when a Gemini 401 indicates Google rejected the key TYPE.
|
||||
|
||||
Google began rejecting unrestricted legacy "Standard" Google Cloud API
|
||||
keys on the Gemini API on June 19, 2026, and ALL Standard keys stop
|
||||
working in September 2026. The rejection surfaces as a misleading 401
|
||||
telling the user to supply an OAuth 2 access token ("Request had invalid
|
||||
authentication credentials. Expected OAuth 2 access token, login cookie
|
||||
or other valid authentication credential."), optionally carrying
|
||||
``google.rpc.ErrorInfo`` reason ``ACCESS_TOKEN_TYPE_UNSUPPORTED``.
|
||||
|
||||
Scoped narrowly so a plain bad key (reason ``API_KEY_INVALID``,
|
||||
"API key not valid") keeps its existing message.
|
||||
"""
|
||||
if status != 401:
|
||||
return False
|
||||
if reason == "ACCESS_TOKEN_TYPE_UNSUPPORTED":
|
||||
return True
|
||||
return "expected oauth 2 access token" in (error_message or "").lower()
|
||||
|
||||
|
||||
_STANDARD_KEY_GUIDANCE = (
|
||||
"\n\nGoogle Gemini rejected this API key's type — you do NOT need OAuth. "
|
||||
"Google began rejecting legacy 'Standard' Google Cloud keys for the "
|
||||
"Gemini API on June 19, 2026, and all Standard keys stop working in "
|
||||
"September 2026. Open https://aistudio.google.com/api-keys, check the "
|
||||
"key's type and status, and create a replacement Gemini API key (or, as "
|
||||
"a temporary bridge, restrict the Standard key to "
|
||||
"generativelanguage.googleapis.com). Then update GEMINI_API_KEY / "
|
||||
"GOOGLE_API_KEY in ~/.hermes/.env and restart your session. "
|
||||
"Details: https://ai.google.dev/gemini-api/docs/api-key"
|
||||
)
|
||||
|
||||
|
||||
class GeminiAPIError(Exception):
|
||||
"""Error shape compatible with Hermes retry/error classification."""
|
||||
|
||||
|
|
@ -824,6 +860,12 @@ def gemini_http_error(
|
|||
if status == 429 and is_free_tier_quota_error(err_message or body_text):
|
||||
message = message + _FREE_TIER_GUIDANCE
|
||||
|
||||
# Legacy "Standard" Google Cloud key rejection (June 19, 2026 onward) ->
|
||||
# Google's raw 401 misleadingly tells the user to use OAuth. Append the
|
||||
# actual fix (mint a new Gemini API key in AI Studio).
|
||||
if is_standard_key_auth_error(status, err_message or body_text, reason):
|
||||
message = message + _STANDARD_KEY_GUIDANCE
|
||||
|
||||
return GeminiAPIError(
|
||||
message,
|
||||
code=code,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any, Dict
|
||||
|
||||
# Gemini's ``FunctionDeclaration.parameters`` field accepts the ``Schema``
|
||||
|
|
@ -76,15 +77,31 @@ def sanitize_gemini_schema(schema: Any) -> Dict[str, Any]:
|
|||
|
||||
# Gemini's Schema validator requires every ``enum`` entry to be a string,
|
||||
# even when the parent ``type`` is ``integer`` / ``number`` / ``boolean``.
|
||||
# OpenAI / OpenRouter / Anthropic accept typed enums (e.g. Discord's
|
||||
# ``auto_archive_duration: {type: integer, enum: [60, 1440, 4320, 10080]}``),
|
||||
# so we only drop the ``enum`` when it would collide with Gemini's rule.
|
||||
# Keeping ``type: integer`` plus the human-readable description gives the
|
||||
# model enough guidance; the tool handler still validates the value.
|
||||
# Preserve those constraints by stringifying scalar values while keeping
|
||||
# the declared type intact; Gemini uses the strings as schema metadata and
|
||||
# still emits typed tool arguments at runtime.
|
||||
enum_val = cleaned.get("enum")
|
||||
type_val = cleaned.get("type")
|
||||
if isinstance(enum_val, list) and type_val in {"integer", "number", "boolean"}:
|
||||
if any(not isinstance(item, str) for item in enum_val):
|
||||
stringified = []
|
||||
for item in enum_val:
|
||||
if isinstance(item, str):
|
||||
value = item
|
||||
elif isinstance(item, bool):
|
||||
value = "true" if item else "false"
|
||||
elif (
|
||||
isinstance(item, (int, float))
|
||||
and not isinstance(item, bool)
|
||||
and math.isfinite(item)
|
||||
):
|
||||
value = str(item)
|
||||
else:
|
||||
continue
|
||||
if value not in stringified:
|
||||
stringified.append(value)
|
||||
if stringified:
|
||||
cleaned["enum"] = stringified
|
||||
else:
|
||||
cleaned.pop("enum", None)
|
||||
|
||||
# Gemini validates ``required`` strictly against the same node's
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ Language resolution order:
|
|||
3. ``display.language`` from config.yaml
|
||||
4. ``"en"`` (baseline)
|
||||
|
||||
Supported languages: en, zh, ja, de, es, fr, tr, uk. Unknown values fall back to en.
|
||||
Supported languages: en, zh, zh-hant, ja, de, es, fr, tr, uk, af, ko, it, ga,
|
||||
pt, ru, hu, ar. Unknown values fall back to en.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -41,7 +42,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
SUPPORTED_LANGUAGES: tuple[str, ...] = (
|
||||
"en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk",
|
||||
"af", "ko", "it", "ga", "pt", "ru", "hu",
|
||||
"af", "ko", "it", "ga", "pt", "ru", "hu", "ar",
|
||||
)
|
||||
DEFAULT_LANGUAGE = "en"
|
||||
|
||||
|
|
@ -78,6 +79,9 @@ _LANGUAGE_ALIASES: dict[str, str] = {
|
|||
"russian": "ru", "русский": "ru", "ru-ru": "ru",
|
||||
# Hungarian
|
||||
"hungarian": "hu", "magyar": "hu", "hu-hu": "hu",
|
||||
# Arabic — bare "arabic"/endonym plus the common regional BCP-47 tags.
|
||||
"arabic": "ar", "العربية": "ar",
|
||||
"ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar",
|
||||
}
|
||||
|
||||
_catalog_cache: dict[str, dict[str, str]] = {}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Extracted from ``run_agent.py``. Each ``AIAgent`` instance (parent or
|
||||
subagent) holds an :class:`IterationBudget`; the parent's cap comes from
|
||||
``max_iterations`` (default 90), each subagent's cap comes from
|
||||
``max_iterations`` (default 500), each subagent's cap comes from
|
||||
``delegation.max_iterations`` (default 50).
|
||||
|
||||
``run_agent`` re-exports ``IterationBudget`` so existing
|
||||
|
|
@ -18,7 +18,7 @@ class IterationBudget:
|
|||
"""Thread-safe iteration counter for an agent.
|
||||
|
||||
Each agent (parent or subagent) gets its own ``IterationBudget``.
|
||||
The parent's budget is capped at ``max_iterations`` (default 90).
|
||||
The parent's budget is capped at ``max_iterations`` (default 500).
|
||||
Each subagent gets an independent budget capped at
|
||||
``delegation.max_iterations`` (default 50) — this means total
|
||||
iterations across parent + subagents can exceed the parent's cap.
|
||||
|
|
|
|||
|
|
@ -267,7 +267,7 @@ def _install_npm(
|
|||
[npm, "install", "--prefix", str(staging), "--silent", "--no-fund", "--no-audit", *install_targets],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=300,
|
||||
stdin=subprocess.DEVNULL,
|
||||
creationflags=windows_hide_flags(),
|
||||
|
|
@ -316,7 +316,7 @@ def _install_go(pkg: str, bin_name: str) -> Optional[str]:
|
|||
[go, "install", pkg],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=600,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
|
|
|
|||
|
|
@ -388,10 +388,10 @@ def _maybe_apply_moa_cache_control(
|
|||
|
||||
Reuses the SAME policy function as the main agent loop
|
||||
(``anthropic_prompt_cache_policy``) resolved against the slot's own
|
||||
provider/base_url/api_mode/model, and the SAME breakpoint layout
|
||||
(``apply_anthropic_cache_control``, system_and_3). This keeps advisor and
|
||||
aggregator calls decorated exactly like an acting agent on that provider
|
||||
would be — no MoA-specific caching logic to drift.
|
||||
provider/base_url/api_mode/model and shared marker helper
|
||||
(``apply_anthropic_cache_control``). MoA has no per-session static prefix,
|
||||
so it uses the helper's legacy system-and-3 fallback without carrying a
|
||||
separate caching strategy.
|
||||
|
||||
Returns the messages unchanged on any resolution error or when the
|
||||
policy says the route doesn't honor markers.
|
||||
|
|
@ -480,10 +480,11 @@ def _run_reference(
|
|||
reserve_output_tokens=max_tokens,
|
||||
context_length_cache=context_length_cache,
|
||||
)
|
||||
# Apply the same Anthropic-style prompt-caching decoration the main
|
||||
# agent loop applies (system_and_3 breakpoints). The advisory view is
|
||||
# append-only across iterations (new turns append before the trailing
|
||||
# synthetic marker), so on cache-honoring routes (Claude via
|
||||
# Apply the Anthropic-style prompt-caching decoration used by the main
|
||||
# agent loop. This fixed reference prompt has no session-specific
|
||||
# prefix split, so the helper uses its legacy system-and-3 fallback.
|
||||
# The advisory view is append-only across iterations (new turns append
|
||||
# before the trailing synthetic marker), so on cache-honoring routes (Claude via
|
||||
# OpenRouter/native, MiniMax, Qwen/DashScope) iteration N+1's prefix
|
||||
# replays iteration N's cached prefix. Without this, Claude advisors
|
||||
# served ZERO cache reads across an entire benchmark run (measured:
|
||||
|
|
@ -1336,6 +1337,63 @@ def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str
|
|||
agg_messages.append({"role": "user", "content": guidance})
|
||||
|
||||
|
||||
def peel_reference_guidance(
|
||||
messages: list[dict[str, Any]],
|
||||
guidance: Any,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Remove reference guidance previously attached by ``_attach_reference_guidance``.
|
||||
|
||||
Exact inverse of the three attach shapes above (string merge, trailing
|
||||
text part, appended user message) — kept adjacent so the two evolve
|
||||
together; a drifting separator or shape would make the peel silently
|
||||
no-op and let a cache breakpoint land on the turn-varying guidance
|
||||
block (the bug class #72626 fixes).
|
||||
|
||||
Used by the failover redecoration chokepoint: redecoration must run on
|
||||
the base transcript so the last cache breakpoint does not land on the
|
||||
guidance; callers then rebase via ``rebase_prepared_request``.
|
||||
|
||||
Returns a new list (input list and its messages are not mutated).
|
||||
"""
|
||||
if not guidance or not messages:
|
||||
return messages
|
||||
guidance_text = str(guidance)
|
||||
last = messages[-1]
|
||||
if not isinstance(last, dict) or last.get("role") != "user":
|
||||
return messages
|
||||
content = last.get("content")
|
||||
if content == guidance_text:
|
||||
# Attach shape (c): guidance was appended as its own user message.
|
||||
return list(messages[:-1])
|
||||
suffix = "\n\n" + guidance_text
|
||||
if isinstance(content, str) and content.endswith(suffix):
|
||||
# Attach shape (a): merged into a trailing string user turn.
|
||||
peeled = dict(last)
|
||||
peeled["content"] = content[: -len(suffix)]
|
||||
return [*messages[:-1], peeled]
|
||||
if isinstance(content, list) and content:
|
||||
last_part = content[-1]
|
||||
if isinstance(last_part, dict) and last_part.get("type", "text") == "text":
|
||||
text = last_part.get("text") or ""
|
||||
if text == suffix or text == guidance_text:
|
||||
# Attach shape (b): guidance rode as its own trailing part.
|
||||
peeled = dict(last)
|
||||
peeled["content"] = list(content[:-1])
|
||||
if not peeled["content"]:
|
||||
# The guidance part was the only content — mirror the
|
||||
# string shape (c) and drop the whole message rather
|
||||
# than leaving an empty-content user turn behind.
|
||||
return list(messages[:-1])
|
||||
return [*messages[:-1], peeled]
|
||||
if text.endswith(suffix):
|
||||
new_part = dict(last_part)
|
||||
new_part["text"] = text[: -len(suffix)]
|
||||
peeled = dict(last)
|
||||
peeled["content"] = [*content[:-1], new_part]
|
||||
return [*messages[:-1], peeled]
|
||||
return messages
|
||||
|
||||
|
||||
class MoAChatCompletions:
|
||||
"""OpenAI-chat-compatible facade where the aggregator is the acting model."""
|
||||
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ DEFAULT_CONTEXT_LENGTHS = {
|
|||
# OpenRouter-prefixed models resolve via OpenRouter live API or models.dev.
|
||||
"claude-fable-5": 1000000,
|
||||
"claude-fable": 1000000,
|
||||
"claude-opus-5": 1000000,
|
||||
"claude-sonnet-5": 1000000,
|
||||
"claude-opus-4-8": 1000000,
|
||||
"claude-opus-4.8": 1000000,
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ def record_nous_rate_limit(
|
|||
# Atomic write: write to temp file + rename
|
||||
fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "w") as f:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f)
|
||||
atomic_replace(tmp_path, path)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
"""Anthropic prompt caching strategy.
|
||||
|
||||
Single layout: ``system_and_3``. 4 cache_control breakpoints — system
|
||||
prompt + last 3 non-system messages, all at the same TTL (5m or 1h).
|
||||
Reduces input token costs by ~75% on multi-turn conversations within a
|
||||
single session.
|
||||
The default layout uses 4 cache_control breakpoints: the static system
|
||||
prefix, the end of the system prompt, and the last 2 non-system messages.
|
||||
When a static system prefix is unavailable, it falls back to one system
|
||||
breakpoint plus the last 3 messages. All markers use the same TTL (5m or 1h).
|
||||
This preserves intra-session caching while allowing new sessions to reuse the
|
||||
stable system-prompt prefix.
|
||||
|
||||
Pure functions -- no class state, no AIAgent dependency.
|
||||
"""
|
||||
|
|
@ -81,15 +83,108 @@ def _build_marker(ttl: str) -> Dict[str, str]:
|
|||
return marker
|
||||
|
||||
|
||||
def _apply_system_cache_markers(
|
||||
message: dict,
|
||||
cache_marker: dict,
|
||||
static_system_prefix: str | None,
|
||||
*,
|
||||
native_anthropic: bool,
|
||||
) -> int:
|
||||
"""Mark the static system prefix and full prompt when they can be split.
|
||||
|
||||
The system prompt remains one stored string. Splitting it only in the
|
||||
outgoing request keeps session persistence and non-Anthropic transports
|
||||
unchanged while making the stable prefix independently cacheable.
|
||||
"""
|
||||
content = message.get("content")
|
||||
if (
|
||||
isinstance(static_system_prefix, str)
|
||||
and static_system_prefix
|
||||
and isinstance(content, str)
|
||||
and content.startswith(static_system_prefix)
|
||||
):
|
||||
suffix = content[len(static_system_prefix):]
|
||||
if suffix:
|
||||
message["content"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": static_system_prefix,
|
||||
"cache_control": cache_marker,
|
||||
},
|
||||
{"type": "text", "text": suffix, "cache_control": cache_marker},
|
||||
]
|
||||
return 2
|
||||
|
||||
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
|
||||
return 1
|
||||
|
||||
|
||||
def strip_anthropic_cache_control(
|
||||
api_messages: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Remove ``cache_control`` markers and undo decoration-produced list shapes.
|
||||
|
||||
Used before re-applying decoration after a mid-turn provider failover so
|
||||
the mutated, undecorated shape (image shrink / ASCII cleanup / etc.) is
|
||||
preserved while markers match the *new* provider's cache policy (#72626).
|
||||
|
||||
Flattening back to a plain string is restricted to the exact shapes
|
||||
:func:`apply_anthropic_cache_control` produces from string content —
|
||||
a single ``{"type": "text"}`` part, or the two-part ``[static, volatile]``
|
||||
system split — so the ``""``-join is provably byte-exact. Organic
|
||||
multi-part text (merged user turns, imported transcripts) and parts
|
||||
carrying extra keys (``citations`` etc.) keep their structure; only
|
||||
per-part markers are removed. Marker removal is copy-on-write on the
|
||||
part dicts: content parts may alias the persistent conversation history
|
||||
(the per-call copy is shallow), and stripping must never rewrite the
|
||||
stored transcript.
|
||||
|
||||
Mutates the top-level message dicts of ``api_messages`` in place and
|
||||
returns the same list.
|
||||
"""
|
||||
for msg in api_messages:
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
msg.pop("cache_control", None)
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
if any(isinstance(part, dict) and "cache_control" in part for part in content):
|
||||
content = [
|
||||
{k: v for k, v in part.items() if k != "cache_control"}
|
||||
if isinstance(part, dict) and "cache_control" in part
|
||||
else part
|
||||
for part in content
|
||||
]
|
||||
msg["content"] = content
|
||||
decoration_shape = content and all(
|
||||
isinstance(part, dict)
|
||||
and part.get("type", "text") == "text"
|
||||
and isinstance(part.get("text"), str)
|
||||
and set(part.keys()) <= {"type", "text"}
|
||||
for part in content
|
||||
) and (
|
||||
len(content) == 1
|
||||
or (msg.get("role") == "system" and len(content) == 2)
|
||||
)
|
||||
if decoration_shape:
|
||||
msg["content"] = "".join(part["text"] for part in content)
|
||||
return api_messages
|
||||
|
||||
|
||||
def apply_anthropic_cache_control(
|
||||
api_messages: List[Dict[str, Any]],
|
||||
cache_ttl: str = "5m",
|
||||
native_anthropic: bool = False,
|
||||
static_system_prefix: str | None = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply system_and_3 caching strategy to messages for Anthropic models.
|
||||
"""Apply Anthropic cache-control markers to API messages.
|
||||
|
||||
Places up to 4 cache_control breakpoints: system prompt + last 3 non-system
|
||||
messages, all at the same TTL.
|
||||
When ``static_system_prefix`` exactly matches the beginning of a string
|
||||
system prompt, it receives an early marker and the full system prompt gets
|
||||
a trailing marker. The remaining two markers target the latest cacheable
|
||||
non-system messages. Without that prefix, the legacy system-and-3 layout
|
||||
is retained.
|
||||
|
||||
Returns:
|
||||
Deep copy of messages with cache_control breakpoints injected.
|
||||
|
|
@ -103,8 +198,12 @@ def apply_anthropic_cache_control(
|
|||
breakpoints_used = 0
|
||||
|
||||
if messages[0].get("role") == "system":
|
||||
_apply_cache_marker(messages[0], marker, native_anthropic=native_anthropic)
|
||||
breakpoints_used += 1
|
||||
breakpoints_used = _apply_system_cache_markers(
|
||||
messages[0],
|
||||
marker,
|
||||
static_system_prefix,
|
||||
native_anthropic=native_anthropic,
|
||||
)
|
||||
|
||||
remaining = 4 - breakpoints_used
|
||||
non_sys = [
|
||||
|
|
|
|||
|
|
@ -706,7 +706,7 @@ def iron_proxy_version(binary: Path) -> str:
|
|||
res = subprocess.run( # noqa: S603
|
||||
[str(binary), "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=_RUN_TIMEOUT,
|
||||
env=minimal_env,
|
||||
)
|
||||
|
|
@ -1052,7 +1052,7 @@ def _detect_docker_bridge_ip() -> Optional[str]:
|
|||
try:
|
||||
res = subprocess.run( # noqa: S603 — ip is a system binary
|
||||
["ip", "-4", "-o", "addr", "show", "docker0"],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2,
|
||||
)
|
||||
if res.returncode == 0:
|
||||
for line in res.stdout.splitlines():
|
||||
|
|
@ -1743,7 +1743,7 @@ def _pid_alive(pid: int) -> bool:
|
|||
try:
|
||||
res = subprocess.run( # noqa: S603
|
||||
["ps", "-p", str(pid), "-o", "comm="],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2,
|
||||
)
|
||||
if res.returncode == 0:
|
||||
comm = (res.stdout or "").strip()
|
||||
|
|
|
|||
|
|
@ -102,9 +102,18 @@ _REASONING_STALE_TIMEOUT_FLOORS: tuple[tuple[str, int], ...] = (
|
|||
# ``claude-opus-4`` so non-thinking Claude 3.x or future
|
||||
# non-reasoning Claude variants don't match.
|
||||
("claude-opus-4", 240),
|
||||
("claude-opus-5", 240),
|
||||
("claude-sonnet-5", 180),
|
||||
("claude-sonnet-4.5", 180),
|
||||
("claude-sonnet-4.6", 180),
|
||||
# Anthropic Mythos-class named reasoning models (claude-fable-5, …).
|
||||
# 1M context + 128K output — heavier thinking phase than the
|
||||
# numbered Claude line, so the floor is in the deep-reasoning tier
|
||||
# alongside o1 / deepseek-r1 / nemotron-3-ultra. Without this
|
||||
# entry the stale-stream detector kills fable-5's thinking phase
|
||||
# at the default 180s (300s with context scaling), tripping the
|
||||
# cross-turn circuit breaker after 5 consecutive stale kills.
|
||||
("claude-fable", 600),
|
||||
# xAI Grok reasoning variants. Explicit reasoning-only keys
|
||||
# plus one for the ``non-reasoning`` variant so users picking
|
||||
# the fast variant don't get the 300s floor. Bare ``grok-3``,
|
||||
|
|
@ -206,6 +215,8 @@ def get_reasoning_stale_timeout_floor(model: object) -> Optional[float]:
|
|||
300.0
|
||||
>>> get_reasoning_stale_timeout_floor("anthropic/claude-opus-4-6")
|
||||
240.0
|
||||
>>> get_reasoning_stale_timeout_floor("anthropic/claude-fable-5")
|
||||
600.0
|
||||
>>> get_reasoning_stale_timeout_floor("gpt-4o") is None
|
||||
True
|
||||
>>> get_reasoning_stale_timeout_floor("olmo-1") is None
|
||||
|
|
|
|||
|
|
@ -175,6 +175,85 @@ _YAML_ASSIGN_RE = re.compile(
|
|||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
# Word-boundary validation for the mixed/lowercase key patterns above
|
||||
# (_CFG_DOTTED_RE, _CFG_ANCHORED_RE, _YAML_ASSIGN_RE).
|
||||
#
|
||||
# Those key classes allow arbitrary alphanumeric affixes around the secret
|
||||
# keyword so real key names like ``client_secret``, ``clientSecret``, and
|
||||
# ``s3.secret-key`` match. The side effect: ordinary prose/document words that
|
||||
# merely CONTAIN a keyword also matched — ``Secretary: J.Smith`` (secret),
|
||||
# ``tokenizer: cl100k_base`` (token), ``author=Smith`` (auth) — mangling
|
||||
# legitimate content on the surfaces that run these passes (browser snapshots,
|
||||
# log lines, kanban summaries, CLI-echoed command output). Ported from
|
||||
# nearai/ironclaw#6129, where the same substring false positive ("Secretary of
|
||||
# the Treasury" matching the ``secret`` marker) scrubbed legitimate tool
|
||||
# results from the replayed transcript and sent the model into a re-fetch
|
||||
# loop.
|
||||
#
|
||||
# A keyword occurrence only counts when it sits at a word boundary within the
|
||||
# key: at the key's edge, next to a non-letter (``_ - . 3``), or at a
|
||||
# camelCase transition (``clientSecret``, ``secretKey``, ``APIToken``). A
|
||||
# trailing plural ``s`` is treated as part of the keyword (``secrets:``,
|
||||
# ``tokens:``). Common concatenated compounds keep matching via explicit
|
||||
# alternatives (``authtoken`` ngrok, ``authkey`` tailscale, ``secretkey``
|
||||
# minio, ``apikey``). Embedded occurrences inside a larger word
|
||||
# (``secretary``, ``tokenizer``, ``authored``, ``credentialing``) no longer
|
||||
# match. ALL-CAPS keys keep the legacy embedded matching (``MYTOKEN=…``) — an
|
||||
# all-caps key is almost never prose, the same rationale as _ENV_ASSIGN_RE.
|
||||
_KEY_KEYWORD_RE = re.compile(
|
||||
r"(?:api|auth|access|refresh|session|secret)[ _.\-]?(?:key|token)"
|
||||
r"|token|secret|passwd|password|credential|auth",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_word_start(s: str, i: int) -> bool:
|
||||
"""True if position ``i`` in ``s`` begins a word (not mid-word)."""
|
||||
if i == 0:
|
||||
return True
|
||||
prev, cur = s[i - 1], s[i]
|
||||
if not prev.isalpha():
|
||||
return True
|
||||
if cur.isupper() and prev.islower():
|
||||
return True # camelCase: clientSecret
|
||||
# Acronym run ending: APIToken — the 'T' begins a new word when it is
|
||||
# followed by lowercase while the preceding run is uppercase.
|
||||
if cur.isupper() and prev.isupper() and i + 1 < len(s) and s[i + 1].islower():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_word_end(s: str, j: int, *, allow_plural: bool = True) -> bool:
|
||||
"""True if position ``j`` (exclusive end) in ``s`` ends a word."""
|
||||
if j >= len(s):
|
||||
return True
|
||||
cur = s[j]
|
||||
if not cur.isalpha():
|
||||
return True
|
||||
if cur.isupper() and s[j - 1].islower():
|
||||
return True # camelCase continuation: secretKey
|
||||
if allow_plural and cur in "sS":
|
||||
return _is_word_end(s, j + 1, allow_plural=False)
|
||||
return False
|
||||
|
||||
|
||||
def _key_has_secret_keyword(key: str) -> bool:
|
||||
"""True if ``key`` contains a secret keyword at a word boundary.
|
||||
|
||||
Post-match validator for _CFG_DOTTED_RE / _CFG_ANCHORED_RE /
|
||||
_YAML_ASSIGN_RE hits — rejects prose words that merely embed a keyword
|
||||
(``secretary``, ``tokenizer``, ``authored``). Safe to call with the
|
||||
_ENV_ASSIGN_RE key too: all-caps keys short-circuit to the legacy
|
||||
embedded-match behavior.
|
||||
"""
|
||||
letters = [c for c in key if c.isalpha()]
|
||||
if letters and all(c.isupper() for c in letters):
|
||||
return True # legacy all-caps behavior (MYTOKEN=…)
|
||||
for m in _KEY_KEYWORD_RE.finditer(key):
|
||||
if _is_word_start(key, m.start()) and _is_word_end(key, m.end()):
|
||||
return True
|
||||
return False
|
||||
|
||||
# JSON field patterns: "apiKey": "value", "token": "value", etc.
|
||||
_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
|
||||
_JSON_FIELD_RE = re.compile(
|
||||
|
|
@ -614,6 +693,13 @@ def redact_sensitive_text(
|
|||
# prose/log contexts (issue #2852): ``KEY=os.getenv('X')``.
|
||||
if _ENV_LOOKUP_VALUE_RE.match(value):
|
||||
return m.group(0)
|
||||
# Keyword must sit at a word boundary within the key —
|
||||
# ``author=Smith`` / ``press.secretary=…`` are prose, not
|
||||
# credentials (ported from nearai/ironclaw#6129). All-caps
|
||||
# keys (the _ENV_ASSIGN_RE shape) short-circuit to legacy
|
||||
# embedded matching inside the helper.
|
||||
if not _key_has_secret_keyword(name):
|
||||
return m.group(0)
|
||||
return f"{name}={quote}{_mask_token(value)}{quote}"
|
||||
text = _ENV_ASSIGN_RE.sub(_redact_env, text)
|
||||
# Lowercase/dotted config keys (issue #16413). Skip URLs entirely —
|
||||
|
|
@ -647,6 +733,11 @@ def redact_sensitive_text(
|
|||
# not a leaked secret value.
|
||||
if _ENV_LOOKUP_VALUE_RE.match(value):
|
||||
return m.group(0)
|
||||
# Keyword must sit at a word boundary within the key —
|
||||
# ``Secretary: J.Smith`` / ``tokenizer: cl100k_base`` are
|
||||
# document text, not credentials (nearai/ironclaw#6129).
|
||||
if not _key_has_secret_keyword(key):
|
||||
return m.group(0)
|
||||
return f"{key}{sep}{_mask_token(value)}"
|
||||
text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
|
||||
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ def run_secret_cli(
|
|||
list(argv),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=timeout,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -200,7 +200,7 @@ def _platform_asset_name() -> str:
|
|||
res = subprocess.run(
|
||||
["ldd", "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=2,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
@ -684,7 +684,7 @@ def _run_bws_list(
|
|||
cmd,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=_BWS_RUN_TIMEOUT,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ emitted by each built-in hook site.
|
|||
child_role – role string of the child agent
|
||||
child_summary – summary of the child's work
|
||||
child_status – exit status string (e.g. "success", "error")
|
||||
tool_call_history – redacted tool name/input summary/byte counts/status list
|
||||
duration_ms – wall-clock time of the child run in milliseconds
|
||||
"""
|
||||
|
||||
|
|
@ -464,7 +465,7 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
|
|||
input=stdin_json,
|
||||
capture_output=True,
|
||||
timeout=spec.timeout,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
shell=False,
|
||||
**_popen_kwargs,
|
||||
)
|
||||
|
|
@ -632,7 +633,7 @@ def allowlist_path() -> Path:
|
|||
def load_allowlist() -> Dict[str, Any]:
|
||||
"""Return the parsed allowlist, or an empty skeleton if absent."""
|
||||
try:
|
||||
raw = json.loads(allowlist_path().read_text())
|
||||
raw = json.loads(allowlist_path().read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
return {"approvals": []}
|
||||
if not isinstance(raw, dict):
|
||||
|
|
|
|||
|
|
@ -54,6 +54,21 @@ _BUNDLE_MARKER = " skill bundle,"
|
|||
_BUNDLE_USER_INSTRUCTION = "\nUser instruction: "
|
||||
_BUNDLE_FIRST_SKILL_BLOCK = "\n\n[Loaded as part of the "
|
||||
|
||||
# The skill name sits in the first quoted span of the activation note, for both
|
||||
# the single-skill and the bundle header ("work" / "/clean /work").
|
||||
_SKILL_NAME_RE = re.compile(re.escape(_SKILL_INVOCATION_PREFIX) + r'"([^"]*)"')
|
||||
|
||||
# SQL LIKE pattern matching a skill-expanded turn, for listing queries that
|
||||
# have to recognize scaffolding before the row reaches Python. The prefix
|
||||
# contains no LIKE wildcards (`%`, `_`), so it needs no ESCAPE clause.
|
||||
SKILL_SCAFFOLD_SQL_LIKE = _SKILL_INVOCATION_PREFIX + "%"
|
||||
|
||||
# Marks where a preview query joined the head and tail of a long scaffolded
|
||||
# message. ``describe_skill_invocation`` may hand back a span that runs across
|
||||
# the joint (a bundle instruction cut off by the head window); callers cut the
|
||||
# description there rather than show the skill body on the far side.
|
||||
SKILL_EXCERPT_JOINT = "\x1e"
|
||||
|
||||
|
||||
def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
|
||||
"""Recover the user's instruction from a slash-skill-expanded turn.
|
||||
|
|
@ -82,6 +97,41 @@ def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]:
|
|||
return None
|
||||
|
||||
|
||||
def describe_skill_invocation(content: Any) -> Optional[str]:
|
||||
"""Render a slash-skill-expanded turn the way the user typed it.
|
||||
|
||||
The expanded message embeds the whole skill body, so any surface that
|
||||
summarizes a user turn from its raw content — session titles, sidebar
|
||||
previews, the ``/rewind`` picker — otherwise shows the skill's own prose
|
||||
as if the user had written it. That is how a skill's opening line ends up
|
||||
as a session title.
|
||||
|
||||
Returns ``"/work — fix the title leak"``, or ``"/work"`` for a bare
|
||||
invocation, or ``None`` when *content* is not skill scaffolding (the
|
||||
caller should then summarize it as an ordinary message).
|
||||
"""
|
||||
if not isinstance(content, str) or not content.startswith(_SKILL_INVOCATION_PREFIX):
|
||||
return None
|
||||
|
||||
match = _SKILL_NAME_RE.match(content)
|
||||
name = (match.group(1) if match else "").strip()
|
||||
# Bundle headers already carry their typed "/a /b" keys; a single skill is
|
||||
# a bare name.
|
||||
label = name if name.startswith("/") else f"/{name}"
|
||||
|
||||
instruction = extract_user_instruction_from_skill_message(content)
|
||||
if instruction and instruction is not content:
|
||||
# An excerpted message (head + tail, joined by SKILL_EXCERPT_JOINT) can
|
||||
# put the joint inside the matched span — keep only the side the
|
||||
# instruction marker was found on.
|
||||
instruction = instruction.split(SKILL_EXCERPT_JOINT)[0]
|
||||
instruction = " ".join(instruction.split())
|
||||
if instruction:
|
||||
return f"{label} — {instruction}" if name else instruction
|
||||
|
||||
return label if name else None
|
||||
|
||||
|
||||
def _extract_single_skill_user_instruction(message: str) -> Optional[str]:
|
||||
# Single-skill format appends the user instruction after the skill body, so
|
||||
# the last occurrence is the user-provided one; the body may quote this text.
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ def run_inline_shell(command: str, cwd: Path | None, timeout: int) -> str:
|
|||
["bash", "-c", command],
|
||||
cwd=str(cwd) if cwd else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=max(1, int(timeout)),
|
||||
check=False,
|
||||
stdin=subprocess.DEVNULL,
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ EXCLUDED_SKILL_DIRS = frozenset(
|
|||
SKILL_SUPPORT_DIRS = frozenset(("references", "templates", "assets", "scripts"))
|
||||
|
||||
|
||||
def is_excluded_skill_path(path) -> bool:
|
||||
def is_excluded_skill_path(path, *, root: Optional[Path] = None) -> bool:
|
||||
"""True if *path* should be skipped by active skill scanners.
|
||||
|
||||
Use this on every ``SKILL.md`` path produced by direct ``rglob`` scans to
|
||||
|
|
@ -66,11 +66,11 @@ def is_excluded_skill_path(path) -> bool:
|
|||
from pathlib import PurePath
|
||||
parts = PurePath(str(path)).parts
|
||||
return any(part in EXCLUDED_SKILL_DIRS for part in parts) or is_skill_support_path(
|
||||
path
|
||||
path, root=root
|
||||
)
|
||||
|
||||
|
||||
def is_skill_support_path(path) -> bool:
|
||||
def is_skill_support_path(path, *, root: Optional[Path] = None) -> bool:
|
||||
"""True if *path* is under a support dir of an actual skill root.
|
||||
|
||||
``references/``, ``templates/``, ``assets/``, and ``scripts/`` are
|
||||
|
|
@ -92,6 +92,8 @@ def is_skill_support_path(path) -> bool:
|
|||
if part not in SKILL_SUPPORT_DIRS or idx == 0:
|
||||
continue
|
||||
skill_root = Path(*parts[:idx])
|
||||
if root is not None and not path_obj.is_absolute():
|
||||
skill_root = root / skill_root
|
||||
if (skill_root / "SKILL.md").exists():
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ def _skip_ssl_guard_enabled() -> bool:
|
|||
|
||||
def _repair_hint() -> str:
|
||||
return (
|
||||
"Repair: python -m pip install --force-reinstall certifi openai httpx\n"
|
||||
"Repair: run `hermes doctor --fix` (auto-reinstalls certifi), or "
|
||||
"manually: python -m pip install --force-reinstall certifi openai httpx\n"
|
||||
"If you configured a custom corporate CA bundle, fix or unset the "
|
||||
"broken CA bundle environment variable."
|
||||
)
|
||||
|
|
|
|||
533
agent/subagent_lifecycle.py
Normal file
533
agent/subagent_lifecycle.py
Normal file
|
|
@ -0,0 +1,533 @@
|
|||
"""Public, plugin-safe lifecycle API for delegated Hermes subagents.
|
||||
|
||||
This module deliberately exposes immutable contracts, not ``AIAgent`` objects.
|
||||
It is the supported boundary for plugins that need to supervise fresh child
|
||||
sessions; plugins must obtain it from ``PluginContext.subagent_lifecycle``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import dataclasses
|
||||
import enum
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import math
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, TimeoutError
|
||||
from typing import Any, Callable, Mapping, Optional
|
||||
|
||||
|
||||
PUBLIC_CONTRACT_VERSION = 1
|
||||
_MAX_GOAL_CHARS = 16_000
|
||||
_MAX_CONTEXT_CHARS = 32_000
|
||||
_MAX_METADATA_BYTES = 8_192
|
||||
_MAX_RESULT_CHARS = 32_000
|
||||
_TERMINAL_RETENTION_SECONDS = 3_600
|
||||
|
||||
|
||||
class SubagentLifecycleError(ValueError):
|
||||
"""A request cannot be safely accepted by the public lifecycle API."""
|
||||
|
||||
|
||||
class SubagentState(str, enum.Enum):
|
||||
PENDING = "PENDING"
|
||||
STARTING = "STARTING"
|
||||
RUNNING = "RUNNING"
|
||||
SUCCEEDED = "SUCCEEDED"
|
||||
FAILED = "FAILED"
|
||||
INTERRUPTED = "INTERRUPTED"
|
||||
CANCEL_REQUESTED = "CANCEL_REQUESTED"
|
||||
CANCELLED = "CANCELLED"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentLaunchRequest:
|
||||
goal: str
|
||||
context: Optional[str] = None
|
||||
role: str = "leaf"
|
||||
model: Optional[str] = None
|
||||
allowed_toolsets: Optional[tuple[str, ...]] = None
|
||||
blocked_tools: tuple[str, ...] = ()
|
||||
working_directory: Optional[str] = None
|
||||
parent_session_id: Optional[str] = None
|
||||
correlation_id: Optional[str] = None
|
||||
metadata: Mapping[str, Any] = dataclasses.field(default_factory=dict)
|
||||
timeout_seconds: Optional[float] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentHandle:
|
||||
contract_version: int
|
||||
subagent_id: str
|
||||
parent_session_id: Optional[str]
|
||||
correlation_id: Optional[str]
|
||||
created_at: float
|
||||
provider: Optional[str]
|
||||
model: Optional[str]
|
||||
role: str
|
||||
depth: int
|
||||
capability: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return dataclasses.asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "SubagentHandle":
|
||||
try:
|
||||
return cls(**dict(value))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise SubagentLifecycleError("Malformed subagent handle.") from exc
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentStatus:
|
||||
handle: SubagentHandle
|
||||
state: SubagentState
|
||||
updated_at: float
|
||||
diagnostic: Optional[str] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentTerminalState:
|
||||
handle: SubagentHandle
|
||||
state: SubagentState
|
||||
completed: bool
|
||||
timed_out: bool = False
|
||||
diagnostic: Optional[str] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentCancelResult:
|
||||
accepted: bool
|
||||
already_terminal: bool = False
|
||||
unknown_handle: bool = False
|
||||
unsupported: bool = False
|
||||
state: SubagentState = SubagentState.UNKNOWN
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentResult:
|
||||
handle: SubagentHandle
|
||||
terminal_state: SubagentState
|
||||
ready: bool
|
||||
summary: Optional[str] = None
|
||||
structured_payload: Optional[Mapping[str, Any]] = None
|
||||
started_at: Optional[float] = None
|
||||
completed_at: Optional[float] = None
|
||||
error_classification: Optional[str] = None
|
||||
error_message: Optional[str] = None
|
||||
usage_metadata: Mapping[str, Any] = dataclasses.field(default_factory=dict)
|
||||
tool_execution_summary: Mapping[str, Any] = dataclasses.field(default_factory=dict)
|
||||
result_hash: Optional[str] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SubagentReconnectResult:
|
||||
connected: bool
|
||||
state: SubagentState
|
||||
diagnostic: Optional[str] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _Record:
|
||||
handle: SubagentHandle
|
||||
state: SubagentState
|
||||
updated_at: float
|
||||
agent: Any = None
|
||||
future: Optional[Future] = None
|
||||
started_at: Optional[float] = None
|
||||
completed_at: Optional[float] = None
|
||||
result: Optional[SubagentResult] = None
|
||||
|
||||
|
||||
class _Registry:
|
||||
"""Thread-safe terminal-retention registry; never returns live records."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.RLock()
|
||||
self.records: dict[str, _Record] = {}
|
||||
self.correlations: dict[tuple[Optional[str], str], str] = {}
|
||||
|
||||
|
||||
_REGISTRY = _Registry()
|
||||
# Daemon worker pool: a wedged/abandoned child must never block interpreter
|
||||
# exit at atexit-join time (same rationale as _run_single_child's timeout
|
||||
# executor and the async-delegation registry pool).
|
||||
from tools.daemon_pool import DaemonThreadPoolExecutor as _DaemonExecutor
|
||||
|
||||
_EXECUTOR = _DaemonExecutor(max_workers=8, thread_name_prefix="hermes-lifecycle")
|
||||
_SECRET = secrets.token_bytes(32)
|
||||
_ACTIVE_PARENT_AGENT: contextvars.ContextVar[Any] = contextvars.ContextVar(
|
||||
"hermes_subagent_lifecycle_parent", default=None
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def bind_subagent_parent(parent_agent: Any):
|
||||
"""Bind the host-owned parent for the current agent turn."""
|
||||
token = _ACTIVE_PARENT_AGENT.set(parent_agent)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_ACTIVE_PARENT_AGENT.reset(token)
|
||||
|
||||
|
||||
def get_active_subagent_parent() -> Any:
|
||||
"""Return the parent bound to this execution context, if any."""
|
||||
return _ACTIVE_PARENT_AGENT.get()
|
||||
|
||||
|
||||
class SubagentLifecycleService:
|
||||
"""Stable public service returned by :attr:`PluginContext.subagent_lifecycle`.
|
||||
|
||||
Running children are in-process only. Completed results remain available
|
||||
until process exit; ``reconnect`` accurately reports that a serialized
|
||||
handle cannot reconnect after a restart instead of launching work again.
|
||||
"""
|
||||
|
||||
def __init__(self, parent_agent_resolver: Callable[[], Any]) -> None:
|
||||
self._parent_agent_resolver = parent_agent_resolver
|
||||
|
||||
def launch(self, request: SubagentLaunchRequest) -> SubagentHandle:
|
||||
parent = self._parent_agent_resolver()
|
||||
if parent is None:
|
||||
raise SubagentLifecycleError(
|
||||
"No active Hermes parent session is available."
|
||||
)
|
||||
self._validate_request(request, parent)
|
||||
parent_session_id = str(getattr(parent, "session_id", "") or "") or None
|
||||
if request.parent_session_id and request.parent_session_id != parent_session_id:
|
||||
raise SubagentLifecycleError(
|
||||
"parent_session_id does not match the active session."
|
||||
)
|
||||
correlation_key = (parent_session_id, request.correlation_id or "")
|
||||
with _REGISTRY.lock:
|
||||
self._cleanup_locked()
|
||||
if request.correlation_id and correlation_key in _REGISTRY.correlations:
|
||||
raise SubagentLifecycleError(
|
||||
"Duplicate correlation_id for this parent session."
|
||||
)
|
||||
|
||||
# Delegate construction remains internal so plugin code never imports
|
||||
# private delegation helpers or manipulates the active-child registry.
|
||||
from tools.delegate_tool import (
|
||||
_build_child_preserving_parent_tools,
|
||||
DEFAULT_MAX_ITERATIONS,
|
||||
)
|
||||
|
||||
child = _build_child_preserving_parent_tools(
|
||||
task_index=0,
|
||||
goal=request.goal,
|
||||
context=request.context,
|
||||
toolsets=list(request.allowed_toolsets)
|
||||
if request.allowed_toolsets
|
||||
else None,
|
||||
model=request.model,
|
||||
max_iterations=DEFAULT_MAX_ITERATIONS,
|
||||
task_count=1,
|
||||
parent_agent=parent,
|
||||
role=request.role,
|
||||
)
|
||||
subagent_id = str(getattr(child, "_subagent_id", "") or "")
|
||||
if not subagent_id:
|
||||
raise SubagentLifecycleError("Hermes failed to assign a child identity.")
|
||||
created = time.time()
|
||||
handle = SubagentHandle(
|
||||
PUBLIC_CONTRACT_VERSION,
|
||||
subagent_id,
|
||||
parent_session_id,
|
||||
request.correlation_id,
|
||||
created,
|
||||
getattr(child, "provider", None),
|
||||
getattr(child, "model", None),
|
||||
getattr(child, "_delegate_role", request.role),
|
||||
int(getattr(child, "_delegate_depth", 1) or 1),
|
||||
self._capability(subagent_id, parent_session_id, created),
|
||||
)
|
||||
record = _Record(handle, SubagentState.PENDING, created, agent=child)
|
||||
with _REGISTRY.lock:
|
||||
_REGISTRY.records[subagent_id] = record
|
||||
if request.correlation_id:
|
||||
_REGISTRY.correlations[correlation_key] = subagent_id
|
||||
record.future = _EXECUTOR.submit(self._run, record, request.goal, parent)
|
||||
return handle
|
||||
|
||||
def status(self, handle: SubagentHandle) -> SubagentStatus:
|
||||
record = self._record(handle)
|
||||
if record is None:
|
||||
return SubagentStatus(
|
||||
handle, SubagentState.UNKNOWN, time.time(), "UNKNOWN_HANDLE"
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
return SubagentStatus(record.handle, record.state, record.updated_at)
|
||||
|
||||
def wait(
|
||||
self, handle: SubagentHandle, *, timeout_seconds: Optional[float] = None
|
||||
) -> SubagentTerminalState:
|
||||
record = self._record(handle)
|
||||
if record is None:
|
||||
return SubagentTerminalState(
|
||||
handle, SubagentState.UNKNOWN, True, diagnostic="UNKNOWN_HANDLE"
|
||||
)
|
||||
future = record.future
|
||||
if future is not None:
|
||||
try:
|
||||
future.result(timeout=timeout_seconds)
|
||||
except TimeoutError:
|
||||
return SubagentTerminalState(record.handle, record.state, False, True)
|
||||
except Exception:
|
||||
pass
|
||||
with _REGISTRY.lock:
|
||||
return SubagentTerminalState(
|
||||
record.handle, record.state, record.result is not None
|
||||
)
|
||||
|
||||
def cancel(self, handle: SubagentHandle, *, reason: str) -> SubagentCancelResult:
|
||||
record = self._record(handle)
|
||||
if record is None:
|
||||
return SubagentCancelResult(False, unknown_handle=True)
|
||||
with _REGISTRY.lock:
|
||||
if record.result is not None:
|
||||
return SubagentCancelResult(
|
||||
False, already_terminal=True, state=record.state
|
||||
)
|
||||
agent = record.agent
|
||||
record.state = SubagentState.CANCEL_REQUESTED
|
||||
record.updated_at = time.time()
|
||||
if agent is None or not hasattr(agent, "interrupt"):
|
||||
return SubagentCancelResult(
|
||||
False, unsupported=True, state=SubagentState.CANCEL_REQUESTED
|
||||
)
|
||||
try:
|
||||
agent.interrupt(f"Lifecycle cancellation requested: {reason[:500]}")
|
||||
except Exception:
|
||||
return SubagentCancelResult(
|
||||
False, unsupported=True, state=SubagentState.CANCEL_REQUESTED
|
||||
)
|
||||
return SubagentCancelResult(True, state=SubagentState.CANCEL_REQUESTED)
|
||||
|
||||
def result(self, handle: SubagentHandle) -> SubagentResult:
|
||||
record = self._record(handle)
|
||||
if record is None:
|
||||
return SubagentResult(
|
||||
handle,
|
||||
SubagentState.UNKNOWN,
|
||||
False,
|
||||
error_classification="UNKNOWN_HANDLE",
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
if record.result is not None:
|
||||
return record.result
|
||||
return SubagentResult(
|
||||
record.handle, record.state, False, error_classification="NOT_READY"
|
||||
)
|
||||
|
||||
def reconnect(self, handle: SubagentHandle) -> SubagentReconnectResult:
|
||||
record = self._record(handle)
|
||||
if record is None:
|
||||
return SubagentReconnectResult(
|
||||
False, SubagentState.UNKNOWN, "RECONNECT_UNAVAILABLE"
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
return SubagentReconnectResult(True, record.state)
|
||||
|
||||
def _record(self, handle: SubagentHandle) -> Optional[_Record]:
|
||||
if (
|
||||
not isinstance(handle, SubagentHandle)
|
||||
or type(handle.contract_version) is not int
|
||||
or handle.contract_version != PUBLIC_CONTRACT_VERSION
|
||||
):
|
||||
return None
|
||||
if (
|
||||
not isinstance(handle.subagent_id, str)
|
||||
or not handle.subagent_id
|
||||
or (
|
||||
handle.parent_session_id is not None
|
||||
and not isinstance(handle.parent_session_id, str)
|
||||
)
|
||||
or (
|
||||
handle.correlation_id is not None
|
||||
and not isinstance(handle.correlation_id, str)
|
||||
)
|
||||
or isinstance(handle.created_at, bool)
|
||||
or not isinstance(handle.created_at, (int, float))
|
||||
or not math.isfinite(handle.created_at)
|
||||
or (handle.provider is not None and not isinstance(handle.provider, str))
|
||||
or (handle.model is not None and not isinstance(handle.model, str))
|
||||
or not isinstance(handle.role, str)
|
||||
or type(handle.depth) is not int
|
||||
or not isinstance(handle.capability, str)
|
||||
):
|
||||
return None
|
||||
if not hmac.compare_digest(
|
||||
handle.capability,
|
||||
self._capability(
|
||||
handle.subagent_id, handle.parent_session_id, handle.created_at
|
||||
),
|
||||
):
|
||||
return None
|
||||
parent = self._parent_agent_resolver()
|
||||
active_parent_id = str(getattr(parent, "session_id", "") or "") or None
|
||||
if active_parent_id != handle.parent_session_id:
|
||||
return None
|
||||
with _REGISTRY.lock:
|
||||
return _REGISTRY.records.get(handle.subagent_id)
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_locked() -> None:
|
||||
"""Retain terminal snapshots for a bounded period, never live work."""
|
||||
cutoff = time.time() - _TERMINAL_RETENTION_SECONDS
|
||||
expired = [
|
||||
subagent_id
|
||||
for subagent_id, record in _REGISTRY.records.items()
|
||||
if record.result is not None
|
||||
and record.completed_at is not None
|
||||
and record.completed_at < cutoff
|
||||
]
|
||||
for subagent_id in expired:
|
||||
record = _REGISTRY.records.pop(subagent_id)
|
||||
if record.handle.correlation_id:
|
||||
_REGISTRY.correlations.pop(
|
||||
(record.handle.parent_session_id, record.handle.correlation_id),
|
||||
None,
|
||||
)
|
||||
|
||||
def _run(self, record: _Record, goal: str, parent: Any) -> None:
|
||||
with _REGISTRY.lock:
|
||||
if record.state is not SubagentState.CANCEL_REQUESTED:
|
||||
record.state = SubagentState.RUNNING
|
||||
record.started_at = time.time()
|
||||
record.updated_at = record.started_at
|
||||
try:
|
||||
from tools.delegate_tool import _run_child_lifecycle
|
||||
|
||||
raw = _run_child_lifecycle(0, goal, record.agent, parent)
|
||||
status = (
|
||||
str(raw.get("status", "error")) if isinstance(raw, dict) else "error"
|
||||
)
|
||||
if status == "completed":
|
||||
state = SubagentState.SUCCEEDED
|
||||
elif status == "interrupted":
|
||||
state = (
|
||||
SubagentState.CANCELLED
|
||||
if record.state == SubagentState.CANCEL_REQUESTED
|
||||
else SubagentState.INTERRUPTED
|
||||
)
|
||||
else:
|
||||
state = SubagentState.FAILED
|
||||
summary = raw.get("summary") if isinstance(raw, dict) else None
|
||||
summary = str(summary)[:_MAX_RESULT_CHARS] if summary is not None else None
|
||||
error = raw.get("error") if isinstance(raw, dict) else None
|
||||
result = SubagentResult(
|
||||
record.handle,
|
||||
state,
|
||||
True,
|
||||
summary=summary,
|
||||
completed_at=time.time(),
|
||||
started_at=record.started_at,
|
||||
error_classification=None
|
||||
if state == SubagentState.SUCCEEDED
|
||||
else status.upper(),
|
||||
error_message=str(error)[:_MAX_RESULT_CHARS] if error else None,
|
||||
usage_metadata={"api_calls": raw.get("api_calls", 0)}
|
||||
if isinstance(raw, dict)
|
||||
else {},
|
||||
tool_execution_summary={
|
||||
"duration_seconds": raw.get("duration_seconds", 0)
|
||||
}
|
||||
if isinstance(raw, dict)
|
||||
else {},
|
||||
)
|
||||
except Exception as exc:
|
||||
result = SubagentResult(
|
||||
record.handle,
|
||||
SubagentState.FAILED,
|
||||
True,
|
||||
started_at=record.started_at,
|
||||
completed_at=time.time(),
|
||||
error_classification=type(exc).__name__,
|
||||
error_message=str(exc)[:_MAX_RESULT_CHARS],
|
||||
)
|
||||
payload = dataclasses.asdict(result)
|
||||
payload.pop("result_hash", None)
|
||||
result = dataclasses.replace(
|
||||
result,
|
||||
result_hash=hashlib.sha256(
|
||||
json.dumps(payload, sort_keys=True, default=str).encode()
|
||||
).hexdigest(),
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
record.agent = None
|
||||
record.result = result
|
||||
record.state = result.terminal_state
|
||||
record.completed_at = result.completed_at
|
||||
record.updated_at = result.completed_at or time.time()
|
||||
|
||||
@staticmethod
|
||||
def _capability(
|
||||
subagent_id: str, parent_session_id: Optional[str], created_at: float
|
||||
) -> str:
|
||||
value = f"{subagent_id}|{parent_session_id or ''}|{created_at:.6f}".encode()
|
||||
return hmac.new(_SECRET, value, hashlib.sha256).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _validate_request(request: SubagentLaunchRequest, parent: Any) -> None:
|
||||
if (
|
||||
not isinstance(request, SubagentLaunchRequest)
|
||||
or not isinstance(request.goal, str)
|
||||
or not request.goal.strip()
|
||||
or len(request.goal) > _MAX_GOAL_CHARS
|
||||
):
|
||||
raise SubagentLifecycleError(
|
||||
"goal must be a non-empty string of at most 16000 characters."
|
||||
)
|
||||
if request.context is not None and (
|
||||
not isinstance(request.context, str)
|
||||
or len(request.context) > _MAX_CONTEXT_CHARS
|
||||
):
|
||||
raise SubagentLifecycleError(
|
||||
"context must be a string of at most 32000 characters."
|
||||
)
|
||||
if request.role not in {"leaf", "orchestrator"}:
|
||||
raise SubagentLifecycleError("role must be 'leaf' or 'orchestrator'.")
|
||||
if request.timeout_seconds is not None:
|
||||
raise SubagentLifecycleError(
|
||||
"Per-launch timeout is not supported; configure delegation timeout explicitly."
|
||||
)
|
||||
if request.working_directory is not None:
|
||||
raise SubagentLifecycleError(
|
||||
"working_directory is not supported because Hermes delegates use isolated task environments."
|
||||
)
|
||||
if request.blocked_tools:
|
||||
raise SubagentLifecycleError(
|
||||
"Per-tool blocking is not supported; use allowed_toolsets. Hermes always blocks unsafe child tools."
|
||||
)
|
||||
try:
|
||||
metadata_bytes = len(
|
||||
json.dumps(dict(request.metadata), sort_keys=True).encode()
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise SubagentLifecycleError("metadata must be JSON-serializable.") from exc
|
||||
if metadata_bytes > _MAX_METADATA_BYTES:
|
||||
raise SubagentLifecycleError("metadata exceeds 8192 bytes.")
|
||||
if request.allowed_toolsets:
|
||||
from toolsets import TOOLSETS
|
||||
|
||||
unknown = set(request.allowed_toolsets) - set(TOOLSETS)
|
||||
if unknown:
|
||||
raise SubagentLifecycleError(
|
||||
f"Unknown toolsets: {', '.join(sorted(unknown))}."
|
||||
)
|
||||
enabled = getattr(parent, "enabled_toolsets", None)
|
||||
if enabled is not None and not set(request.allowed_toolsets).issubset(
|
||||
set(enabled)
|
||||
):
|
||||
raise SubagentLifecycleError(
|
||||
"Requested toolsets would broaden parent permissions."
|
||||
)
|
||||
|
|
@ -12,9 +12,11 @@ Three tiers are joined with ``\\n\\n``:
|
|||
* ``stable`` — identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool
|
||||
guidance, computer-use guidance, nous subscription block, tool-use
|
||||
enforcement guidance + per-model operational guidance, skills prompt,
|
||||
alibaba model-name workaround, environment hints, platform hints.
|
||||
alibaba model-name workaround, environment hints, coding guidance,
|
||||
platform hints.
|
||||
* ``context`` — caller-supplied ``system_message`` plus context files
|
||||
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``.
|
||||
(AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``,
|
||||
plus the session's coding-workspace snapshot.
|
||||
* ``volatile`` — memory snapshot, USER.md profile, external memory
|
||||
provider block, timestamp/session/model/provider line.
|
||||
|
||||
|
|
@ -24,6 +26,7 @@ Pure helpers that read the agent's state. AIAgent keeps thin forwarders.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
|
@ -49,6 +52,8 @@ from agent.runtime_cwd import resolve_context_cwd
|
|||
from hermes_constants import get_hermes_home
|
||||
from utils import is_truthy_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ra():
|
||||
"""Lazy reference to the ``run_agent`` module.
|
||||
|
|
@ -145,14 +150,14 @@ def _tui_embedded_pane_clarifier(hint: str) -> str:
|
|||
|
||||
|
||||
def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]:
|
||||
"""Assemble the system prompt as three ordered parts.
|
||||
"""Assemble the system prompt as three ordered cache tiers.
|
||||
|
||||
Returns a dict with three keys:
|
||||
* ``stable`` — identity, tool guidance, skills prompt,
|
||||
environment hints, platform hints, model-family operational
|
||||
guidance.
|
||||
* ``context`` — context files (AGENTS.md, .cursorrules, etc.)
|
||||
and caller-supplied system_message.
|
||||
* ``stable`` — the cross-session-stable prefix, through the coding
|
||||
operating brief when a workspace snapshot follows.
|
||||
* ``context`` — the workspace snapshot followed by the remaining
|
||||
session-stable guidance, context files, and caller-supplied
|
||||
system_message.
|
||||
* ``volatile`` — memory snapshot, user profile, external
|
||||
memory provider block, timestamp line.
|
||||
|
||||
|
|
@ -345,25 +350,35 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
stable_parts.append(_env_hints)
|
||||
|
||||
# Coding posture (base Hermes, any interactive coding surface in a code
|
||||
# workspace — see agent/coding_context.py). The operating brief + the live
|
||||
# git/workspace snapshot are built once here and cached for the session;
|
||||
# the snapshot is never re-probed per turn (that would break the prompt
|
||||
# cache), so the brief tells the model to re-check git before relying on it.
|
||||
# workspace — see agent/coding_context.py). Keep the operating brief in
|
||||
# the cross-session-stable prefix, while placing the live git/workspace
|
||||
# snapshot behind its own cache boundary. The post-snapshot blocks must
|
||||
# stay in their historical position after the workspace snapshot.
|
||||
coding_workspace_parts: List[str] = []
|
||||
coding_trailing_parts: List[str] = []
|
||||
if agent.valid_tool_names:
|
||||
try:
|
||||
from agent.coding_context import coding_system_blocks
|
||||
from agent.coding_context import coding_system_prompt_parts
|
||||
|
||||
stable_parts.extend(
|
||||
coding_system_blocks(
|
||||
platform=agent.platform,
|
||||
cwd=resolve_context_cwd(),
|
||||
model=agent.model,
|
||||
)
|
||||
coding_prefix_parts, coding_workspace_parts, coding_trailing_parts = coding_system_prompt_parts(
|
||||
platform=agent.platform,
|
||||
cwd=resolve_context_cwd(),
|
||||
model=agent.model,
|
||||
)
|
||||
stable_parts.extend(coding_prefix_parts)
|
||||
except Exception:
|
||||
# Coding-context probing must never block prompt build.
|
||||
pass
|
||||
|
||||
# Guidance assembled after the coding posture historically followed the
|
||||
# workspace snapshot. With no snapshot, the coding tail instead remains
|
||||
# directly after the coding prefix in the cacheable prefix.
|
||||
if coding_workspace_parts:
|
||||
post_workspace_parts: List[str] = []
|
||||
else:
|
||||
stable_parts.extend(coding_trailing_parts)
|
||||
post_workspace_parts = stable_parts
|
||||
|
||||
# Local Python toolchain probe — names python/pip/uv/PEP-668 state when
|
||||
# something is non-default so the model can pick the right install
|
||||
# strategy without discovering by failure. Emits a single line; emits
|
||||
|
|
@ -376,7 +391,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
from tools.env_probe import get_environment_probe_line
|
||||
_probe_line = get_environment_probe_line()
|
||||
if _probe_line:
|
||||
stable_parts.append(_probe_line)
|
||||
post_workspace_parts.append(_probe_line)
|
||||
except Exception:
|
||||
# Probe failure must never block prompt build.
|
||||
pass
|
||||
|
|
@ -394,7 +409,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
except Exception:
|
||||
active_profile = "default"
|
||||
if active_profile == "default":
|
||||
stable_parts.append(
|
||||
post_workspace_parts.append(
|
||||
"Active Hermes profile: default. Other profiles (if any) live "
|
||||
"under " + str(get_hermes_home()) + "/profiles/<name>/. Each profile has its own "
|
||||
"skills/, plugins/, cron/, and memories/ that affect a different "
|
||||
|
|
@ -403,7 +418,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
"you to."
|
||||
)
|
||||
else:
|
||||
stable_parts.append(
|
||||
post_workspace_parts.append(
|
||||
f"Active Hermes profile: {active_profile}. This session reads "
|
||||
f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default "
|
||||
f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, "
|
||||
|
|
@ -449,11 +464,16 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
if platform_key == "tui" and _effective_hint:
|
||||
_effective_hint = _tui_embedded_pane_clarifier(_effective_hint)
|
||||
if _effective_hint:
|
||||
stable_parts.append(_effective_hint)
|
||||
post_workspace_parts.append(_effective_hint)
|
||||
|
||||
# ── Context tier (cwd-dependent, may change between sessions) ─
|
||||
context_parts: List[str] = []
|
||||
|
||||
if coding_workspace_parts:
|
||||
context_parts.extend(coding_workspace_parts)
|
||||
context_parts.extend(coding_trailing_parts)
|
||||
context_parts.extend(post_workspace_parts)
|
||||
|
||||
# Note: ephemeral_system_prompt is NOT included here. It's injected at
|
||||
# API-call time only so it stays out of the cached/stored system prompt.
|
||||
if system_message is not None:
|
||||
|
|
@ -515,6 +535,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
|
|||
timestamp_line += f"\nModel: {agent.model}"
|
||||
if agent.provider:
|
||||
timestamp_line += f"\nProvider: {agent.provider}"
|
||||
if agent.platform:
|
||||
timestamp_line += f"\nPlatform: {agent.platform}"
|
||||
volatile_parts.append(timestamp_line)
|
||||
|
||||
return {
|
||||
|
|
@ -541,6 +563,7 @@ def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str
|
|||
"""
|
||||
parts = build_system_prompt_parts(agent, system_message=system_message)
|
||||
joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p)
|
||||
agent._cached_system_prompt_static = parts["stable"]
|
||||
|
||||
# Surface context-file truncation warnings through the normal agent status
|
||||
# channel so gateway/CLI users see them in chat instead of only in logs.
|
||||
|
|
@ -557,10 +580,65 @@ def invalidate_system_prompt(agent: Any) -> None:
|
|||
so the rebuilt prompt captures any writes from this session.
|
||||
"""
|
||||
agent._cached_system_prompt = None
|
||||
agent._cached_system_prompt_static = None
|
||||
if agent._memory_store:
|
||||
agent._memory_store.load_from_disk()
|
||||
|
||||
|
||||
def reconstruct_static_prefix(
|
||||
agent: Any,
|
||||
system_message: Optional[str] = None,
|
||||
*,
|
||||
log_label: str = "restore",
|
||||
) -> None:
|
||||
"""Reconstruct ``_cached_system_prompt_static`` for a stored prompt.
|
||||
|
||||
The static prefix is not persisted (only the full prompt is), so any
|
||||
path that adopts a stored/kept ``_cached_system_prompt`` — session
|
||||
restore, the compression keep-prompt path, or a failover to a cache-on
|
||||
provider mid-turn (#72626) — must rebuild the stable tier to regain the
|
||||
two-block ``[static, volatile]`` system layout.
|
||||
|
||||
Safety: the rebuilt stable tier is used ONLY when the stored prompt
|
||||
literally starts with it (checked here AND re-checked by
|
||||
``_apply_system_cache_markers``'s ``startswith`` gate). If any
|
||||
stable-tier input changed since the prompt was persisted (skills
|
||||
edited, identity changed), the prefix mismatches, the static stays
|
||||
None, and requests fall back to the legacy layout with the stored
|
||||
prompt bytes untouched — never a rewritten prompt.
|
||||
|
||||
A failed reconstruction is memoized per stored prompt
|
||||
(``_static_rebuild_failed_for``): ``build_system_prompt_parts`` does
|
||||
real file I/O (SOUL.md, context files, memory), and callers on the
|
||||
retry-loop hot path must not re-run it every attempt when the inputs
|
||||
haven't changed. A legitimately changed stored prompt retries once.
|
||||
"""
|
||||
if not getattr(agent, "_use_prompt_caching", False):
|
||||
return
|
||||
stored = getattr(agent, "_cached_system_prompt", None)
|
||||
if not isinstance(stored, str) or not stored:
|
||||
return
|
||||
existing = getattr(agent, "_cached_system_prompt_static", None)
|
||||
if isinstance(existing, str) and existing and stored.startswith(existing):
|
||||
return
|
||||
if getattr(agent, "_static_rebuild_failed_for", None) == stored:
|
||||
return
|
||||
try:
|
||||
static = build_system_prompt_parts(agent, system_message=system_message)["stable"]
|
||||
if static and stored.startswith(static):
|
||||
agent._cached_system_prompt_static = static
|
||||
agent._static_rebuild_failed_for = None
|
||||
return
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"static system-prefix reconstruction failed on %s",
|
||||
log_label,
|
||||
exc_info=True,
|
||||
)
|
||||
agent._cached_system_prompt_static = None
|
||||
agent._static_rebuild_failed_for = stored
|
||||
|
||||
|
||||
def format_tools_for_system_message(agent: Any) -> str:
|
||||
"""Format tool definitions for the system message in the trajectory format.
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _summarize_user_message(user_message: str) -> str:
|
||||
"""Collapse a slash-skill-expanded turn back to what the user typed.
|
||||
|
||||
A ``/skill`` invocation expands into a message that embeds the whole skill
|
||||
body, so feeding it to the titler verbatim titles the session after the
|
||||
*skill's* prose — "Kick off a task in a fresh isolated git worktree" — not
|
||||
after the user's request. Reuse the canonical scaffolding parser so the
|
||||
model sees ``/work — fix the title leak`` instead.
|
||||
"""
|
||||
if not user_message:
|
||||
return ""
|
||||
try:
|
||||
from agent.skill_commands import describe_skill_invocation
|
||||
|
||||
described = describe_skill_invocation(user_message)
|
||||
except Exception:
|
||||
logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True)
|
||||
return user_message
|
||||
return described if described is not None else user_message
|
||||
|
||||
|
||||
def generate_title(
|
||||
user_message: str,
|
||||
assistant_response: str,
|
||||
|
|
@ -110,7 +131,7 @@ def generate_title(
|
|||
logger.debug("Title runtime validator raised; proceeding", exc_info=True)
|
||||
|
||||
# Truncate long messages to keep the request small
|
||||
user_snippet = user_message[:500] if user_message else ""
|
||||
user_snippet = _summarize_user_message(user_message)[:500]
|
||||
assistant_snippet = assistant_response[:500] if assistant_response else ""
|
||||
|
||||
language = _title_language()
|
||||
|
|
@ -143,6 +164,11 @@ def generate_title(
|
|||
title = title.strip('"\'')
|
||||
if title.lower().startswith("title:"):
|
||||
title = title[6:].strip()
|
||||
# A title is one line. A model that ignores "return ONLY the title" and
|
||||
# answers the prompt instead (a shell transcript, a bulleted plan) would
|
||||
# otherwise be stored verbatim and truncated mid-command. Keep the first
|
||||
# non-empty line — the closest thing to a title in that response.
|
||||
title = next((line.strip() for line in title.splitlines() if line.strip()), "")
|
||||
# Enforce reasonable length
|
||||
if len(title) > 80:
|
||||
title = title[:77] + "..."
|
||||
|
|
|
|||
|
|
@ -140,8 +140,8 @@ def _flush_session_db_after_tool_progress(
|
|||
messages: list,
|
||||
*,
|
||||
stage: str,
|
||||
) -> None:
|
||||
"""Best-effort incremental SessionDB flush for tool-call progress.
|
||||
) -> bool:
|
||||
"""Flush tool-call progress before projecting it to any UI surface.
|
||||
|
||||
Tool execution can perform side effects that terminate or restart the
|
||||
current Hermes process before the normal turn-end persistence path runs.
|
||||
|
|
@ -149,9 +149,14 @@ def _flush_session_db_after_tool_progress(
|
|||
transcript survives destructive-but-valid tool calls.
|
||||
"""
|
||||
try:
|
||||
agent._flush_messages_to_session_db(messages)
|
||||
persisted = agent._flush_messages_to_session_db(messages) is not False
|
||||
if not persisted:
|
||||
agent._incremental_persistence_failed = True
|
||||
return persisted
|
||||
except Exception as exc:
|
||||
agent._incremental_persistence_failed = True
|
||||
logger.warning("Incremental tool-call persistence failed after %s: %s", stage, exc)
|
||||
return False
|
||||
|
||||
|
||||
def _ra():
|
||||
|
|
@ -431,8 +436,15 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
_underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args)
|
||||
if not _err and _underlying:
|
||||
if _underlying in _tool_search_scoped_names(agent):
|
||||
function_name = _underlying
|
||||
function_args = _underlying_args
|
||||
# Probe-validate before unwrapping (ironclaw#5149):
|
||||
# missing required args return the parameter schema
|
||||
# instead of dispatching into an opaque failure.
|
||||
_probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args)
|
||||
if _probe_err is not None:
|
||||
_ts_scope_block = _probe_err
|
||||
else:
|
||||
function_name = _underlying
|
||||
function_args = _underlying_args
|
||||
else:
|
||||
_ts_scope_block = json.dumps({
|
||||
"error": (
|
||||
|
|
@ -854,6 +866,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls):
|
||||
r = results[i]
|
||||
blocked = False
|
||||
is_error = True
|
||||
progress_function_name = name
|
||||
# A worker can finish and write results[i] in the window between the
|
||||
# deadline snapshot (timed_out_indices, taken from not_done) and this
|
||||
# loop. Prefer that real result over a fabricated timeout message — the
|
||||
|
|
@ -909,6 +923,7 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
tool_duration = 0.0
|
||||
else:
|
||||
function_name, function_args, function_result, tool_duration, is_error, blocked, middleware_trace = r
|
||||
progress_function_name = function_name
|
||||
if blocked:
|
||||
effect_disposition = "none"
|
||||
|
||||
|
|
@ -936,43 +951,15 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
except Exception as _ver_err:
|
||||
logging.debug("file-mutation verifier record failed: %s", _ver_err)
|
||||
|
||||
if not blocked and agent.tool_progress_callback:
|
||||
try:
|
||||
agent.tool_progress_callback(
|
||||
"tool.completed", function_name, None, None,
|
||||
duration=tool_duration, is_error=is_error,
|
||||
result=function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
if agent.verbose_logging:
|
||||
logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s")
|
||||
logging.debug(f"Tool result ({len(function_result)} chars): {function_result}")
|
||||
|
||||
# Print cute message per tool
|
||||
if agent._should_emit_quiet_tool_messages():
|
||||
cute_msg = _get_cute_tool_message_impl(name, args, tool_duration, result=function_result)
|
||||
agent._safe_print(f" {cute_msg}")
|
||||
elif not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off":
|
||||
_preview_str = _multimodal_text_summary(function_result)
|
||||
if agent.verbose_logging:
|
||||
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s")
|
||||
print(agent._wrap_verbose("Result: ", _preview_str))
|
||||
else:
|
||||
response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str
|
||||
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}")
|
||||
|
||||
agent._current_tool = None
|
||||
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)")
|
||||
|
||||
if not blocked and agent.tool_complete_callback:
|
||||
try:
|
||||
display_args = _redact_tool_args_for_display(name, args) or args
|
||||
agent.tool_complete_callback(tc.id, name, display_args, function_result)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool complete callback error: {cb_err}")
|
||||
_status_suffix = " (error)" if is_error else ""
|
||||
agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}")
|
||||
|
||||
display_function_result = function_result
|
||||
function_result = maybe_persist_tool_result(
|
||||
content=function_result,
|
||||
tool_name=name,
|
||||
|
|
@ -1007,6 +994,50 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
)
|
||||
messages.append(tool_message)
|
||||
risk_metadata = tool_message.get("_tool_output_risk")
|
||||
if not _flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"tool result {name}",
|
||||
):
|
||||
return
|
||||
|
||||
# Every completion surface is downstream of the canonical append. If
|
||||
# the UI bridge or process dies while projecting one of these events,
|
||||
# resume can reconstruct the tool result that was already visible.
|
||||
if not blocked and agent.tool_progress_callback:
|
||||
try:
|
||||
agent.tool_progress_callback(
|
||||
"tool.completed", progress_function_name, None, None,
|
||||
duration=tool_duration, is_error=is_error,
|
||||
result=display_function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
# Print cute message per tool
|
||||
if agent._should_emit_quiet_tool_messages():
|
||||
cute_msg = _get_cute_tool_message_impl(
|
||||
name, args, tool_duration, result=display_function_result,
|
||||
)
|
||||
agent._safe_print(f" {cute_msg}")
|
||||
elif not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off":
|
||||
_preview_str = _multimodal_text_summary(display_function_result)
|
||||
if agent.verbose_logging:
|
||||
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s")
|
||||
print(agent._wrap_verbose("Result: ", _preview_str))
|
||||
else:
|
||||
response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str
|
||||
print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}")
|
||||
|
||||
if not blocked and agent.tool_complete_callback:
|
||||
try:
|
||||
display_args = _redact_tool_args_for_display(name, args) or args
|
||||
agent.tool_complete_callback(
|
||||
tc.id, name, display_args, display_function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool complete callback error: {cb_err}")
|
||||
|
||||
if (
|
||||
risk_metadata is not None
|
||||
and risk_metadata.get("risk") != "low"
|
||||
|
|
@ -1023,11 +1054,6 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
|
|||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug("Tool output risk callback error: %s", cb_err)
|
||||
_flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"tool result {name}",
|
||||
)
|
||||
|
||||
# ── Per-tool /steer drain ───────────────────────────────────
|
||||
# Same as the sequential path: drain between each collected
|
||||
|
|
@ -1059,6 +1085,8 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
# Resolve the context-scaled tool-output budget once per turn.
|
||||
_tool_budget = _budget_for_agent(agent)
|
||||
for i, tool_call in enumerate(assistant_message.tool_calls, 1):
|
||||
if getattr(agent, "_incremental_persistence_failed", False):
|
||||
return
|
||||
# SAFETY: check interrupt BEFORE starting each tool.
|
||||
# If the user sent "stop" during a previous tool's execution,
|
||||
# do NOT start any more tools -- skip them all immediately.
|
||||
|
|
@ -1074,11 +1102,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
skipped_tc.id,
|
||||
effect_disposition="none",
|
||||
))
|
||||
_flush_session_db_after_tool_progress(
|
||||
if not _flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"cancelled tool result {skipped_name}",
|
||||
)
|
||||
):
|
||||
return
|
||||
break
|
||||
|
||||
function_name = tool_call.function.name
|
||||
|
|
@ -1094,11 +1123,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
tool_call.id,
|
||||
)
|
||||
)
|
||||
_flush_session_db_after_tool_progress(
|
||||
if not _flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"invalid tool arguments {function_name}",
|
||||
)
|
||||
):
|
||||
return
|
||||
agent._apply_pending_steer_to_tool_results(messages, 1)
|
||||
continue
|
||||
|
||||
|
|
@ -1112,8 +1142,25 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
_underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args)
|
||||
if not _err and _underlying:
|
||||
if _underlying in _tool_search_scoped_names(agent):
|
||||
function_name = _underlying
|
||||
function_args = _underlying_args
|
||||
# Probe-validate before unwrapping (ironclaw#5149):
|
||||
# missing required args return the parameter schema
|
||||
# instead of dispatching into an opaque failure.
|
||||
_probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args)
|
||||
if _probe_err is not None:
|
||||
# This path wraps _block_msg in {"error": ...} —
|
||||
# flatten the probe payload to one plain string.
|
||||
try:
|
||||
_probe = json.loads(_probe_err)
|
||||
_ts_scope_block = (
|
||||
f"{_probe.get('error', '')} Parameters schema: "
|
||||
f"{json.dumps(_probe.get('parameters', {}), ensure_ascii=False)}. "
|
||||
f"{_probe.get('hint', '')}"
|
||||
).strip()
|
||||
except Exception:
|
||||
_ts_scope_block = _probe_err
|
||||
else:
|
||||
function_name = _underlying
|
||||
function_args = _underlying_args
|
||||
else:
|
||||
_ts_scope_block = (
|
||||
f"'{_underlying}' is not available in this session. "
|
||||
|
|
@ -1359,6 +1406,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
return _clarify_tool(
|
||||
question=next_args.get("question", ""),
|
||||
choices=next_args.get("choices"),
|
||||
multi_select=next_args.get("multi_select", False),
|
||||
callback=agent.clarify_callback,
|
||||
)
|
||||
function_result, function_args = _run_agent_tool_execution_middleware(
|
||||
|
|
@ -1644,31 +1692,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
except Exception as _ver_err:
|
||||
logging.debug("file-mutation verifier record failed: %s", _ver_err)
|
||||
|
||||
if not _execution_blocked and agent.tool_progress_callback:
|
||||
try:
|
||||
agent.tool_progress_callback(
|
||||
"tool.completed", function_name, None, None,
|
||||
duration=tool_duration, is_error=_is_error_result,
|
||||
result=function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
agent._current_tool = None
|
||||
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)")
|
||||
_status_suffix = " (error)" if _is_error_result else ""
|
||||
agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}")
|
||||
|
||||
if agent.verbose_logging:
|
||||
logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s")
|
||||
_log_result = _multimodal_text_summary(function_result)
|
||||
logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}")
|
||||
|
||||
if not _execution_blocked and agent.tool_complete_callback:
|
||||
try:
|
||||
display_args = _redact_tool_args_for_display(function_name, function_args) or function_args
|
||||
agent.tool_complete_callback(tool_call.id, function_name, display_args, function_result)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool complete callback error: {cb_err}")
|
||||
|
||||
display_function_result = function_result
|
||||
function_result = maybe_persist_tool_result(
|
||||
content=function_result,
|
||||
tool_name=function_name,
|
||||
|
|
@ -1691,6 +1724,40 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
tool_message = make_tool_result_message(function_name, _tool_content, tool_call.id)
|
||||
messages.append(tool_message)
|
||||
risk_metadata = tool_message.get("_tool_output_risk")
|
||||
if not _flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"tool result {function_name}",
|
||||
):
|
||||
return
|
||||
|
||||
# UI completion/progress events are projections of the canonical tool
|
||||
# row, never a competing in-memory authority.
|
||||
if not _execution_blocked and agent.tool_progress_callback:
|
||||
try:
|
||||
agent.tool_progress_callback(
|
||||
"tool.completed", function_name, None, None,
|
||||
duration=tool_duration, is_error=_is_error_result,
|
||||
result=display_function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
if not _execution_blocked and agent.tool_complete_callback:
|
||||
try:
|
||||
display_args = (
|
||||
_redact_tool_args_for_display(function_name, function_args)
|
||||
or function_args
|
||||
)
|
||||
agent.tool_complete_callback(
|
||||
tool_call.id,
|
||||
function_name,
|
||||
display_args,
|
||||
display_function_result,
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool complete callback error: {cb_err}")
|
||||
|
||||
if (
|
||||
risk_metadata is not None
|
||||
and risk_metadata.get("risk") != "low"
|
||||
|
|
@ -1707,11 +1774,6 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
)
|
||||
except Exception as cb_err:
|
||||
logging.debug("Tool output risk callback error: %s", cb_err)
|
||||
_flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"tool result {function_name}",
|
||||
)
|
||||
|
||||
# ── Per-tool /steer drain ───────────────────────────────────
|
||||
# Drain pending steer BETWEEN individual tool calls so the
|
||||
|
|
@ -1739,11 +1801,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
|
|||
skipped_tc.id,
|
||||
effect_disposition="none",
|
||||
))
|
||||
_flush_session_db_after_tool_progress(
|
||||
if not _flush_session_db_after_tool_progress(
|
||||
agent,
|
||||
messages,
|
||||
stage=f"skipped tool result {skipped_name}",
|
||||
)
|
||||
):
|
||||
return
|
||||
break
|
||||
|
||||
if agent.tool_delay > 0 and i < len(assistant_message.tool_calls):
|
||||
|
|
@ -1794,6 +1857,8 @@ def execute_tool_calls_segmented(agent, assistant_message, messages: list, effec
|
|||
segments = _plan_tool_batch_segments(assistant_message.tool_calls, execution_cwd=_exec_cwd)
|
||||
|
||||
for kind, calls in segments:
|
||||
if getattr(agent, "_incremental_persistence_failed", False):
|
||||
return
|
||||
segment_message = SimpleNamespace(tool_calls=list(calls))
|
||||
if kind == "parallel":
|
||||
execute_tool_calls_concurrent(
|
||||
|
|
@ -1806,6 +1871,9 @@ def execute_tool_calls_segmented(agent, assistant_message, messages: list, effec
|
|||
finalize=False,
|
||||
)
|
||||
|
||||
if getattr(agent, "_incremental_persistence_failed", False):
|
||||
return
|
||||
|
||||
# ── Whole-turn finalize (budget + /steer) ─────────────────────────
|
||||
total_tools = len(assistant_message.tool_calls)
|
||||
if total_tools > 0:
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ class ToolCallGuardrailConfig:
|
|||
no_progress_block_after: int = 5
|
||||
idempotent_tools: frozenset[str] = field(default_factory=lambda: IDEMPOTENT_TOOL_NAMES)
|
||||
mutating_tools: frozenset[str] = field(default_factory=lambda: MUTATING_TOOL_NAMES)
|
||||
loop_caps: "LoopCapConfig" = field(default_factory=lambda: LoopCapConfig())
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, data: Mapping[str, Any] | None) -> "ToolCallGuardrailConfig":
|
||||
|
|
@ -121,6 +122,54 @@ class ToolCallGuardrailConfig:
|
|||
hard_stop_after.get("idempotent_no_progress", data.get("no_progress_block_after")),
|
||||
defaults.no_progress_block_after,
|
||||
),
|
||||
loop_caps=LoopCapConfig.from_mapping(data.get("loop_caps")),
|
||||
)
|
||||
|
||||
|
||||
# Default session-wide caps, matching Claude Code's v2.1.212 runaway-loop
|
||||
# Per-turn (per-agent-loop) caps on runaway-prone tool calls. Counts reset at
|
||||
# the start of every agent loop (reset_for_turn), so the limit is "within a
|
||||
# single turn" rather than cumulative over the whole session. A single loop
|
||||
# issuing dozens of web searches or spawning dozens of subagents is already
|
||||
# pathological, so the defaults are deliberately low.
|
||||
_DEFAULT_MAX_WEB_SEARCHES_PER_TURN = 50
|
||||
_DEFAULT_MAX_SUBAGENTS_PER_TURN = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoopCapConfig:
|
||||
"""Per-turn caps on runaway-prone tool calls.
|
||||
|
||||
Inspired by Claude Code v2.1.212 (Week 29, July 2026), which added caps on
|
||||
WebSearch calls and subagent spawns to stop runaway search / delegation
|
||||
loops. Here the caps count *within a single agent loop* (one turn): the
|
||||
counters reset in ``reset_for_turn`` at the start of every
|
||||
``run_conversation``, so a legitimate multi-turn session is never starved,
|
||||
but a single turn that spirals into an unbounded search / delegation loop
|
||||
is stopped.
|
||||
|
||||
Semantics differ from the per-turn loop *detector* above (which keys on
|
||||
repeated identical/failing calls): these caps are a hard ceiling on the
|
||||
total count of a tool within the turn and fire regardless of
|
||||
``hard_stop_enabled``. A value of ``0`` disables the cap (unlimited).
|
||||
"""
|
||||
|
||||
max_web_searches: int = _DEFAULT_MAX_WEB_SEARCHES_PER_TURN
|
||||
max_subagents: int = _DEFAULT_MAX_SUBAGENTS_PER_TURN
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, data: Mapping[str, Any] | None) -> "LoopCapConfig":
|
||||
"""Build config from the ``tool_loop_guardrails.loop_caps`` section."""
|
||||
if not isinstance(data, Mapping):
|
||||
return cls()
|
||||
defaults = cls()
|
||||
return cls(
|
||||
max_web_searches=_non_negative_int(
|
||||
data.get("max_web_searches"), defaults.max_web_searches
|
||||
),
|
||||
max_subagents=_non_negative_int(
|
||||
data.get("max_subagents"), defaults.max_subagents
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -233,6 +282,11 @@ class ToolCallGuardrailController:
|
|||
self._same_tool_failure_counts: dict[str, int] = {}
|
||||
self._no_progress: dict[ToolCallSignature, tuple[str, int]] = {}
|
||||
self._halt_decision: ToolGuardrailDecision | None = None
|
||||
# Per-turn runaway-loop cap counters. Reset every turn (this method
|
||||
# runs at the start of each run_conversation), so the caps bound a
|
||||
# single agent loop rather than accumulating across the session.
|
||||
self._turn_web_search_count = 0
|
||||
self._turn_subagent_count = 0
|
||||
|
||||
@property
|
||||
def halt_decision(self) -> ToolGuardrailDecision | None:
|
||||
|
|
@ -240,6 +294,17 @@ class ToolCallGuardrailController:
|
|||
|
||||
def before_call(self, tool_name: str, args: Mapping[str, Any] | None) -> ToolGuardrailDecision:
|
||||
signature = ToolCallSignature.from_call(tool_name, _coerce_args(args))
|
||||
|
||||
# ── Per-turn runaway-loop caps ──────────────────────────────────
|
||||
# These are hard ceilings on how many times a runaway-prone tool may
|
||||
# be called within a single agent loop (turn). They apply regardless
|
||||
# of hard_stop_enabled (which only governs the per-turn loop detector).
|
||||
# We block BEFORE the call runs once the count is already at the cap,
|
||||
# then increment for an allowed call so the (cap+1)-th is refused.
|
||||
cap_block = self._check_loop_cap(tool_name, _coerce_args(args), signature)
|
||||
if cap_block is not None:
|
||||
return cap_block
|
||||
|
||||
if not self.config.hard_stop_enabled:
|
||||
return ToolGuardrailDecision(tool_name=tool_name, signature=signature)
|
||||
|
||||
|
|
@ -379,6 +444,68 @@ class ToolCallGuardrailController:
|
|||
return False
|
||||
return tool_name in self.config.idempotent_tools
|
||||
|
||||
def _check_loop_cap(
|
||||
self,
|
||||
tool_name: str,
|
||||
args: Mapping[str, Any],
|
||||
signature: ToolCallSignature,
|
||||
) -> ToolGuardrailDecision | None:
|
||||
"""Enforce and advance the per-turn runaway-loop counters.
|
||||
|
||||
Returns a ``block`` decision when the cap is already reached, otherwise
|
||||
increments the relevant counter for the allowed call and returns
|
||||
``None``. A cap of 0 disables that limit entirely. Counters reset each
|
||||
turn via ``reset_for_turn``.
|
||||
"""
|
||||
caps = self.config.loop_caps
|
||||
|
||||
if tool_name == "web_search":
|
||||
cap = caps.max_web_searches
|
||||
if cap and self._turn_web_search_count >= cap:
|
||||
decision = ToolGuardrailDecision(
|
||||
action="block",
|
||||
code="loop_web_search_cap",
|
||||
message=(
|
||||
f"Blocked web_search: this turn has already made {cap} "
|
||||
"web searches, the per-turn limit. This looks like a "
|
||||
"runaway search loop. Work with the results you already "
|
||||
"have and give the user your answer."
|
||||
),
|
||||
tool_name=tool_name,
|
||||
count=self._turn_web_search_count,
|
||||
signature=signature,
|
||||
)
|
||||
self._halt_decision = decision
|
||||
return decision
|
||||
self._turn_web_search_count += 1
|
||||
return None
|
||||
|
||||
if tool_name == "delegate_task":
|
||||
cap = caps.max_subagents
|
||||
if not cap:
|
||||
return None
|
||||
spawn_count = _subagent_spawn_count(args)
|
||||
if self._turn_subagent_count >= cap:
|
||||
decision = ToolGuardrailDecision(
|
||||
action="block",
|
||||
code="loop_subagent_cap",
|
||||
message=(
|
||||
f"Blocked delegate_task: this turn has already spawned "
|
||||
f"{self._turn_subagent_count} subagents (limit {cap}). "
|
||||
"This looks like a runaway delegation loop. Finish the "
|
||||
"work with the results you have and answer the user."
|
||||
),
|
||||
tool_name=tool_name,
|
||||
count=self._turn_subagent_count,
|
||||
signature=signature,
|
||||
)
|
||||
self._halt_decision = decision
|
||||
return decision
|
||||
self._turn_subagent_count += spawn_count
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def toolguard_synthetic_result(decision: ToolGuardrailDecision) -> str:
|
||||
"""Build a synthetic role=tool content string for a blocked tool call."""
|
||||
|
|
@ -471,6 +598,32 @@ def _positive_int(value: Any, default: int) -> int:
|
|||
return parsed if parsed >= 1 else default
|
||||
|
||||
|
||||
def _non_negative_int(value: Any, default: int) -> int:
|
||||
"""Parse a session-cap value. 0 is a valid (disable) value; negatives and
|
||||
junk fall back to the default."""
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return parsed if parsed >= 0 else default
|
||||
|
||||
|
||||
def _subagent_spawn_count(args: Mapping[str, Any]) -> int:
|
||||
"""How many subagents a single delegate_task call spawns.
|
||||
|
||||
delegate_task runs in one of two modes: a batch (``tasks`` is a non-empty
|
||||
list, one child per item) or a single task (``goal``). Count the batch size
|
||||
when present, otherwise 1, so the session subagent cap reflects real spawns
|
||||
rather than delegate_task invocations.
|
||||
"""
|
||||
tasks = args.get("tasks") if isinstance(args, Mapping) else None
|
||||
if isinstance(tasks, list) and tasks:
|
||||
return len(tasks)
|
||||
return 1
|
||||
|
||||
|
||||
def _sha256(value: str) -> str:
|
||||
# surrogatepass: tool results scraped from the web can carry unpaired
|
||||
# UTF-16 surrogates (e.g. half of a mathematical-bold pair); a strict
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ def build_trace_jsonl(
|
|||
if cwd:
|
||||
r = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
capture_output=True, text=True, timeout=3, cwd=cwd,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3, cwd=cwd,
|
||||
)
|
||||
if r.returncode == 0:
|
||||
git_branch = r.stdout.strip()
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ streaming, or the _run_codex_stream() call path.
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.transports.base import ProviderTransport
|
||||
|
|
@ -27,6 +28,49 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
|
|||
return f"pck_{digest}"
|
||||
|
||||
|
||||
_EXTENDED_PROMPT_CACHE_MODELS = (
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5.5",
|
||||
"gpt-5.4",
|
||||
"gpt-5.2",
|
||||
"gpt-5.1-codex-max",
|
||||
"gpt-5.1-codex-mini",
|
||||
"gpt-5.1-chat-latest",
|
||||
"gpt-5.1-codex",
|
||||
"gpt-5.1",
|
||||
"gpt-5-codex",
|
||||
"gpt-5",
|
||||
"gpt-4.1",
|
||||
)
|
||||
_EXTENDED_PROMPT_CACHE_MODEL_RE = re.compile(
|
||||
rf"(?:^|[./:])(?:{'|'.join(re.escape(name) for name in _EXTENDED_PROMPT_CACHE_MODELS)})"
|
||||
r"(?:-\d{4}-\d{2}-\d{2})?$"
|
||||
)
|
||||
|
||||
|
||||
def _default_prompt_cache_retention_for_request(
|
||||
model: str,
|
||||
base_url: Any,
|
||||
) -> Optional[str]:
|
||||
"""Return ``24h`` for supported models on Amazon Bedrock Mantle."""
|
||||
from utils import base_url_hostname
|
||||
|
||||
hostname_parts = base_url_hostname(str(base_url or "")).split(".")
|
||||
is_bedrock_mantle = (
|
||||
len(hostname_parts) == 4
|
||||
and hostname_parts[0] == "bedrock-mantle"
|
||||
and bool(hostname_parts[1])
|
||||
and hostname_parts[2:] == ["api", "aws"]
|
||||
)
|
||||
if not is_bedrock_mantle:
|
||||
return None
|
||||
|
||||
normalized = str(model or "").strip().lower().replace("_", "-")
|
||||
if _EXTENDED_PROMPT_CACHE_MODEL_RE.search(normalized):
|
||||
return "24h"
|
||||
return None
|
||||
|
||||
|
||||
def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]:
|
||||
"""Content-address the prompt cache key from the static request prefix.
|
||||
|
||||
|
|
@ -284,6 +328,13 @@ class ResponsesApiTransport(ProviderTransport):
|
|||
if not is_github_responses and not is_xai_responses and cache_key:
|
||||
kwargs["prompt_cache_key"] = cache_key
|
||||
|
||||
cache_retention = _default_prompt_cache_retention_for_request(
|
||||
model,
|
||||
params.get("base_url"),
|
||||
)
|
||||
if cache_retention:
|
||||
kwargs.setdefault("prompt_cache_retention", cache_retention)
|
||||
|
||||
if reasoning_enabled and is_xai_responses:
|
||||
from agent.model_metadata import grok_supports_reasoning_effort
|
||||
|
||||
|
|
|
|||
|
|
@ -394,7 +394,7 @@ def check_codex_binary(
|
|||
proc = subprocess.run(
|
||||
[codex_bin, "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
text=True, encoding='utf-8', errors='replace',
|
||||
timeout=10,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from agent.conversation_compression import (
|
|||
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
|
||||
compression_skipped_due_to_lock,
|
||||
conversation_history_after_compression,
|
||||
recover_rotated_compression_session,
|
||||
)
|
||||
from agent.context_engine import automatic_compaction_status_message
|
||||
from agent.iteration_budget import IterationBudget
|
||||
|
|
@ -353,6 +354,13 @@ def build_turn_context(
|
|||
# Guard stdio against OSError from broken pipes (systemd/headless/daemon).
|
||||
install_safe_stdio()
|
||||
|
||||
# Recover a session rotated by another path before binding log/turn ids or
|
||||
# copying client-supplied history. Everything in this turn must consistently
|
||||
# belong to the canonical child, including observability metadata.
|
||||
recovered_history = recover_rotated_compression_session(agent)
|
||||
if recovered_history is not None:
|
||||
conversation_history = recovered_history
|
||||
|
||||
# NOTE: the DB session row is created later, AFTER the system prompt is
|
||||
# restored/built (see _ensure_db_session() below the system-prompt block).
|
||||
# Creating it here — before _cached_system_prompt is populated — inserts a
|
||||
|
|
|
|||
310
agent/turn_summary.py
Normal file
310
agent/turn_summary.py
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
"""Per-turn accounting for the interactive CLI.
|
||||
|
||||
Two display-only pieces live here:
|
||||
|
||||
* :class:`TurnSummaryCollector` — a tiny observer that rides the existing
|
||||
``tool_progress_callback`` feed (``tool.completed`` events already carry
|
||||
the tool name and its raw result) and tallies what a turn actually did.
|
||||
It holds **no** agent-loop state: the display layer already sees every
|
||||
tool call, so nothing new is threaded through the conversation loop.
|
||||
* :func:`format_turn_summary` — a pure formatter that turns a tally plus a
|
||||
wall-clock duration into one dim line, e.g.::
|
||||
|
||||
⋯ 12.4s · edited 2 files +18 -3 · read 4 files · ran 3 commands
|
||||
|
||||
Ported from Claude Code's post-turn accounting line
|
||||
("Edited 1 file +6 -2, read 1 file … Worked for 10s").
|
||||
|
||||
:func:`format_token_flow` is the spinner-side counterpart: a cumulative
|
||||
token readout appended to the live elapsed timer (``↓ 1.2k tok``).
|
||||
|
||||
Everything in this module is pure/side-effect free apart from the
|
||||
collector's own counters, which makes it directly unit-testable without a
|
||||
terminal, an agent, or a network call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"TurnSummaryCollector",
|
||||
"TurnTally",
|
||||
"format_turn_summary",
|
||||
"format_token_flow",
|
||||
"format_elapsed",
|
||||
]
|
||||
|
||||
|
||||
# Leading glyph for the summary line. Deliberately not an emoji — the line is
|
||||
# meant to read as terminal chrome, not as agent speech.
|
||||
SUMMARY_PREFIX = "⋯"
|
||||
|
||||
# A turn that called no tools and finished this fast has nothing worth
|
||||
# reporting (plain chat reply). Below the threshold the formatter returns "".
|
||||
_MIN_TOOLLESS_SECONDS = 2.0
|
||||
|
||||
# Max number of "verb + count" segments rendered before collapsing the rest
|
||||
# into a "+N more" tail, so a 12-tool turn cannot blow past one line.
|
||||
_MAX_SEGMENTS = 4
|
||||
|
||||
|
||||
# Tool name -> (verb, singular noun, plural noun).
|
||||
#
|
||||
# Verbs are past tense because the line is printed *after* the turn. Tools not
|
||||
# listed here fall into a generic "called N tools" bucket rather than inventing
|
||||
# phrasing for plugin/MCP tools whose semantics we don't know.
|
||||
_VERB_GROUPS: dict[str, tuple[str, str, str]] = {
|
||||
"write_file": ("edited", "file", "files"),
|
||||
"patch": ("edited", "file", "files"),
|
||||
"read_file": ("read", "file", "files"),
|
||||
"web_extract": ("read", "page", "pages"),
|
||||
"terminal": ("ran", "command", "commands"),
|
||||
"execute_code": ("ran", "script", "scripts"),
|
||||
"search_files": ("searched", "path", "paths"),
|
||||
"web_search": ("searched the web", "time", "times"),
|
||||
"session_search": ("searched sessions", "time", "times"),
|
||||
"browser_navigate": ("browsed", "page", "pages"),
|
||||
"skill_view": ("read", "skill", "skills"),
|
||||
"skill_manage": ("updated", "skill", "skills"),
|
||||
"skills_list": ("listed skills", "time", "times"),
|
||||
"todo": ("updated", "task list", "task lists"),
|
||||
"delegate_task": ("delegated", "task", "tasks"),
|
||||
"memory": ("updated", "memory", "memories"),
|
||||
}
|
||||
|
||||
# Verb groups that carry file-edit line deltas (+X -Y) when known.
|
||||
_EDIT_VERB = "edited"
|
||||
|
||||
# Render order: edits first (the thing users most want confirmed), then reads,
|
||||
# then commands. Anything else follows in first-seen order.
|
||||
_VERB_PRIORITY: tuple[str, ...] = ("edited", "read", "ran")
|
||||
|
||||
# Tools whose results may report a unified diff we can count lines from.
|
||||
_DIFF_RESULT_TOOLS = frozenset({"patch"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurnTally:
|
||||
"""What a single turn did, as observed from the tool-progress feed."""
|
||||
|
||||
# verb -> {noun_plural: count}; keeps insertion order for stable rendering.
|
||||
verbs: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
# Tools with no curated verb, counted together.
|
||||
other_tools: int = 0
|
||||
# Aggregated unified-diff line deltas across edit tools, when reported.
|
||||
lines_added: int = 0
|
||||
lines_removed: int = 0
|
||||
# True once at least one edit tool reported a countable diff, so the
|
||||
# formatter knows the difference between "+0 -0" and "unknown".
|
||||
has_line_deltas: bool = False
|
||||
|
||||
@property
|
||||
def total_tools(self) -> int:
|
||||
counted = sum(sum(nouns.values()) for nouns in self.verbs.values())
|
||||
return counted + self.other_tools
|
||||
|
||||
|
||||
def _count_diff_lines(diff: str) -> tuple[int, int]:
|
||||
"""Count added/removed lines in unified-diff text.
|
||||
|
||||
File headers (``+++``/``---``) are excluded so a one-line edit does not
|
||||
read as three additions.
|
||||
"""
|
||||
added = removed = 0
|
||||
for line in diff.splitlines():
|
||||
if line.startswith("+++") or line.startswith("---"):
|
||||
continue
|
||||
if line.startswith("+"):
|
||||
added += 1
|
||||
elif line.startswith("-"):
|
||||
removed += 1
|
||||
return added, removed
|
||||
|
||||
|
||||
def _extract_line_deltas(tool_name: str, result: Any) -> tuple[int, int] | None:
|
||||
"""Pull (added, removed) from a tool result, or None when unavailable.
|
||||
|
||||
Only tools that already report a diff in their result payload are
|
||||
inspected — we never shell out to git and never re-read files to
|
||||
synthesise a delta.
|
||||
"""
|
||||
if tool_name not in _DIFF_RESULT_TOOLS:
|
||||
return None
|
||||
payload: Any = result
|
||||
if isinstance(payload, str):
|
||||
text = payload.strip()
|
||||
if not text.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
import json
|
||||
|
||||
# strict=False tolerates literal control characters inside strings
|
||||
# (raw newlines in an embedded diff), which some tool serialisers
|
||||
# emit. A tally line is never worth failing over formatting.
|
||||
payload = json.loads(text, strict=False)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
diff = payload.get("diff")
|
||||
if not isinstance(diff, str) or not diff.strip():
|
||||
return None
|
||||
added, removed = _count_diff_lines(diff)
|
||||
# A diff that carries no +/- content lines (e.g. a bare hunk header) tells
|
||||
# us nothing — report it as unknown rather than rendering a misleading
|
||||
# "+0 -0" next to a real edit.
|
||||
if added == 0 and removed == 0:
|
||||
return None
|
||||
return added, removed
|
||||
|
||||
|
||||
class TurnSummaryCollector:
|
||||
"""Accumulate per-turn tool tallies from the tool-progress feed.
|
||||
|
||||
Wired into the CLI's existing ``_on_tool_progress`` handler: the display
|
||||
layer already receives every ``tool.completed`` event with the tool name
|
||||
and raw result, so no agent-loop bookkeeping is added.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tally = TurnTally()
|
||||
|
||||
def begin(self) -> None:
|
||||
"""Start a fresh turn (drops any prior tally)."""
|
||||
self._tally = TurnTally()
|
||||
|
||||
def record_tool(
|
||||
self,
|
||||
tool_name: str | None,
|
||||
*,
|
||||
result: Any = None,
|
||||
is_error: bool = False,
|
||||
) -> None:
|
||||
"""Record one completed tool call.
|
||||
|
||||
Failed calls are skipped: a summary claiming "edited 2 files" when one
|
||||
write was denied would be exactly the over-claim the file-mutation
|
||||
verifier exists to catch.
|
||||
"""
|
||||
if not tool_name or is_error:
|
||||
return
|
||||
# Internal/pseudo tools (``_thinking``) are not user-visible work.
|
||||
if tool_name.startswith("_"):
|
||||
return
|
||||
|
||||
group = _VERB_GROUPS.get(tool_name)
|
||||
if group is None:
|
||||
self._tally.other_tools += 1
|
||||
return
|
||||
|
||||
verb, _singular, plural = group
|
||||
nouns = self._tally.verbs.setdefault(verb, {})
|
||||
nouns[plural] = nouns.get(plural, 0) + 1
|
||||
|
||||
if verb == _EDIT_VERB:
|
||||
deltas = _extract_line_deltas(tool_name, result)
|
||||
if deltas is not None:
|
||||
added, removed = deltas
|
||||
self._tally.lines_added += added
|
||||
self._tally.lines_removed += removed
|
||||
self._tally.has_line_deltas = True
|
||||
|
||||
@property
|
||||
def tally(self) -> TurnTally:
|
||||
return self._tally
|
||||
|
||||
def render(self, elapsed_seconds: float) -> str:
|
||||
"""Render this turn's summary line (see :func:`format_turn_summary`)."""
|
||||
return format_turn_summary(elapsed_seconds, self._tally)
|
||||
|
||||
|
||||
def format_elapsed(seconds: float) -> str:
|
||||
"""Format a wall-clock duration compactly (``12.4s`` / ``2m05s``)."""
|
||||
if seconds < 0:
|
||||
seconds = 0.0
|
||||
if seconds < 60:
|
||||
return f"{seconds:.1f}s"
|
||||
minutes, rest = divmod(int(round(seconds)), 60)
|
||||
return f"{minutes}m{rest:02d}s"
|
||||
|
||||
|
||||
def _pluralize(count: int, plural_noun: str) -> str:
|
||||
"""Return ``"1 file"`` / ``"3 files"`` from a plural noun form."""
|
||||
if count == 1:
|
||||
singular = plural_noun
|
||||
if plural_noun.endswith("ies"):
|
||||
singular = plural_noun[:-3] + "y"
|
||||
elif plural_noun.endswith("ses"):
|
||||
singular = plural_noun[:-2]
|
||||
elif plural_noun.endswith("s"):
|
||||
singular = plural_noun[:-1]
|
||||
return f"1 {singular}"
|
||||
return f"{count} {plural_noun}"
|
||||
|
||||
|
||||
def _ordered_verbs(tally: TurnTally) -> list[str]:
|
||||
"""Verbs in render order: priority verbs first, then first-seen order."""
|
||||
seen = list(tally.verbs.keys())
|
||||
ranked = [v for v in _VERB_PRIORITY if v in tally.verbs]
|
||||
ranked += [v for v in seen if v not in _VERB_PRIORITY]
|
||||
return ranked
|
||||
|
||||
|
||||
def format_turn_summary(
|
||||
elapsed_seconds: float,
|
||||
tally: TurnTally | None,
|
||||
*,
|
||||
max_segments: int = _MAX_SEGMENTS,
|
||||
) -> str:
|
||||
"""Render the per-turn accounting line, or ``""`` when there's nothing to say.
|
||||
|
||||
Pure function — no config lookups, no terminal access, no I/O. Gating
|
||||
(``display.turn_summary``, quiet mode, CLI-only) is the caller's job.
|
||||
"""
|
||||
if tally is None:
|
||||
tally = TurnTally()
|
||||
|
||||
segments: list[str] = []
|
||||
for verb in _ordered_verbs(tally):
|
||||
nouns = tally.verbs[verb]
|
||||
parts = [_pluralize(count, plural) for plural, count in nouns.items() if count]
|
||||
if not parts:
|
||||
continue
|
||||
segment = f"{verb} {', '.join(parts)}"
|
||||
if verb == _EDIT_VERB and tally.has_line_deltas:
|
||||
segment += f" +{tally.lines_added} -{tally.lines_removed}"
|
||||
segments.append(segment)
|
||||
|
||||
if tally.other_tools:
|
||||
segments.append(f"called {_pluralize(tally.other_tools, 'tools')}")
|
||||
|
||||
if not segments and tally.total_tools == 0 and elapsed_seconds < _MIN_TOOLLESS_SECONDS:
|
||||
return ""
|
||||
|
||||
if max_segments > 0 and len(segments) > max_segments:
|
||||
hidden = len(segments) - max_segments
|
||||
segments = segments[:max_segments] + [f"+{hidden} more"]
|
||||
|
||||
pieces = [format_elapsed(elapsed_seconds)] + segments
|
||||
return f"{SUMMARY_PREFIX} " + " · ".join(pieces)
|
||||
|
||||
|
||||
def format_token_flow(output_tokens: Any, *, arrow: str = "↓") -> str:
|
||||
"""Render cumulative turn tokens for the live spinner (``↓ 1.2k tok``).
|
||||
|
||||
Returns ``""`` for a non-positive count so the spinner shows nothing
|
||||
rather than a misleading ``↓ 0 tok`` before the first API response lands.
|
||||
"""
|
||||
try:
|
||||
count = int(output_tokens)
|
||||
except (TypeError, ValueError):
|
||||
return ""
|
||||
if count <= 0:
|
||||
return ""
|
||||
if count < 1000:
|
||||
return f"{arrow} {count} tok"
|
||||
if count < 1_000_000:
|
||||
return f"{arrow} {count / 1000:.1f}k tok"
|
||||
return f"{arrow} {count / 1_000_000:.1f}M tok"
|
||||
|
|
@ -13,10 +13,11 @@ import shlex
|
|||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Iterator, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
|
|
@ -65,13 +66,38 @@ def _connect() -> sqlite3.Connection:
|
|||
path = _db_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.row_factory = sqlite3.Row
|
||||
_ensure_schema(conn)
|
||||
try:
|
||||
apply_wal_with_fallback(conn, db_label="verification_evidence.db")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
_ensure_schema(conn)
|
||||
except Exception:
|
||||
# A PRAGMA/DDL failure after a successful connect() must not leak the
|
||||
# just-opened connection back to the caller.
|
||||
conn.close()
|
||||
raise
|
||||
return conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction() -> Iterator[sqlite3.Connection]:
|
||||
"""Open a connection, commit/rollback on exit, and ALWAYS close it.
|
||||
|
||||
``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the
|
||||
transaction; they do not close the connection. Using ``with _connect()``
|
||||
alone therefore leaks a connection — and its WAL/SHM file descriptors — on
|
||||
every call, deferring the close to the garbage collector, which over a
|
||||
long-running process can exhaust ``RLIMIT_NOFILE`` (the cron-ledger sibling
|
||||
of this bug was #69567 / PR #69594).
|
||||
"""
|
||||
conn = _connect()
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
|
|
@ -454,7 +480,7 @@ def record_terminal_result(
|
|||
|
||||
created_at = _utc_now()
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO verification_events(
|
||||
|
|
@ -520,7 +546,7 @@ def mark_workspace_edited(
|
|||
edited_at = _utc_now()
|
||||
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT changed_paths_json FROM verification_state
|
||||
|
|
@ -570,7 +596,7 @@ def verification_status(
|
|||
sid = str(session_id or "default")
|
||||
root = str(facts.get("root") or Path(cwd or ".").resolve())
|
||||
with _DB_LOCK:
|
||||
with _connect() as conn:
|
||||
with _transaction() as conn:
|
||||
state = conn.execute(
|
||||
"""
|
||||
SELECT last_event_id, last_edit_at, changed_paths_json
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@
|
|||
//! 4. Worker iterates stages, calling `install.ps1 -Stage NAME -NonInteractive -Json`.
|
||||
//! 5. On success → `complete`. On any stage failure → `failed`. On cancel → `failed`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::{AppHandle, Emitter, State};
|
||||
use tokio::sync::{mpsc, Mutex};
|
||||
|
|
@ -260,6 +260,107 @@ pub(crate) fn hermes_is_installed(install_root: &std::path::Path) -> bool {
|
|||
&& resolve_hermes_desktop_exe(install_root).is_some()
|
||||
}
|
||||
|
||||
fn resolve_marker_commit(install_root: &Path, pin: &Pin) -> Option<String> {
|
||||
if let Some(commit) = pin
|
||||
.commit
|
||||
.as_ref()
|
||||
.filter(|commit| !commit.trim().is_empty())
|
||||
{
|
||||
return Some(commit.clone());
|
||||
}
|
||||
|
||||
let output = std::process::Command::new("git")
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.current_dir(install_root)
|
||||
.output()
|
||||
.ok()?;
|
||||
if !output.status.success() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let commit = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
if commit.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(commit)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_bootstrap_complete_marker(install_root: &Path, pin: &Pin) -> Result<serde_json::Value> {
|
||||
use std::io::Write;
|
||||
|
||||
let marker_path = crate::paths::likely_bootstrap_marker(install_root);
|
||||
if let Some(parent) = marker_path.parent() {
|
||||
std::fs::create_dir_all(parent).with_context(|| {
|
||||
format!(
|
||||
"could not create bootstrap marker directory {}",
|
||||
parent.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
let completed_at_unix = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or_default();
|
||||
let marker = serde_json::json!({
|
||||
"schemaVersion": 1,
|
||||
"pinnedCommit": resolve_marker_commit(install_root, pin),
|
||||
"pinnedBranch": pin.branch.clone(),
|
||||
"completedAtUnix": completed_at_unix,
|
||||
});
|
||||
let mut body = serde_json::to_vec_pretty(&marker)?;
|
||||
body.push(b'\n');
|
||||
|
||||
// Atomic publish (temp sibling + flush + rename), matching Electron's
|
||||
// writeFileAtomic(). hermes_is_installed() only checks existence, so a
|
||||
// partial direct write would incorrectly enable the launcher fast path.
|
||||
let tmp_path = install_root.join(".hermes-bootstrap-complete.tmp");
|
||||
{
|
||||
let mut file = std::fs::File::create(&tmp_path).with_context(|| {
|
||||
format!(
|
||||
"could not create temp bootstrap marker {}",
|
||||
tmp_path.display()
|
||||
)
|
||||
})?;
|
||||
file.write_all(&body).with_context(|| {
|
||||
format!(
|
||||
"could not write temp bootstrap marker {}",
|
||||
tmp_path.display()
|
||||
)
|
||||
})?;
|
||||
file.sync_all().with_context(|| {
|
||||
format!(
|
||||
"could not flush temp bootstrap marker {}",
|
||||
tmp_path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
// Windows rename fails if the destination already exists; drop any prior
|
||||
// marker first so a re-run can still publish a fresh payload.
|
||||
if marker_path.exists() {
|
||||
std::fs::remove_file(&marker_path).with_context(|| {
|
||||
format!(
|
||||
"could not replace existing bootstrap marker {}",
|
||||
marker_path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Err(err) = std::fs::rename(&tmp_path, &marker_path) {
|
||||
let _ = std::fs::remove_file(&tmp_path);
|
||||
return Err(err).with_context(|| {
|
||||
format!(
|
||||
"could not publish bootstrap marker {} → {}",
|
||||
tmp_path.display(),
|
||||
marker_path.display()
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
tracing::info!(path = %marker_path.display(), "bootstrap marker written");
|
||||
Ok(marker)
|
||||
}
|
||||
|
||||
/// Spawn the already-built desktop app, detached. Returns Err if no built app
|
||||
/// exists or the spawn fails, so the caller can fall back to showing the
|
||||
/// installer UI.
|
||||
|
|
@ -644,6 +745,23 @@ async fn run_bootstrap(
|
|||
.unwrap_or_else(|| crate::paths::hermes_home().to_string_lossy().into_owned());
|
||||
let install_root = PathBuf::from(&hermes_home).join("hermes-agent");
|
||||
|
||||
// Marker publish is terminal for this run: a write failure must emit Failed
|
||||
// so the UI leaves the progress state (it does not poll get_bootstrap_status).
|
||||
let marker = match write_bootstrap_complete_marker(&install_root, &pin) {
|
||||
Ok(marker) => marker,
|
||||
Err(err) => {
|
||||
let msg = format!("write bootstrap marker failed: {err:#}");
|
||||
emit_event(
|
||||
&app,
|
||||
BootstrapEvent::Failed {
|
||||
stage: None,
|
||||
error: msg.clone(),
|
||||
},
|
||||
);
|
||||
return Err(anyhow!(msg));
|
||||
}
|
||||
};
|
||||
|
||||
// Copy ourselves to HERMES_HOME/hermes-setup.exe so the desktop app can
|
||||
// re-invoke us with `--update` and shortcuts have a stable target. This is
|
||||
// a one-shot install concern; an `--update` re-invocation no-ops because
|
||||
|
|
@ -660,10 +778,7 @@ async fn run_bootstrap(
|
|||
&app,
|
||||
BootstrapEvent::Complete {
|
||||
install_root: install_root.to_string_lossy().into_owned(),
|
||||
marker: Some(serde_json::json!({
|
||||
"pinnedCommit": pin.commit,
|
||||
"pinnedBranch": pin.branch,
|
||||
})),
|
||||
marker: Some(marker),
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -903,4 +1018,103 @@ mod tests {
|
|||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_complete_marker_uses_desktop_compatible_schema() {
|
||||
let root = unique_tmp_dir("marker-schema");
|
||||
let pin = Pin {
|
||||
commit: Some("abcdef1234567890".to_string()),
|
||||
branch: Some("main".to_string()),
|
||||
};
|
||||
|
||||
let marker =
|
||||
write_bootstrap_complete_marker(&root, &pin).expect("marker write should succeed");
|
||||
let marker_path = root.join(".hermes-bootstrap-complete");
|
||||
let from_disk: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&marker_path).unwrap()).unwrap();
|
||||
|
||||
assert_eq!(marker, from_disk);
|
||||
assert_eq!(from_disk["schemaVersion"], 1);
|
||||
assert_eq!(from_disk["pinnedCommit"], "abcdef1234567890");
|
||||
assert_eq!(from_disk["pinnedBranch"], "main");
|
||||
assert!(
|
||||
from_disk["completedAtUnix"].as_u64().is_some(),
|
||||
"marker must carry a completion timestamp"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_complete_marker_is_published_atomically() {
|
||||
let root = unique_tmp_dir("marker-atomic");
|
||||
make_release_tree(&root);
|
||||
let pin = Pin {
|
||||
commit: Some("abcdef1234567890".to_string()),
|
||||
branch: Some("main".to_string()),
|
||||
};
|
||||
|
||||
write_bootstrap_complete_marker(&root, &pin).expect("marker write should succeed");
|
||||
|
||||
let marker_path = root.join(".hermes-bootstrap-complete");
|
||||
let tmp_path = root.join(".hermes-bootstrap-complete.tmp");
|
||||
assert!(
|
||||
marker_path.is_file(),
|
||||
"final marker must exist after atomic publish"
|
||||
);
|
||||
assert!(
|
||||
!tmp_path.exists(),
|
||||
"temp sibling must not remain after atomic publish"
|
||||
);
|
||||
assert!(
|
||||
hermes_is_installed(&root),
|
||||
"atomically published marker must enable the installer fast path"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hermes_is_installed_treats_marker_existence_as_sufficient() {
|
||||
// Documents why write_bootstrap_complete_marker must publish atomically:
|
||||
// the launcher predicate only checks existence, so a partial/corrupt
|
||||
// final marker would still enable the fast path.
|
||||
let root = unique_tmp_dir("marker-existence-only");
|
||||
make_release_tree(&root);
|
||||
std::fs::write(root.join(".hermes-bootstrap-complete"), b"").unwrap();
|
||||
|
||||
assert!(
|
||||
hermes_is_installed(&root),
|
||||
"empty/partial marker content still counts as installed"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn marker_write_failure_leaves_no_final_marker() {
|
||||
// install_root is a regular file → create_dir_all on its path fails
|
||||
// before any marker bytes are published under the final name.
|
||||
let base = unique_tmp_dir("marker-fail");
|
||||
let not_a_dir = base.join("not-a-dir");
|
||||
std::fs::write(¬_a_dir, b"not a directory").unwrap();
|
||||
let pin = Pin {
|
||||
commit: Some("abcdef1234567890".to_string()),
|
||||
branch: Some("main".to_string()),
|
||||
};
|
||||
|
||||
let err = write_bootstrap_complete_marker(¬_a_dir, &pin)
|
||||
.expect_err("marker write against a non-directory root must fail");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(
|
||||
msg.contains("bootstrap marker"),
|
||||
"error should mention the marker path: {msg}"
|
||||
);
|
||||
assert!(
|
||||
!not_a_dir.join(".hermes-bootstrap-complete").exists(),
|
||||
"failed write must not leave a final marker that enables the fast path"
|
||||
);
|
||||
assert!(
|
||||
!not_a_dir.join(".hermes-bootstrap-complete.tmp").exists(),
|
||||
"failed write must not leave a temp marker sibling either"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,8 +149,8 @@ fn repair_macos_installer_helper(path: &Path) {
|
|||
#[cfg(not(target_os = "macos"))]
|
||||
fn repair_macos_installer_helper(_path: &Path) {}
|
||||
|
||||
/// Where install.ps1 writes the bootstrap-complete marker (existence-only file
|
||||
/// the Electron app also checks). Per main.ts:
|
||||
/// Where the bootstrap-complete marker lives (existence-only for the Rust
|
||||
/// installer fast path; JSON schema-checked by the Electron app). Per main.ts:
|
||||
/// const BOOTSTRAP_COMPLETE_MARKER = path.join(ACTIVE_HERMES_ROOT, '.hermes-bootstrap-complete')
|
||||
/// We don't always know ACTIVE_HERMES_ROOT until install.ps1 reports it, so
|
||||
/// this is a probe helper, not a definitive path.
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ Already have the Hermes CLI? Just run:
|
|||
hermes desktop
|
||||
```
|
||||
|
||||
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. On first launch Hermes walks you through picking a provider and model; nothing else to configure.
|
||||
It builds and launches the GUI against your existing install — same config, keys, sessions, and skills. If Desktop cannot find a usable runtime or saved remote connection, first launch lets you connect to an existing Hermes gateway or install Hermes locally. Local onboarding then walks you through choosing a provider and model.
|
||||
|
||||
### Prebuilt installers
|
||||
|
||||
|
|
@ -134,6 +134,19 @@ Desktop supports a managed local backend, explicit remote gateways, and Hermes
|
|||
Cloud connections. Remote and cloud modes use the same remote-capability path;
|
||||
authentication and discovery differ, not the renderer feature model.
|
||||
|
||||
When no usable local runtime or saved remote connection exists, the first-run
|
||||
screen offers **Connect to existing Hermes** before starting the local installer.
|
||||
Desktop probes the gateway to discover token or OAuth authentication, requires a
|
||||
successful HTTP and WebSocket connection test, and saves the connection using
|
||||
the same encrypted Desktop configuration used by Settings. A saved remote
|
||||
connection bypasses this choice on later launches. The regular Desktop build
|
||||
still includes the local-install option; this is a remote operating mode, not a
|
||||
separate client-only application.
|
||||
|
||||
In remote mode the gateway host is the execution boundary: agent tools,
|
||||
terminal commands, and file operations run against the remote Hermes host, not
|
||||
the computer displaying the Desktop UI.
|
||||
|
||||
Projects are the workspace abstraction. A project may own multiple folders,
|
||||
repositories, worktrees, and sessions; a bare new chat remains detached unless
|
||||
the user enters a project or configures a default project directory. Use the
|
||||
|
|
|
|||
|
|
@ -21,8 +21,16 @@ const INFERENCE_SWITCH_TRIGGER = 'E2E_INFERENCE_SWITCH_TRIGGER'
|
|||
const INFERENCE_PROMPT = `${INFERENCE_SWITCH_TRIGGER}: original inference prompt must remain singular.`
|
||||
const INFERENCE_CORRECTION = `${INFERENCE_SWITCH_TRIGGER}: correction sent while inference is live.`
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
|
||||
function activeSurface(page: Page) {
|
||||
return page.locator(SURFACE).last()
|
||||
}
|
||||
|
||||
async function send(page: Page, text: string): Promise<void> {
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
const composer = activeSurface(page).locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await composer.click()
|
||||
await composer.type(text, { delay: 5 })
|
||||
|
|
@ -30,8 +38,9 @@ async function send(page: Page, text: string): Promise<void> {
|
|||
}
|
||||
|
||||
async function steer(page: Page, text: string): Promise<void> {
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
const primary = page.locator('[data-slot="composer-root"] button[type="submit"]')
|
||||
const surface = activeSurface(page)
|
||||
const composer = surface.locator('[contenteditable="true"]').first()
|
||||
const primary = surface.locator('[data-slot="composer-root"] button[type="submit"]')
|
||||
|
||||
await composer.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await composer.click()
|
||||
|
|
@ -42,55 +51,80 @@ async function steer(page: Page, text: string): Promise<void> {
|
|||
|
||||
async function waitForTranscriptText(page: Page, text: string): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
(expected: string) => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
text,
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function textNodeOccurrences(page: Page, text: string): Promise<number> {
|
||||
return page.evaluate((expected: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return 0
|
||||
return page.evaluate(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return 0
|
||||
|
||||
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
|
||||
let count = 0
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent?.includes(expected)) {
|
||||
count += 1
|
||||
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
|
||||
let count = 0
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent?.includes(expected)) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return count
|
||||
}, text)
|
||||
return count
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
)
|
||||
}
|
||||
|
||||
async function transcriptTextOrder(page: Page): Promise<string[]> {
|
||||
return page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return []
|
||||
|
||||
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="message"], [data-message-id]'))
|
||||
.map(message => message.textContent?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
async function transcriptMessageOrder(page: Page): Promise<string[]> {
|
||||
return page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const viewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return []
|
||||
|
||||
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"]'))
|
||||
return Array.from(
|
||||
viewport.querySelectorAll<HTMLElement>('[data-role="user"], [data-role="assistant"], [data-role="system"]'),
|
||||
)
|
||||
.map(message => message.textContent?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidebar "+" opens a NEW TAB beside the current chat rather than
|
||||
* replacing it, so the prior session stays mounted in its own surface. Wait
|
||||
* for the newly-mounted surface to show an empty transcript instead of waiting
|
||||
* for the old text to disappear from the page (it never will).
|
||||
*/
|
||||
async function openFreshDraft(page: Page, priorSessionText: string): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await page.waitForFunction(
|
||||
(priorText: string) => !(document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(priorText),
|
||||
priorSessionText,
|
||||
([priorText, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
const transcript = active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return surfaces.length > 0 && !transcript.includes(priorText)
|
||||
},
|
||||
[priorSessionText, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
|
@ -116,7 +150,12 @@ async function reopenInferenceSession(page: Page): Promise<void> {
|
|||
}
|
||||
|
||||
function relevantOrder(messages: string[]): string[] {
|
||||
return messages.filter(message => message.includes(ORIGINAL_PROMPT) || message.includes(CORRECTION))
|
||||
return messages.flatMap(message => {
|
||||
if (message.includes(ORIGINAL_PROMPT)) return [ORIGINAL_PROMPT]
|
||||
if (message.includes(CORRECTION)) return [CORRECTION]
|
||||
|
||||
return []
|
||||
})
|
||||
}
|
||||
|
||||
function steerTurnOrder(messages: string[]): string[] {
|
||||
|
|
|
|||
197
apps/desktop/e2e/image-attachment-resume.spec.ts
Normal file
197
apps/desktop/e2e/image-attachment-resume.spec.ts
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
/**
|
||||
* Regression coverage for an attached image in a durable session. The gateway
|
||||
* persists the turn, the builder exits, and desktop renders it from SessionDB
|
||||
* for the first time — the "quit and relaunch" case, where the transcript used
|
||||
* to come back as vision-enrichment prose instead of a thumbnail.
|
||||
*
|
||||
* The fixture pins `image_input_mode: native` because that is the majority
|
||||
* routing path (any vision-capable model) and the one where a text-only
|
||||
* persist override is silently dropped. The image also sits behind directory
|
||||
* and file names containing spaces, mirroring the macOS composer's
|
||||
* `~/Library/Application Support/...` staging path.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import {
|
||||
buildAppEnv,
|
||||
createSandbox,
|
||||
launchDesktop,
|
||||
type Sandbox,
|
||||
waitForAppReady,
|
||||
writeEnvFile,
|
||||
writeMockProviderConfig,
|
||||
} from './fixtures'
|
||||
import { type MockServer, startMockServer } from './mock-server'
|
||||
import { RealSessionBuilder } from './real-session-builder'
|
||||
import { type ElectronApplication, expect, type Page, test } from './test'
|
||||
|
||||
// A seeded session has no generated title, so every label falls back to the
|
||||
// session preview — the first 60 characters of the first user message.
|
||||
const SESSION_TITLE = 'E2E attached image session'
|
||||
const CAPTION = 'E2E attached image must survive a relaunch'
|
||||
const IMAGE_DIR = 'Application Support/e2e shots'
|
||||
const IMAGE_NAME = 'e2e capture.png'
|
||||
const NATIVE_IMAGE_CONFIG = 'agent:\n image_input_mode: native'
|
||||
|
||||
/** A 160x100 framed magenta block — small, but visible in the screenshots. */
|
||||
const PNG_BASE64 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAKAAAABkCAIAAACO1KzYAAAA30lEQVR42u3dwQ2AIBAAQTAWAx1iBXYI7diCuWhEMvP2dZsj+CL30hLr2oxAYARGYARGYARGYIERGIH53n7nozpOk5rQqIcNdkQjMAIjMNPeomP3N54V+5exwY5oBEZgBEZgBEZggREYgREYgREYgQVGYARGYARGYARGYIERGIERGIERGIEFRmAERmAERmAEFhiBERiBERiBERiBBUZgBEZgBEZgBBYYgREYgREYgRFYYCMQGIERmPSj94Njb9ligxEYgRFYYJaQe2mmYIMRGIERGIERGIEFRmAERmDedAFtjAtAGWDnoAAAAABJRU5ErkJggg=='
|
||||
|
||||
interface SeededFixture {
|
||||
app: ElectronApplication
|
||||
mock: MockServer
|
||||
page: Page
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
function writeImage(sandbox: Sandbox): string {
|
||||
const dir = path.join(sandbox.root, IMAGE_DIR)
|
||||
fs.mkdirSync(dir, { recursive: true })
|
||||
|
||||
const imagePath = path.join(dir, IMAGE_NAME)
|
||||
fs.writeFileSync(imagePath, Buffer.from(PNG_BASE64, 'base64'))
|
||||
|
||||
return imagePath
|
||||
}
|
||||
|
||||
async function setupSeededDesktop(): Promise<SeededFixture> {
|
||||
const mock = await startMockServer()
|
||||
const sandbox = createSandbox('image-attachment')
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url, undefined, NATIVE_IMAGE_CONFIG)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
|
||||
|
||||
try {
|
||||
await builder.createSession({
|
||||
title: SESSION_TITLE,
|
||||
turns: [{ images: [writeImage(sandbox)], text: CAPTION }],
|
||||
})
|
||||
} finally {
|
||||
await builder.close()
|
||||
}
|
||||
|
||||
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
|
||||
|
||||
return {
|
||||
app,
|
||||
mock,
|
||||
page,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function sessionRow(page: Page) {
|
||||
return page.locator('[data-slot="sidebar"] button').filter({ hasText: CAPTION }).first()
|
||||
}
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
|
||||
function activeViewportText(surfaceSelector: string): string {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
|
||||
return surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
}
|
||||
|
||||
async function openSeededSession(page: Page): Promise<void> {
|
||||
const row = sessionRow(page)
|
||||
await row.waitFor({ state: 'visible', timeout: 60_000 })
|
||||
await row.click()
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return text.includes(expected)
|
||||
},
|
||||
[CAPTION, SURFACE] as [string, string],
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidebar "+" opens a NEW TAB beside the current chat instead of replacing
|
||||
* it, so the seeded session stays mounted in its own surface. Assert the new
|
||||
* surface is empty rather than waiting for the old caption to leave the page.
|
||||
*/
|
||||
async function openNewSession(page: Page): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const text = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? ''
|
||||
|
||||
return surfaces.length > 0 && !text.includes(expected)
|
||||
},
|
||||
[CAPTION, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function transcriptText(page: Page): Promise<string> {
|
||||
return page.evaluate(activeViewportText, SURFACE)
|
||||
}
|
||||
|
||||
async function assertRendersThumbnail(page: Page, label: string): Promise<void> {
|
||||
const thumbnail = page.locator('[data-slot="aui_directive-image"] img')
|
||||
await expect(thumbnail, `${label}: the attachment should render as an image`).toHaveCount(1)
|
||||
await expect(thumbnail, `${label}: the thumbnail should resolve off disk`).toHaveAttribute('src', /^data:image\//)
|
||||
|
||||
const text = await transcriptText(page)
|
||||
expect(text, `${label}: the caption should survive alongside the image`).toContain(CAPTION)
|
||||
// A broken ref falls back to a chip whose label leaks the path, and a
|
||||
// flattened multimodal turn leaves the agent's placeholder behind.
|
||||
expect(text, `${label}: the raw image path should not leak into the transcript`).not.toContain(IMAGE_NAME)
|
||||
expect(text, `${label}: the image directive should not render literally`).not.toContain('@image:')
|
||||
expect(text, `${label}: the flattening placeholder should not render`).not.toContain('[screenshot]')
|
||||
}
|
||||
|
||||
test.describe('attached image resume', () => {
|
||||
let fixture: SeededFixture | null = null
|
||||
|
||||
test.afterEach(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test('renders a persisted attachment as a thumbnail on first open and after a cold reload', async ({}, testInfo) => {
|
||||
// Seeding through the real gateway plus two full app boots does not fit the
|
||||
// default per-test budget on a cold runner.
|
||||
test.slow()
|
||||
|
||||
fixture = await setupSeededDesktop()
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
|
||||
// The sidebar labels a session by its preview, so the caption has to lead
|
||||
// the persisted turn — a leading directive reads as a truncated file path.
|
||||
const row = sessionRow(fixture.page)
|
||||
await row.waitFor({ state: 'visible', timeout: 60_000 })
|
||||
|
||||
const label = (await row.textContent())?.trim() ?? ''
|
||||
expect(label.startsWith(CAPTION), `sidebar label should open with the caption: ${label}`).toBe(true)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
await assertRendersThumbnail(fixture.page, 'first open')
|
||||
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-first-open.png') })
|
||||
|
||||
// A reload drops every cached attachment ref, so the transcript has to come
|
||||
// back from the persisted turn alone.
|
||||
await fixture.page.reload()
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
await openNewSession(fixture.page)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
await assertRendersThumbnail(fixture.page, 'cold reload')
|
||||
await fixture.page.screenshot({ path: testInfo.outputPath('attachment-cold-reload.png') })
|
||||
})
|
||||
})
|
||||
|
|
@ -19,6 +19,12 @@ import { RealSessionBuilder } from './real-session-builder'
|
|||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const SESSION_TITLE = 'E2E large persisted session'
|
||||
const EXPECTED_TEXT = 'E2E persisted user message 52'
|
||||
// The oldest seeded turn (HISTORY_TURNS[0]). The transcript first paints only
|
||||
// the newest turns (FIRST_PAINT_BUDGET) and backfills the rest in a rAF; a
|
||||
// baseline count taken before that backfill sees a clipped transcript and
|
||||
// falsely reports duplicates once the full list mounts. Waiting for this
|
||||
// oldest row means the baseline reflects the fully-mounted transcript.
|
||||
const OLDEST_SEEDED_TEXT = 'E2E persisted user message 0: audit the compatibility matrix'
|
||||
const BACKGROUND_PROMPT = 'E2E background inference must remain attached across resume'
|
||||
const HISTORY_TURNS = Array.from(
|
||||
{ length: 27 },
|
||||
|
|
@ -210,6 +216,16 @@ test.describe('large session resume', () => {
|
|||
await waitForAppReady(fixture, 120_000)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
// The transcript first paints only the newest turns (FIRST_PAINT_BUDGET)
|
||||
// and backfills older turns in a rAF. Wait for the oldest seeded row to
|
||||
// mount before taking the baseline so it reflects the full transcript —
|
||||
// otherwise a clipped baseline makes the backfilled rows look like
|
||||
// duplicates of the completed reply.
|
||||
await fixture.page.waitForFunction(
|
||||
expected => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
OLDEST_SEEDED_TEXT,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
const initialMockReplyCount = await textNodeOccurrences(fixture.page, MOCK_REPLY)
|
||||
await submitPrompt(fixture.page, BACKGROUND_PROMPT)
|
||||
await fixture.mock.waitForHeldStream()
|
||||
|
|
|
|||
|
|
@ -14,8 +14,11 @@
|
|||
* prove the full boot → gateway → inference → renderer chain works.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs'
|
||||
import http from 'node:http'
|
||||
import type { ServerResponse } from 'node:http'
|
||||
import os from 'node:os'
|
||||
import nodePath from 'node:path'
|
||||
|
||||
/** A canned assistant reply used for every chat completion request. */
|
||||
export const MOCK_REPLY = 'Hello from the mock inference server! The full boot chain is working.'
|
||||
|
|
@ -27,6 +30,14 @@ export interface MockServerOptions {
|
|||
holdFirstCompletionContaining?: string
|
||||
/** Absolute sandbox path written by the verify-on-stop scripted tool call. */
|
||||
verificationWritePath?: string
|
||||
/**
|
||||
* Sentinel path that ends the E2E_SIDEBAR_CROSS background process.
|
||||
*
|
||||
* Without it that process is a bare `sleep 5`, which races the agent turn and
|
||||
* the 4s auto-dismiss linger — see `createBackgroundReleaseHandle`. Pass a
|
||||
* handle's `path` to let the test decide when the process exits.
|
||||
*/
|
||||
backgroundReleasePath?: string
|
||||
}
|
||||
|
||||
export interface MockServer {
|
||||
|
|
@ -167,37 +178,68 @@ const SIDEBAR_SCRIPT: ScriptedTurn[] = [
|
|||
|
||||
// ─── Sidebar cross-session script ──────────────────────────────────────
|
||||
//
|
||||
// E2E_SIDEBAR_CROSS trigger uses a longer background process (sleep 5) so
|
||||
// the "background running" dot is visible long enough for the test to:
|
||||
// E2E_SIDEBAR_CROSS starts a long background process plus a subagent so the
|
||||
// tests can:
|
||||
// 1. See the background dot while the subagent runs.
|
||||
// 2. Open a different session and see session A's dot transition to
|
||||
// "finished unread" when the background process completes.
|
||||
//
|
||||
// The background process must outlive the agent turn — the whole point is a
|
||||
// dot that is still "running" after the final answer lands. A fixed `sleep`
|
||||
// cannot guarantee that: on a loaded CI runner the turn (two model round
|
||||
// trips + a real subagent delegation) can take longer than the sleep, the
|
||||
// process exits early, the 4s success linger elapses, and the dot is gone
|
||||
// before the test looks. That is a wall-clock race between three independent
|
||||
// timers, and it made this the flakiest spec in the suite.
|
||||
//
|
||||
// When `backgroundReleasePath` is set the process instead blocks until the
|
||||
// test creates that sentinel file, so the test — not the clock — decides when
|
||||
// the dot clears. `sleep 5` remains the fallback for callers that don't pass
|
||||
// a handle.
|
||||
function sidebarCrossBgCommand(releasePath?: string): string {
|
||||
if (!releasePath) {
|
||||
return 'echo "long bg output" && sleep 5 && echo "finished"'
|
||||
}
|
||||
// Bounded wait (60s): if a test forgets to release (or crashes mid-way),
|
||||
// the process still exits instead of hanging the worker until the suite
|
||||
// times out.
|
||||
const quoted = JSON.stringify(releasePath)
|
||||
return [
|
||||
'echo "long bg output"',
|
||||
`for _ in $(seq 1 600); do [ -e ${quoted} ] && break; sleep 0.1; done`,
|
||||
'echo "finished"',
|
||||
].join(' && ')
|
||||
}
|
||||
|
||||
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = [
|
||||
{
|
||||
text: 'Starting a long background task and delegating work.',
|
||||
toolCalls: [
|
||||
{
|
||||
name: 'terminal',
|
||||
args: {
|
||||
command: 'echo "long bg output" && sleep 5 && echo "finished"',
|
||||
background: true,
|
||||
notify_on_complete: true,
|
||||
function sidebarCrossScript(releasePath?: string): ScriptedTurn[] {
|
||||
return [
|
||||
{
|
||||
text: 'Starting a long background task and delegating work.',
|
||||
toolCalls: [
|
||||
{
|
||||
name: 'terminal',
|
||||
args: {
|
||||
command: sidebarCrossBgCommand(releasePath),
|
||||
background: true,
|
||||
notify_on_complete: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'delegate_task',
|
||||
args: {
|
||||
goal: 'Analyze cross-session state',
|
||||
context: 'Testing that the background dot updates across sessions.',
|
||||
{
|
||||
name: 'delegate_task',
|
||||
args: {
|
||||
goal: 'Analyze cross-session state',
|
||||
context: 'Testing that the background dot updates across sessions.',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
text: 'Both tasks are running in the background now.',
|
||||
},
|
||||
]
|
||||
],
|
||||
},
|
||||
{
|
||||
text: 'Both tasks are running in the background now.',
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
const SIDEBAR_CROSS_SCRIPT: ScriptedTurn[] = sidebarCrossScript()
|
||||
|
||||
const QUEUE_STOP_SCRIPT: ScriptedTurn[] = [
|
||||
{
|
||||
|
|
@ -423,7 +465,8 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
}
|
||||
|
||||
if (isSidebarCrossTrigger) {
|
||||
const turn = SIDEBAR_CROSS_SCRIPT[_sidebarCrossIndex] ?? SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1]
|
||||
const script = sidebarCrossScript(options.backgroundReleasePath)
|
||||
const turn = script[_sidebarCrossIndex] ?? script[script.length - 1]
|
||||
_sidebarCrossIndex++
|
||||
|
||||
if (stream) {
|
||||
|
|
@ -722,6 +765,65 @@ export function restartMockServer(): void {
|
|||
resetScriptIndex()
|
||||
}
|
||||
|
||||
/** Test-controlled lifetime for the E2E_SIDEBAR_CROSS background process. */
|
||||
export interface BackgroundReleaseHandle {
|
||||
/** Sentinel path — pass as `backgroundReleasePath` to `startMockServer`. */
|
||||
path: string
|
||||
/** End the background process now (creates the sentinel). */
|
||||
release: () => void
|
||||
/** Remove the sentinel if it still exists. Safe to call twice. */
|
||||
cleanup: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a sentinel that keeps the E2E_SIDEBAR_CROSS background process alive
|
||||
* until the test explicitly releases it.
|
||||
*
|
||||
* The cross-session sidebar tests need a background process that is still
|
||||
* RUNNING after the agent turn finishes — that is the state under test (a
|
||||
* session whose turn is done but whose background work is not). With a fixed
|
||||
* `sleep`, three independent clocks race: the sleep, the agent turn (two model
|
||||
* round trips plus a real subagent delegation), and the 4s success linger
|
||||
* before a finished task auto-dismisses. When a loaded CI runner makes the
|
||||
* turn slower than the sleep, the process is already gone and the assertion
|
||||
* samples an empty sidebar. Observed on CI 2026-07-26 across two unrelated
|
||||
* PRs: the "should appear" poll needed 7.5s to see the dot, by which point
|
||||
* `sleep 5` had exited.
|
||||
*
|
||||
* With a sentinel there is one clock and the test owns it:
|
||||
*
|
||||
* ```ts
|
||||
* const release = createBackgroundReleaseHandle()
|
||||
* const mock = await startMockServer({ backgroundReleasePath: release.path })
|
||||
* // ... assert the dot is visible; it cannot vanish on its own ...
|
||||
* release.release() // now, and only now, the process exits
|
||||
* ```
|
||||
*/
|
||||
export function createBackgroundReleaseHandle(): BackgroundReleaseHandle {
|
||||
const path = nodePath.join(
|
||||
os.tmpdir(),
|
||||
`hermes-e2e-bg-release-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
)
|
||||
return {
|
||||
path,
|
||||
release: () => {
|
||||
try {
|
||||
fs.writeFileSync(path, 'release')
|
||||
} catch {
|
||||
// The process also has a bounded fallback wait; a failed write must
|
||||
// not crash the test before its real assertions run.
|
||||
}
|
||||
},
|
||||
cleanup: () => {
|
||||
try {
|
||||
fs.rmSync(path, { force: true })
|
||||
} catch {
|
||||
// Best-effort — the sentinel lives in the OS temp dir.
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The interim script's text constants, exported for test assertions.
|
||||
* Each entry is the visible text of one turn. Turns with empty text
|
||||
|
|
@ -756,8 +858,12 @@ export const SIDEBAR_CROSS_TEXTS = {
|
|||
interimText: SIDEBAR_CROSS_SCRIPT[0].text,
|
||||
/** The final answer text. */
|
||||
finalText: SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1].text,
|
||||
/** The longer background process command (sleep 5). */
|
||||
bgCommand: 'echo "long bg output" && sleep 5 && echo "finished"',
|
||||
/**
|
||||
* The default (unheld) background process command. Tests that pass a
|
||||
* `backgroundReleasePath` get a sentinel-waiting command instead — see
|
||||
* `createBackgroundReleaseHandle`.
|
||||
*/
|
||||
bgCommand: sidebarCrossBgCommand(),
|
||||
/** The subagent's goal. */
|
||||
subagentGoal: 'Analyze cross-session state',
|
||||
} as const
|
||||
|
|
|
|||
|
|
@ -28,11 +28,18 @@ interface CreatedSession {
|
|||
stored_session_id: string
|
||||
}
|
||||
|
||||
export interface RealSessionTurn {
|
||||
/** Local image paths attached before the prompt, as the composer would. */
|
||||
images?: readonly string[]
|
||||
text: string
|
||||
}
|
||||
|
||||
export interface RealSessionSpec {
|
||||
/** Human-visible sidebar title, persisted by the first completed turn. */
|
||||
/** Session label. The durable row stores no title, so clients fall back to
|
||||
* the preview (the first 60 characters of the first user message). */
|
||||
title: string
|
||||
/** Each item becomes one real user prompt followed by the mock provider's reply. */
|
||||
turns: readonly string[]
|
||||
turns: readonly (RealSessionTurn | string)[]
|
||||
}
|
||||
|
||||
export interface RealSession {
|
||||
|
|
@ -107,7 +114,13 @@ export class RealSessionBuilder {
|
|||
const runtimeId = requireString(created, 'session_id')
|
||||
const sessionId = requireString(created, 'stored_session_id')
|
||||
|
||||
for (const text of spec.turns) {
|
||||
for (const turn of spec.turns) {
|
||||
const { images = [], text } = typeof turn === 'string' ? { text: turn } : turn
|
||||
|
||||
for (const image of images) {
|
||||
await this.request('image.attach', { session_id: runtimeId, path: image })
|
||||
}
|
||||
|
||||
const completion = this.waitForEvent(
|
||||
frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,11 +4,7 @@
|
|||
|
||||
import { expect, test, type Page } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type MockBackendFixture,
|
||||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { type MockBackendFixture, setupMockBackend, waitForAppReady } from './fixtures'
|
||||
import { MOCK_REPLY, receivedUserTexts, restartMockServer } from './mock-server'
|
||||
|
||||
async function send(page: Page, text: string, delay = 15): Promise<void> {
|
||||
|
|
@ -25,12 +21,11 @@ async function pasteAndSend(page: Page, text: string): Promise<void> {
|
|||
await page.keyboard.press('Enter')
|
||||
}
|
||||
|
||||
|
||||
async function waitForTranscript(page: Page, text: string, timeout = 90_000): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
expected => document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(expected) ?? false,
|
||||
text,
|
||||
{ timeout },
|
||||
{ timeout }
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -62,23 +57,16 @@ test.describe('session compression', () => {
|
|||
await send(page, 'E2E_COMPRESSION_THIRD')
|
||||
await expect.poll(() => receivedUserTexts().filter(text => text === 'E2E_COMPRESSION_THIRD').length).toBe(1)
|
||||
|
||||
// Commit the command before typing its argument. This waits for the async
|
||||
// completion request on cold CI workers, then uses the composer's own
|
||||
// keyboard accept path to replace the `/compress` trigger with a command
|
||||
// chip. Clicking a later completion after typing the argument can insert a
|
||||
// second command token (for example `//compress ...`) as plain text.
|
||||
// This test covers compression and continuation, not slash completion.
|
||||
// Insert the complete command atomically and click Send so an async
|
||||
// completion response cannot consume Enter as a picker acceptance.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.click()
|
||||
await composer.type('/compress', { delay: 15 })
|
||||
await page.getByText('/compress').first().waitFor({ state: 'visible' })
|
||||
await page.keyboard.press('Enter')
|
||||
await composer.type(' preserve the three test turns', { delay: 15 })
|
||||
await page.keyboard.press('Enter')
|
||||
await page.keyboard.insertText('/compress preserve the three test turns')
|
||||
await expect.poll(() => composer.textContent()).toContain('preserve the three test turns')
|
||||
await page.getByRole('button', { name: 'Send', exact: true }).click()
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator('[data-slot="aui_thread-viewport"]').textContent(),
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.poll(() => page.locator('[data-slot="aui_thread-viewport"]').textContent(), { timeout: 90_000 })
|
||||
.toMatch(/Compressed|No changes from compression/)
|
||||
|
||||
// Compression rotates the agent's live session id. A post-compression
|
||||
|
|
@ -105,7 +93,7 @@ auxiliary:
|
|||
provider: custom
|
||||
model: mock-model`,
|
||||
mockServer: {
|
||||
holdFirstCompletionContaining: 'You are a summarization agent creating a context checkpoint.',
|
||||
holdFirstCompletionContaining: 'You are a summarization agent creating a context checkpoint.'
|
||||
}
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,12 @@ import {
|
|||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { SIDEBAR_CROSS_TEXTS, SIDEBAR_TEXTS, restartMockServer } from './mock-server'
|
||||
import {
|
||||
createBackgroundReleaseHandle,
|
||||
restartMockServer,
|
||||
SIDEBAR_CROSS_TEXTS,
|
||||
SIDEBAR_TEXTS,
|
||||
} from './mock-server'
|
||||
|
||||
/** Background-running dot aria-label (from i18n en.ts). */
|
||||
const BG_DOT_LABEL = 'Background task running'
|
||||
|
|
@ -176,21 +181,30 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
// Keeps the background process alive until this test releases it, so the
|
||||
// "still running after the turn finished" state can't expire on its own.
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
// Release first so the process exits even if the test failed early,
|
||||
// then drop the sentinel file.
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('background dot transitions to finished when viewing another session', async () => {
|
||||
const page = fixture.page
|
||||
|
||||
// Start a turn with a long background process (sleep 5).
|
||||
// Start a turn whose background process runs until we release it.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 10_000 })
|
||||
await composer.click()
|
||||
|
|
@ -212,8 +226,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
{ timeout: 90_000 },
|
||||
)
|
||||
|
||||
// The background dot should still be visible (sleep 5 hasn't finished yet,
|
||||
// or auto-dismiss hasn't fired).
|
||||
// The background dot must still be visible: the turn is done but the
|
||||
// process is held open by the sentinel, so this is a stable state rather
|
||||
// than a window we have to catch in time.
|
||||
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
|
||||
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
|
||||
|
||||
|
|
@ -225,8 +240,9 @@ test.describe('sidebar states — cross-session dot transition', () => {
|
|||
await page.locator('button:has-text("New session")').first().click()
|
||||
await page.waitForTimeout(2000)
|
||||
|
||||
// Now wait for the background process to finish (sleep 5 + auto-dismiss).
|
||||
// The session A dot should transition away from "background running".
|
||||
// Now let the background process finish. The session A dot should
|
||||
// transition away from "background running".
|
||||
bgRelease.release()
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),
|
||||
|
|
|
|||
|
|
@ -22,7 +22,12 @@ import {
|
|||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { SIDEBAR_CROSS_TEXTS, restartMockServer } from './mock-server'
|
||||
import {
|
||||
type BackgroundReleaseHandle,
|
||||
createBackgroundReleaseHandle,
|
||||
restartMockServer,
|
||||
SIDEBAR_CROSS_TEXTS,
|
||||
} from './mock-server'
|
||||
|
||||
/** Finished-unread dot aria-label. */
|
||||
const UNREAD_DOT_LABEL = 'Finished — unread'
|
||||
|
|
@ -34,7 +39,7 @@ function sessionRow(page: import('@playwright/test').Page, text: string) {
|
|||
return page.locator('[data-slot="sidebar"] button').filter({ hasText: text }).first()
|
||||
}
|
||||
|
||||
/** Common setup: start a turn with a sleep 5 bg process + subagent, wait for
|
||||
/** Common setup: start a turn with a held bg process + subagent, wait for
|
||||
* the turn to complete, then switch to a new session so the first session is
|
||||
* no longer $selectedStoredSessionId (required before opening a tile). */
|
||||
async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
||||
|
|
@ -67,7 +72,9 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
|||
{ timeout: 90_000 },
|
||||
)
|
||||
|
||||
// The background dot should still be visible (sleep 5 hasn't finished).
|
||||
// The background dot must still be visible: the turn is done but the
|
||||
// process is held open by the sentinel, so this is a stable state rather
|
||||
// than a window we have to catch in time.
|
||||
const bgDuringTurn = await page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count()
|
||||
expect(bgDuringTurn, 'background dot should still be visible after turn completes').toBeGreaterThan(0)
|
||||
|
||||
|
|
@ -77,8 +84,12 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) {
|
|||
await page.waitForTimeout(2000)
|
||||
}
|
||||
|
||||
/** Wait for the background process to finish (sleep 5 + auto-dismiss). */
|
||||
async function waitForBgProcessToFinish(page: import('@playwright/test').Page) {
|
||||
/** Release the held background process, then wait for its dot to clear. */
|
||||
async function waitForBgProcessToFinish(
|
||||
page: import('@playwright/test').Page,
|
||||
release?: BackgroundReleaseHandle,
|
||||
) {
|
||||
release?.release()
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${BG_DOT_LABEL}"]`).count(),
|
||||
|
|
@ -95,15 +106,20 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('session opened as a tab (not visible) correctly gets unread dot', async () => {
|
||||
|
|
@ -123,12 +139,21 @@ test.describe('sidebar states — tab (hidden) unread is correct', () => {
|
|||
// Evidence: the tab is open but the session is not visible on screen.
|
||||
await page.screenshot({ path: 'test-results/tile-bug-tab-opened.png' })
|
||||
|
||||
await waitForBgProcessToFinish(page)
|
||||
await waitForBgProcessToFinish(page, bgRelease)
|
||||
|
||||
// A tab that's not the active tab IS hidden — the unread dot is correct.
|
||||
// The user is NOT looking at it, so marking it "unread" is right.
|
||||
const unreadCount = await page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count()
|
||||
expect(unreadCount, 'hidden tab should be marked unread').toBeGreaterThan(0)
|
||||
//
|
||||
// Poll rather than sampling once: "finished-unread" is an event-driven
|
||||
// transition that lands slightly after the running dot clears, and with a
|
||||
// released (rather than slowly-expiring) process there is no incidental
|
||||
// slack between the two. Same reasoning as the cross-session spec.
|
||||
await expect
|
||||
.poll(
|
||||
() => page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`).count(),
|
||||
{ timeout: 30_000, message: 'hidden tab should be marked unread' },
|
||||
)
|
||||
.toBeGreaterThan(0)
|
||||
|
||||
await page.screenshot({ path: 'test-results/tile-bug-tab-unread-correct.png' })
|
||||
})
|
||||
|
|
@ -142,15 +167,20 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
|
|||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
let fixture: MockBackendFixture
|
||||
const bgRelease = createBackgroundReleaseHandle()
|
||||
|
||||
test.beforeAll(async () => {
|
||||
restartMockServer()
|
||||
fixture = await setupMockBackend()
|
||||
fixture = await setupMockBackend({
|
||||
mockServer: { backgroundReleasePath: bgRelease.path },
|
||||
})
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
bgRelease.release()
|
||||
await fixture?.cleanup()
|
||||
bgRelease.cleanup()
|
||||
})
|
||||
|
||||
test('session visible in a split tile does NOT get unread dot when it finishes', async () => {
|
||||
|
|
@ -196,7 +226,7 @@ test.describe.skip('sidebar states — split (visible) unread bug (RED)', () =>
|
|||
// Evidence: the split tile is now open side-by-side — both sessions visible.
|
||||
await page.screenshot({ path: 'test-results/tile-bug-split-opened.png' })
|
||||
|
||||
await waitForBgProcessToFinish(page)
|
||||
await waitForBgProcessToFinish(page, bgRelease)
|
||||
|
||||
// THE BUG: the session visible in the split tile should NOT have the green
|
||||
// "finished unread" dot — the user is looking right at it. This assertion
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@
|
|||
* MutationObserver burst), but `$messages` was still set twice.
|
||||
*
|
||||
* The test passes when bursts === 1 AND reconciles === 0.
|
||||
* The sidebar "+" keeps the session warm in another tab. Its reactivation
|
||||
* follows the same contract: one additive paint and zero reconciles.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
|
@ -43,6 +45,11 @@ import { startMockServer } from './mock-server'
|
|||
import { RealSessionBuilder } from './real-session-builder'
|
||||
|
||||
const SESSION_TITLE = 'E2E Warm Resume Jitter Test'
|
||||
|
||||
// Inactive tabs stay mounted under a data-pane-hidden ancestor. Match the
|
||||
// renderer's keep-alive visibility policy instead of relying on DOM order.
|
||||
const SURFACE = '[data-composer-target]:not([data-pane-hidden] [data-composer-target])'
|
||||
const ALL_SURFACES = '[data-composer-target]'
|
||||
/** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */
|
||||
const MESSAGE_COUNT = 32
|
||||
/** Seeded PRNG so the generated content is deterministic across runs. */
|
||||
|
|
@ -154,15 +161,29 @@ test.afterAll(async () => {
|
|||
* after the initial paint, catching key-based reconciles that don't
|
||||
* add/remove nodes.
|
||||
*/
|
||||
async function installRenderCounter(page: import('@playwright/test').Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
async function installRenderCounter(
|
||||
page: import('@playwright/test').Page,
|
||||
transcriptText?: string,
|
||||
): Promise<void> {
|
||||
await page.evaluate(([visibleSelector, allSelector, expected]: [string, string, string | undefined]) => {
|
||||
const surfaces = [...document.querySelectorAll(expected ? allSelector : visibleSelector)]
|
||||
const surface = expected
|
||||
? surfaces.find(candidate =>
|
||||
(candidate.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
)
|
||||
: surfaces.at(-1)
|
||||
const viewport = surface?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) {
|
||||
throw new Error('Thread viewport not found before warm resume')
|
||||
}
|
||||
|
||||
const state = { bursts: 0, mutations: 0, timeline: [] as number[], stopped: false, reconciles: 0 }
|
||||
;(window as unknown as { __RENDER_COUNT__: typeof state }).__RENDER_COUNT__ = state
|
||||
const debugWindow = window as unknown as {
|
||||
__RENDER_COUNT__: typeof state
|
||||
__RENDER_VIEWPORT__: Element
|
||||
}
|
||||
debugWindow.__RENDER_COUNT__ = state
|
||||
debugWindow.__RENDER_VIEWPORT__ = viewport
|
||||
|
||||
let currentBatch = 0
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
|
@ -224,7 +245,53 @@ async function installRenderCounter(page: import('@playwright/test').Page): Prom
|
|||
hasMessages = true
|
||||
}
|
||||
}, 2)
|
||||
})
|
||||
}, [SURFACE, ALL_SURFACES, transcriptText] as [string, string, string | undefined])
|
||||
}
|
||||
|
||||
/** Wait until the ACTIVE chat surface's transcript contains `text`. */
|
||||
async function waitForActiveTranscriptText(
|
||||
page: import('@playwright/test').Page,
|
||||
text: string,
|
||||
timeout = 30_000,
|
||||
): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return (active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout },
|
||||
)
|
||||
}
|
||||
|
||||
async function waitForActiveTranscriptWithoutText(
|
||||
page: import('@playwright/test').Page,
|
||||
text: string,
|
||||
): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
([expected, surfaceSelector]: [string, string]) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const active = surfaces[surfaces.length - 1]
|
||||
|
||||
return !(active?.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected)
|
||||
},
|
||||
[text, SURFACE] as [string, string],
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
||||
/** Replace the primary surface with a draft while retaining its warm cache. */
|
||||
async function openFreshDraft(page: import('@playwright/test').Page, priorText: string): Promise<void> {
|
||||
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+N' : 'Control+N')
|
||||
await waitForActiveTranscriptWithoutText(page, priorText)
|
||||
}
|
||||
|
||||
/** Stack an empty tab while leaving the current transcript mounted and warm. */
|
||||
async function openNewSessionTab(page: import('@playwright/test').Page, priorText: string): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await waitForActiveTranscriptWithoutText(page, priorText)
|
||||
}
|
||||
|
||||
/** Stop the render counter and return the recorded burst/reconcile counts. */
|
||||
|
|
@ -245,6 +312,30 @@ async function readRenderCount(page: import('@playwright/test').Page): Promise<{
|
|||
})
|
||||
}
|
||||
|
||||
async function observedViewportIsActive(page: import('@playwright/test').Page): Promise<boolean> {
|
||||
return page.evaluate((surfaceSelector: string) => {
|
||||
const surfaces = document.querySelectorAll(surfaceSelector)
|
||||
const activeViewport = surfaces[surfaces.length - 1]?.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
const observedViewport = (window as unknown as { __RENDER_VIEWPORT__?: Element }).__RENDER_VIEWPORT__
|
||||
|
||||
return activeViewport === observedViewport
|
||||
}, SURFACE)
|
||||
}
|
||||
|
||||
/** A kept-alive tab must become visible without rebuilding its transcript. */
|
||||
function assertNoRepaint(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
|
||||
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
|
||||
expect(
|
||||
result!.bursts,
|
||||
`Expected no additive render bursts for a kept-alive tab, but got ${result!.bursts}. ` +
|
||||
`Mutation timeline: ${JSON.stringify(result!.timeline)}.`,
|
||||
).toBe(0)
|
||||
expect(
|
||||
result!.reconciles,
|
||||
`Expected no transcript reconciles for a kept-alive tab, but got ${result!.reconciles}.`,
|
||||
).toBe(0)
|
||||
}
|
||||
|
||||
/** Assert the render counter shows exactly one paint with no re-renders. */
|
||||
function assertNoJitter(result: { bursts: number; mutations: number; timeline: number[]; reconciles: number } | null): void {
|
||||
expect(result, 'MutationObserver should have recorded render data').toBeTruthy()
|
||||
|
|
@ -261,7 +352,7 @@ function assertNoJitter(result: { bursts: number; mutations: number; timeline: n
|
|||
).toBe(0)
|
||||
}
|
||||
|
||||
test('warm-route resume paints transcript exactly once (no jitter)', async ({}, testInfo) => {
|
||||
test('tab reactivation preserves the mounted transcript without repainting', async ({}, testInfo) => {
|
||||
const page = fixture!.page
|
||||
|
||||
// Wait for the sidebar to populate with our seeded session.
|
||||
|
|
@ -277,63 +368,29 @@ test('warm-route resume paints transcript exactly once (no jitter)', async ({},
|
|||
|
||||
// Wait for the transcript to appear — the first user message text confirms
|
||||
// the cold-path prefetch painted.
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
|
||||
// Wait for the session to fully settle (cold-path RPC + reconciliation).
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Step 2: Navigate away to a new chat — this does NOT evict the warm cache.
|
||||
const newSessionButton = page
|
||||
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
|
||||
.first()
|
||||
await newSessionButton.click()
|
||||
|
||||
// Wait for the new-chat empty state.
|
||||
await page.waitForFunction(
|
||||
(firstMsg: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return false
|
||||
const text = viewport.textContent ?? ''
|
||||
return !text.includes(firstMsg)
|
||||
},
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
// Stack a new tab, then observe the seeded transcript while it is hidden.
|
||||
// Installing after the switch isolates reactivation from mutations caused
|
||||
// while the new tab was being created.
|
||||
await openNewSessionTab(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(500)
|
||||
await installRenderCounter(page, FIRST_USER_MSG)
|
||||
|
||||
// Step 3: Install render counter, click back (warm resume), wait, assert.
|
||||
await installRenderCounter(page)
|
||||
// Step 3: Click back and verify the same kept-alive viewport becomes active
|
||||
// without rebuilding or reconciling its transcript.
|
||||
await sessionRow.click()
|
||||
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
|
||||
// Wait for at least 1 burst, then settle.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const w = window as unknown as { __RENDER_COUNT__?: { bursts: number } }
|
||||
return Boolean(w.__RENDER_COUNT__ && w.__RENDER_COUNT__.bursts > 0)
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 10_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(2_000)
|
||||
expect(await observedViewportIsActive(page), 'Reactivation should reveal the observed kept-alive viewport').toBe(true)
|
||||
|
||||
const result = await readRenderCount(page)
|
||||
await page.screenshot({ path: testInfo.outputPath('warm-resume-idle.png') })
|
||||
assertNoJitter(result)
|
||||
assertNoRepaint(result)
|
||||
})
|
||||
|
||||
test('warm-route resume after background inference completes (no jitter)', async ({}, testInfo) => {
|
||||
|
|
@ -354,13 +411,7 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
|
||||
// Step 1: Cold resume — populate the warm cache.
|
||||
await sessionRow.click()
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Step 2: Send a message — triggers inference via the mock server.
|
||||
|
|
@ -373,34 +424,15 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
// Wait for the mock response to appear in the transcript, confirming
|
||||
// the turn completed and message.complete fired (which updates the warm
|
||||
// cache via updateSessionState).
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
return viewport?.textContent?.includes('mock inference server') ?? false
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, 'mock inference server', 60_000)
|
||||
// Extra settle for message.complete → updateSessionState → cache write.
|
||||
await page.waitForTimeout(2_000)
|
||||
|
||||
// Verify the prompt was received by the mock server.
|
||||
expect(mock.receivedPrompts).toContain(PROMPT)
|
||||
|
||||
// Step 3: Navigate away — the warm cache retains the updated messages.
|
||||
const newSessionButton = page
|
||||
.locator('[data-slot="sidebar"] button[aria-label="New session"]')
|
||||
.first()
|
||||
await newSessionButton.click()
|
||||
await page.waitForFunction(
|
||||
(prompt: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return false
|
||||
return !(viewport.textContent ?? '').includes(prompt)
|
||||
},
|
||||
PROMPT,
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
// Step 3: Replace the primary chat; the warm cache retains the updated messages.
|
||||
await openFreshDraft(page, PROMPT)
|
||||
await page.waitForTimeout(500)
|
||||
|
||||
// Step 4: Install render counter, click back (warm resume), wait, assert.
|
||||
|
|
@ -409,13 +441,7 @@ test('warm-route resume after background inference completes (no jitter)', async
|
|||
|
||||
// Wait for the transcript to reappear — the warm cache should already
|
||||
// have the completed turn (updated by message.complete events).
|
||||
await page.waitForFunction(
|
||||
(text: string) =>
|
||||
document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent?.includes(text) ??
|
||||
false,
|
||||
FIRST_USER_MSG,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await waitForActiveTranscriptText(page, FIRST_USER_MSG)
|
||||
|
||||
// Wait for at least 1 burst, then settle.
|
||||
await page.waitForFunction(
|
||||
|
|
|
|||
60
apps/desktop/electron/active-runtime-state.test.ts
Normal file
60
apps/desktop/electron/active-runtime-state.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { classifyActiveRuntime, hasValidBootstrapMarker } from './active-runtime-state'
|
||||
|
||||
const VALID_MARKER = {
|
||||
pinnedCommit: '1234567890abcdef1234567890abcdef12345678',
|
||||
schemaVersion: 1
|
||||
}
|
||||
|
||||
test('hasValidBootstrapMarker accepts the current schema with a real-looking commit', () => {
|
||||
assert.equal(hasValidBootstrapMarker(VALID_MARKER, 1), true)
|
||||
})
|
||||
|
||||
test('hasValidBootstrapMarker rejects missing, wrong-schema, and too-short markers', () => {
|
||||
assert.equal(hasValidBootstrapMarker(null, 1), false)
|
||||
assert.equal(hasValidBootstrapMarker({ schemaVersion: 2, pinnedCommit: VALID_MARKER.pinnedCommit }, 1), false)
|
||||
assert.equal(hasValidBootstrapMarker({ schemaVersion: 1, pinnedCommit: 'abc123' }, 1), false)
|
||||
})
|
||||
|
||||
test('classifyActiveRuntime uses a healthy active runtime even when the bootstrap marker is missing', () => {
|
||||
assert.deepEqual(classifyActiveRuntime(null, 1, true), {
|
||||
hasValidMarker: false,
|
||||
shouldUseActiveRuntime: true,
|
||||
usabilityReason: 'usable'
|
||||
})
|
||||
})
|
||||
|
||||
test('classifyActiveRuntime uses a healthy active runtime even when the marker is stale or malformed', () => {
|
||||
assert.deepEqual(classifyActiveRuntime({ schemaVersion: 999, pinnedCommit: 'abc1234' }, 1, true), {
|
||||
hasValidMarker: false,
|
||||
shouldUseActiveRuntime: true,
|
||||
usabilityReason: 'usable'
|
||||
})
|
||||
})
|
||||
|
||||
test('classifyActiveRuntime refuses an unusable runtime even if a valid marker exists', () => {
|
||||
assert.deepEqual(classifyActiveRuntime(VALID_MARKER, 1, false), {
|
||||
hasValidMarker: true,
|
||||
shouldUseActiveRuntime: false,
|
||||
usabilityReason: 'unusable'
|
||||
})
|
||||
})
|
||||
|
||||
test('a CLI-installed runtime with no marker launches instead of re-running bootstrap', () => {
|
||||
// The reported symptom (#60721): install.sh / install.ps1 produced a healthy
|
||||
// repo+venv, no desktop-managed marker was ever written, and every launch
|
||||
// dropped the user back into the first-run installer.
|
||||
const state = classifyActiveRuntime(null, 1, true)
|
||||
|
||||
assert.equal(state.shouldUseActiveRuntime, true, 'a usable runtime must launch')
|
||||
assert.equal(state.hasValidMarker, false, 'marker provenance stays honest')
|
||||
})
|
||||
|
||||
test('a repair that deleted the marker does not strand a healthy install', () => {
|
||||
// #72166: the repair handler clears the marker unconditionally. Runtime
|
||||
// usability, not marker presence, must decide the next boot.
|
||||
assert.equal(classifyActiveRuntime(null, 1, true).shouldUseActiveRuntime, true)
|
||||
})
|
||||
58
apps/desktop/electron/active-runtime-state.ts
Normal file
58
apps/desktop/electron/active-runtime-state.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
export interface BootstrapMarkerLike {
|
||||
pinnedCommit?: unknown
|
||||
schemaVersion?: unknown
|
||||
}
|
||||
|
||||
export interface ActiveRuntimeState {
|
||||
hasValidMarker: boolean
|
||||
shouldUseActiveRuntime: boolean
|
||||
usabilityReason: 'usable' | 'unusable'
|
||||
}
|
||||
|
||||
export function hasValidBootstrapMarker(
|
||||
marker: BootstrapMarkerLike | null | undefined,
|
||||
schemaVersion: number
|
||||
): boolean {
|
||||
if (!marker || typeof marker !== 'object') {
|
||||
return false
|
||||
}
|
||||
|
||||
if (marker.schemaVersion !== schemaVersion) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (typeof marker.pinnedCommit !== 'string' || marker.pinnedCommit.length < 7) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// The active install at ~/.hermes/hermes-agent can be real and runnable even if
|
||||
// Desktop never wrote its first-run bootstrap marker (for example when Hermes
|
||||
// was installed by the CLI first, or when a past desktop build forgot the
|
||||
// marker). Runtime usability is authoritative for "can we launch local Hermes
|
||||
// right now?"; the marker is only provenance about how that install was
|
||||
// created. A missing/stale marker must never force a healthy local install into
|
||||
// the first-run bootstrap UI.
|
||||
export function classifyActiveRuntime(
|
||||
marker: BootstrapMarkerLike | null | undefined,
|
||||
schemaVersion: number,
|
||||
runtimeUsable: boolean
|
||||
): ActiveRuntimeState {
|
||||
const hasValidMarker = hasValidBootstrapMarker(marker, schemaVersion)
|
||||
|
||||
if (!runtimeUsable) {
|
||||
return {
|
||||
hasValidMarker,
|
||||
shouldUseActiveRuntime: false,
|
||||
usabilityReason: 'unusable'
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
hasValidMarker,
|
||||
shouldUseActiveRuntime: true,
|
||||
usabilityReason: 'usable'
|
||||
}
|
||||
}
|
||||
|
|
@ -68,6 +68,26 @@ test('buildDesktopBackendEnv extends PYTHONPATH and backend PATH together', () =
|
|||
assert.ok(env.PATH.includes('/opt/homebrew/bin'))
|
||||
})
|
||||
|
||||
test('buildDesktopBackendEnv forces PYTHONUTF8 unless the user set it explicitly', () => {
|
||||
const defaulted = buildDesktopBackendEnv({
|
||||
hermesHome: '/Users/test/.hermes',
|
||||
currentEnv: { PATH: '/usr/bin' },
|
||||
platform: 'darwin',
|
||||
pathModule: path.posix
|
||||
})
|
||||
|
||||
assert.equal(defaulted.PYTHONUTF8, '1')
|
||||
|
||||
const optedOut = buildDesktopBackendEnv({
|
||||
hermesHome: '/Users/test/.hermes',
|
||||
currentEnv: { PATH: '/usr/bin', PYTHONUTF8: '0' },
|
||||
platform: 'darwin',
|
||||
pathModule: path.posix
|
||||
})
|
||||
|
||||
assert.equal(optedOut.PYTHONUTF8, '0')
|
||||
})
|
||||
|
||||
test('normalizeHermesHomeRoot maps profile homes back to the global Hermes root', () => {
|
||||
assert.equal(
|
||||
normalizeHermesHomeRoot('/Users/test/.hermes/profiles/oracle', { pathModule: path.posix }),
|
||||
|
|
|
|||
|
|
@ -104,6 +104,13 @@ function buildDesktopBackendEnv({
|
|||
|
||||
return {
|
||||
PYTHONPATH: appendUniquePathEntries([...pythonPathEntries, currentPythonPath], { delimiter }),
|
||||
// Force PEP 540 UTF-8 mode in the spawned Python backend so its stdio and
|
||||
// subprocess defaults are UTF-8 even on non-UTF-8 Windows locales (GBK,
|
||||
// cp1252, ...). hermes_bootstrap sets this inside the child too, but only
|
||||
// after import — anything emitted earlier (interpreter startup errors,
|
||||
// pre-bootstrap tracebacks) still decodes with the locale default without
|
||||
// this. User's explicit setting wins. Re-port of PR #56499 (echoriver89).
|
||||
PYTHONUTF8: currentEnv?.PYTHONUTF8 ?? '1',
|
||||
[key]: buildDesktopBackendPath({
|
||||
hermesHome,
|
||||
venvRoot,
|
||||
|
|
|
|||
340
apps/desktop/electron/backend-health.test.ts
Normal file
340
apps/desktop/electron/backend-health.test.ts
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import {
|
||||
DEFAULT_HEALTH_PROBE_TIMEOUT_MS,
|
||||
isAuthRejectionError,
|
||||
isGatedMissingHealthError,
|
||||
isMissingHealthEndpointError,
|
||||
isReauthRequiredError,
|
||||
waitForHermesReady
|
||||
} from './backend-health'
|
||||
|
||||
const GATE_401 = '401: {"error":"unauthenticated","detail":"Unauthorized","reason":"no_cookie","login_url":"/login"}'
|
||||
|
||||
test('uses lightweight /api/health for current backends', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000/', {
|
||||
token: 'secret-token',
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
|
||||
return { ok: true }
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
throw new Error('status should not be called')
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [['public', 'http://127.0.0.1:9000/api/health']])
|
||||
})
|
||||
|
||||
test('falls back to /api/status only for old backends without /api/health', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000', {
|
||||
token: 'secret-token',
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
fetchJson: async (url, token) => {
|
||||
calls.push(['token', url, token ?? ''])
|
||||
|
||||
return { version: 'old' }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['public', 'http://127.0.0.1:9000/api/health'],
|
||||
['token', 'http://127.0.0.1:9000/api/status', 'secret-token']
|
||||
])
|
||||
})
|
||||
|
||||
test('does not fall back to heavyweight /api/status for transient health failures', async () => {
|
||||
const calls: string[][] = []
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error('Timed out connecting to Hermes backend after 15000ms')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
},
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 50,
|
||||
pollMs: 1
|
||||
}),
|
||||
/Timed out connecting/
|
||||
)
|
||||
|
||||
assert.ok(calls.length > 0)
|
||||
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
||||
})
|
||||
|
||||
test('probes health on a short timeout but leaves the legacy fallback its own', async () => {
|
||||
const timeouts: (number | undefined)[] = []
|
||||
|
||||
await waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async (_url, options) => {
|
||||
timeouts.push(options?.timeoutMs)
|
||||
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
fetchJson: async (_url, _token, options) => {
|
||||
timeouts.push(options?.timeoutMs)
|
||||
|
||||
return { version: 'old' }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(timeouts, [DEFAULT_HEALTH_PROBE_TIMEOUT_MS, undefined])
|
||||
})
|
||||
|
||||
test('aborts as superseded when the bootstrap signal fires', async () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
signal: controller.signal,
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('should not probe after abort')
|
||||
},
|
||||
fetchJson: async () => {
|
||||
throw new Error('should not probe after abort')
|
||||
},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => error.kind === 'superseded'
|
||||
)
|
||||
})
|
||||
|
||||
test('recognizes missing-route shapes only', () => {
|
||||
assert.equal(isMissingHealthEndpointError(new Error('404: {"detail":"Not Found"}')), true)
|
||||
assert.equal(
|
||||
isMissingHealthEndpointError(
|
||||
new Error('Expected JSON from /api/health but got HTML. The endpoint is likely missing on the Hermes backend.')
|
||||
),
|
||||
true
|
||||
)
|
||||
assert.equal(isMissingHealthEndpointError(new Error('Timed out connecting to Hermes backend after 15000ms')), false)
|
||||
assert.equal(isMissingHealthEndpointError(new Error('500: boom')), false)
|
||||
})
|
||||
|
||||
// --- Gated backends that predate /api/health (release 0.19.0 and earlier) ---
|
||||
//
|
||||
// The dashboard auth gate runs ahead of the SPA catch-all, so on a backend
|
||||
// without the route an ANONYMOUS probe is rejected as unauthenticated rather
|
||||
// than 404 — verified against a simulated 0.19.0 backend:
|
||||
// credential-free: /api/health -> 401 no_cookie, /api/status -> 200
|
||||
// credentialed: /api/health -> 404, /api/sessions -> 200
|
||||
|
||||
test('anonymous gate-shaped 401 falls back to /api/status (backend predates /api/health)', async () => {
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('http://192.168.1.132:9119', {
|
||||
token: null,
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error(GATE_401)
|
||||
},
|
||||
fetchJson: async (url, token) => {
|
||||
calls.push(['token', url, token == null ? 'null' : token])
|
||||
|
||||
return { version: '0.19.0', auth_required: true }
|
||||
},
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['public', 'http://192.168.1.132:9119/api/health'],
|
||||
['token', 'http://192.168.1.132:9119/api/status', 'null']
|
||||
])
|
||||
})
|
||||
|
||||
test('a credentialed 401 fails fast for reauth instead of reporting a dead session ready', async () => {
|
||||
// The regression a blanket 401->fallback introduces: /api/status is public,
|
||||
// so an expired session would answer 200 and boot would report "ready",
|
||||
// deferring the no_cookie to the first real API call.
|
||||
const calls: string[][] = []
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
token: 'session-token',
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('public probe must not be used when credentialed')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['status', url])
|
||||
|
||||
return { version: '0.19.0' }
|
||||
},
|
||||
probeHealth: async url => {
|
||||
calls.push(['probe', url])
|
||||
throw new Error(GATE_401)
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => {
|
||||
assert.equal(isReauthRequiredError(error), true)
|
||||
assert.equal(error.needsOauthLogin, true)
|
||||
assert.match(error.message, /remote gateway session has expired/i)
|
||||
|
||||
return true
|
||||
}
|
||||
)
|
||||
|
||||
// Fail fast: never reached the public /api/status leg.
|
||||
assert.deepEqual(calls, [['probe', 'https://gateway.example/api/health']])
|
||||
})
|
||||
|
||||
test('a credentialed 403 is also a terminal reauth failure', async () => {
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
fetchPublicJson: async () => ({}),
|
||||
fetchJson: async () => ({}),
|
||||
probeHealth: async () => {
|
||||
throw new Error('403: {"detail":"Forbidden"}')
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => isReauthRequiredError(error)
|
||||
)
|
||||
})
|
||||
|
||||
test('a credentialed probe still uses the 404 fallback for a genuinely missing route', async () => {
|
||||
// With credentials the gate lets the request through to the SPA catch-all,
|
||||
// so an old backend answers a real 404 — that must still fall back, not be
|
||||
// mistaken for a rejected session.
|
||||
const calls: string[][] = []
|
||||
|
||||
await waitForHermesReady('https://gateway.example', {
|
||||
token: 'session-token',
|
||||
fetchPublicJson: async () => {
|
||||
throw new Error('public probe must not be used when credentialed')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['status', url])
|
||||
|
||||
return { version: '0.19.0' }
|
||||
},
|
||||
probeHealth: async url => {
|
||||
calls.push(['probe', url])
|
||||
throw new Error('404: {"detail":"Not Found"}')
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
['probe', 'https://gateway.example/api/health'],
|
||||
['status', 'https://gateway.example/api/status']
|
||||
])
|
||||
})
|
||||
|
||||
test('a non-gate 401 keeps polling rather than skipping a misconfigured health route', async () => {
|
||||
const calls: string[][] = []
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('http://127.0.0.1:9000', {
|
||||
fetchPublicJson: async url => {
|
||||
calls.push(['public', url])
|
||||
throw new Error('401: {"detail":"Unauthorized"}')
|
||||
},
|
||||
fetchJson: async url => {
|
||||
calls.push(['token', url])
|
||||
},
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 50,
|
||||
pollMs: 1
|
||||
}),
|
||||
/401: \{"detail":"Unauthorized"\}/
|
||||
)
|
||||
|
||||
assert.ok(calls.length > 0)
|
||||
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
||||
})
|
||||
|
||||
test('credentialed 5xx and 429 keep polling — only 401/403 are terminal', async () => {
|
||||
for (const transient of ['500: boom', '429: {"detail":"Too Many Requests"}']) {
|
||||
let attempts = 0
|
||||
let currentTime = 0
|
||||
|
||||
await assert.rejects(
|
||||
waitForHermesReady('https://gateway.example', {
|
||||
fetchPublicJson: async () => ({}),
|
||||
fetchJson: async () => ({}),
|
||||
probeHealth: async () => {
|
||||
attempts += 1
|
||||
throw new Error(transient)
|
||||
},
|
||||
probeIsCredentialed: true,
|
||||
sleep: async () => {},
|
||||
now: () => {
|
||||
currentTime += 20
|
||||
|
||||
return currentTime
|
||||
},
|
||||
timeoutMs: 100,
|
||||
pollMs: 1
|
||||
}),
|
||||
(error: any) => isReauthRequiredError(error) === false
|
||||
)
|
||||
|
||||
assert.ok(attempts > 1, `${transient} should have retried, got ${attempts} attempt(s)`)
|
||||
}
|
||||
})
|
||||
|
||||
test('error-shape predicates', () => {
|
||||
assert.equal(isGatedMissingHealthError(new Error(GATE_401)), true)
|
||||
assert.equal(isGatedMissingHealthError(new Error('401: {"detail":"Unauthorized"}')), false)
|
||||
assert.equal(isGatedMissingHealthError(new Error('404: {"detail":"Not Found"}')), false)
|
||||
|
||||
assert.equal(isAuthRejectionError(new Error(GATE_401)), true)
|
||||
assert.equal(isAuthRejectionError(new Error('403: {"detail":"Forbidden"}')), true)
|
||||
assert.equal(isAuthRejectionError(new Error('404: {"detail":"Not Found"}')), false)
|
||||
assert.equal(isAuthRejectionError(new Error('429: slow down')), false)
|
||||
assert.equal(isAuthRejectionError(new Error('500: boom')), false)
|
||||
|
||||
// A gated 401 must NOT be conflated with a missing route by the 404 predicate.
|
||||
assert.equal(isMissingHealthEndpointError(new Error(GATE_401)), false)
|
||||
})
|
||||
169
apps/desktop/electron/backend-health.ts
Normal file
169
apps/desktop/electron/backend-health.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
export const DEFAULT_BACKEND_READY_TIMEOUT_MS = 45_000
|
||||
export const DEFAULT_BACKEND_READY_POLL_MS = 500
|
||||
// A cold backend can stall its event loop for tens of seconds while Windows
|
||||
// scans and byte-compiles the gateway import tree. At the default 15s socket
|
||||
// timeout only three probes fit in the budget; a short one keeps retrying
|
||||
// across the stall. Health only — the legacy /api/status fallback is genuinely
|
||||
// slow to answer and keeps the caller's default timeout.
|
||||
export const DEFAULT_HEALTH_PROBE_TIMEOUT_MS = 5_000
|
||||
|
||||
type FetchPublicJson = (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
type FetchJson = (url: string, token?: string | null, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
|
||||
export interface HermesReadyOptions {
|
||||
fetchPublicJson: FetchPublicJson
|
||||
fetchJson: FetchJson
|
||||
token?: string | null
|
||||
signal?: AbortSignal
|
||||
timeoutMs?: number
|
||||
pollMs?: number
|
||||
healthProbeTimeoutMs?: number
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
now?: () => number
|
||||
/**
|
||||
* Credentialed health probe. When supplied, readiness is probed with the
|
||||
* connection's own credentials instead of anonymously — which is what lets
|
||||
* a gated backend answer 404 for a genuinely missing /api/health, and what
|
||||
* makes a 401 from this probe mean "session rejected" rather than "route
|
||||
* behind a gate". Defaults to the credential-free `fetchPublicJson`.
|
||||
*/
|
||||
probeHealth?: (url: string, options?: { timeoutMs?: number }) => Promise<unknown>
|
||||
/**
|
||||
* Whether `probeHealth` actually presents credentials. Distinguishes the
|
||||
* two very different meanings of a 401 (see `waitForHermesReady`).
|
||||
*/
|
||||
probeIsCredentialed?: boolean
|
||||
}
|
||||
|
||||
export const REMOTE_SESSION_EXPIRED_MESSAGE =
|
||||
'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.'
|
||||
|
||||
export function isMissingHealthEndpointError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return /^404:/.test(message) || message.includes('endpoint is likely missing')
|
||||
}
|
||||
|
||||
/**
|
||||
* True for a hard auth rejection (401/403) as opposed to a transient failure.
|
||||
* Deliberately shape-based: 429 is a throttle and 5xx is a server fault, and
|
||||
* both must keep polling.
|
||||
*/
|
||||
export function isAuthRejectionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return /^40[13]:/.test(message)
|
||||
}
|
||||
|
||||
/**
|
||||
* True for an auth rejection carrying the dashboard gate's "no session at all"
|
||||
* shape. On a backend that predates `/api/health`, the gate runs ahead of the
|
||||
* SPA catch-all, so an unknown `/api/*` path is rejected as unauthenticated
|
||||
* instead of 404 — this is the signal that an ANONYMOUS probe cannot reach the
|
||||
* route, and the reason a credential-free 401 must fall back to `/api/status`
|
||||
* rather than be reported as a boot failure.
|
||||
*/
|
||||
export function isGatedMissingHealthError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
return isAuthRejectionError(error) && message.includes('no_cookie')
|
||||
}
|
||||
|
||||
/** Tag a terminal reauth failure the main process latches and the overlay keys on. */
|
||||
export function makeReauthRequiredError(detail?: string): Error {
|
||||
const error = new Error(REMOTE_SESSION_EXPIRED_MESSAGE) as any
|
||||
error.needsOauthLogin = true
|
||||
error.isReauthRequired = true
|
||||
|
||||
if (detail) {
|
||||
error.detail = detail
|
||||
}
|
||||
|
||||
return error
|
||||
}
|
||||
|
||||
export function isReauthRequiredError(error: unknown): boolean {
|
||||
return Boolean((error as any)?.isReauthRequired)
|
||||
}
|
||||
|
||||
function supersededError() {
|
||||
const error: any = new Error('SSH bootstrap was superseded by newer connection settings.')
|
||||
error.kind = 'superseded'
|
||||
|
||||
return error
|
||||
}
|
||||
|
||||
export async function waitForHermesReady(baseUrl: string, options: HermesReadyOptions): Promise<void> {
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_BACKEND_READY_TIMEOUT_MS
|
||||
const pollMs = options.pollMs ?? DEFAULT_BACKEND_READY_POLL_MS
|
||||
const healthProbeTimeoutMs = options.healthProbeTimeoutMs ?? DEFAULT_HEALTH_PROBE_TIMEOUT_MS
|
||||
const now = options.now ?? Date.now
|
||||
const signal = options.signal
|
||||
|
||||
const sleep =
|
||||
options.sleep ??
|
||||
(ms =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms)
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timer)
|
||||
reject(supersededError())
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
}))
|
||||
|
||||
const base = baseUrl.replace(/\/+$/, '')
|
||||
const deadline = now() + timeoutMs
|
||||
const probeHealth = options.probeHealth ?? options.fetchPublicJson
|
||||
const probeIsCredentialed = Boolean(options.probeIsCredentialed)
|
||||
let lastError: unknown = null
|
||||
let useStatusFallback = false
|
||||
|
||||
while (now() < deadline) {
|
||||
if (signal?.aborted) {
|
||||
throw supersededError()
|
||||
}
|
||||
|
||||
try {
|
||||
if (useStatusFallback) {
|
||||
await options.fetchJson(`${base}/api/status`, options.token)
|
||||
} else {
|
||||
await probeHealth(`${base}/api/health`, { timeoutMs: healthProbeTimeoutMs })
|
||||
}
|
||||
|
||||
return
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
|
||||
// A confirmed 401/403 from a CREDENTIALED probe means the session was
|
||||
// rejected, not that the route is missing. Fail fast into a reauth
|
||||
// state: falling back to the public /api/status would answer 200 and
|
||||
// report a dead session as "ready", deferring the failure to the first
|
||||
// real API call. Applies to the /api/status leg too — it is routed
|
||||
// through the same credentials.
|
||||
if (probeIsCredentialed && isAuthRejectionError(error)) {
|
||||
throw makeReauthRequiredError(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
|
||||
// An explicitly missing route means the backend predates /api/health.
|
||||
// So does a gate-shaped 401 on an ANONYMOUS probe: the dashboard auth
|
||||
// gate runs ahead of the SPA catch-all, so a pre-/api/health backend
|
||||
// rejects the unknown path as unauthenticated instead of 404 and a
|
||||
// credential-free probe can never observe the 404. Timeouts, 5xx, 429,
|
||||
// and non-gate 401s keep polling health.
|
||||
if (!useStatusFallback && (isMissingHealthEndpointError(error) || isGatedMissingHealthError(error))) {
|
||||
useStatusFallback = true
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
await sleep(pollMs)
|
||||
}
|
||||
}
|
||||
|
||||
const detail = lastError instanceof Error ? lastError.message : 'timeout'
|
||||
throw new Error(`Hermes backend did not become ready: ${detail}`)
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
|
|||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { shouldLatchBackendStartFailure } from './backend-start-failure'
|
||||
import { shouldLatchBackendStartFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure'
|
||||
|
||||
test('latches a LOCAL backend failure so the install-retry loop is broken', () => {
|
||||
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false }), true)
|
||||
|
|
@ -21,3 +21,32 @@ test('the two branches are mutually exclusive (a failure either latches or stays
|
|||
assert.equal(latched, !attemptedRemote)
|
||||
}
|
||||
})
|
||||
|
||||
test('latches a CONFIRMED remote reauth failure so the overlay stays clickable', () => {
|
||||
// Without this the non-latching remote path re-runs startHermes on every
|
||||
// getConnection/api call, re-emits running:true, and the overlay hides
|
||||
// itself — the "Sign in" button flickers away before it can be clicked.
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: true }), true)
|
||||
})
|
||||
|
||||
test('does not latch a transient remote failure as reauth', () => {
|
||||
// A mint timeout or a host unreachable across sleep must still self-heal.
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: false }), false)
|
||||
})
|
||||
|
||||
test('never latches a LOCAL failure as reauth (that is backendStartFailure job)', () => {
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: true }), false)
|
||||
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: false }), false)
|
||||
})
|
||||
|
||||
test('the two latches never fire for the same failure', () => {
|
||||
// They are complementary, not overlapping: local failures latch via
|
||||
// backendStartFailure, confirmed remote reauth latches via its own flag.
|
||||
for (const attemptedRemote of [true, false]) {
|
||||
for (const isReauth of [true, false]) {
|
||||
const start = shouldLatchBackendStartFailure({ attemptedRemote })
|
||||
const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth })
|
||||
assert.ok(!(start && reauth), `both latched for remote=${attemptedRemote} reauth=${isReauth}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -39,3 +39,34 @@ export interface BackendStartFailureContext {
|
|||
export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean {
|
||||
return !context.attemptedRemote
|
||||
}
|
||||
|
||||
export interface RemoteReauthFailureContext {
|
||||
/** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */
|
||||
attemptedRemote: boolean
|
||||
/**
|
||||
* True when the failure was a CONFIRMED auth rejection (a credentialed
|
||||
* probe got 401/403), not a transient connectivity fault.
|
||||
*/
|
||||
isReauth: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a failed remote boot should latch as a reauth failure.
|
||||
*
|
||||
* This is the deliberate counterpart to `shouldLatchBackendStartFailure`,
|
||||
* which never latches a remote failure because remote faults are usually
|
||||
* transient and must stay retryable. A *confirmed* reauth rejection is the
|
||||
* exception: it cannot self-heal, because nothing will change until the user
|
||||
* signs in again.
|
||||
*
|
||||
* Without a latch, the non-latching remote path actively prevents recovery.
|
||||
* Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits
|
||||
* `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error)
|
||||
* && !boot.running`) hides itself — so the "Sign in" button flickers out from
|
||||
* under the user before they can click it. Latching holds the overlay still
|
||||
* and clickable. Cleared on every recovery path (reset, repair, apply-config,
|
||||
* and a confirmed sign-in) so a fresh session boots normally.
|
||||
*/
|
||||
export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean {
|
||||
return context.attemptedRemote && context.isReauth
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
async function applyConnectionChange({
|
||||
cancelAndWait,
|
||||
isPrimary,
|
||||
rehomePrimary = null,
|
||||
scope,
|
||||
sendApplied,
|
||||
stopPool,
|
||||
|
|
@ -16,6 +17,12 @@ async function applyConnectionChange({
|
|||
return
|
||||
}
|
||||
|
||||
if (rehomePrimary) {
|
||||
await rehomePrimary()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await teardownPrimary()
|
||||
sendApplied()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import {
|
|||
profileRemoteOverride,
|
||||
profileSshOverride,
|
||||
resolveAuthMode,
|
||||
resolveProfileBackendRoute,
|
||||
resolveTestWsUrl,
|
||||
RT_COOKIE_VARIANTS,
|
||||
savedProfileSsh,
|
||||
|
|
@ -187,6 +188,65 @@ test('saved SSH drafts are inactive and explicit overrides take precedence', ()
|
|||
assert.equal(profileHasRemoteConnection(config, 'coder'), true)
|
||||
})
|
||||
|
||||
// --- resolveProfileBackendRoute ---
|
||||
|
||||
const ROUTES = [
|
||||
{
|
||||
name: 'the primary profile owns the window backend',
|
||||
profile: 'default',
|
||||
opts: { primaryProfile: 'default' },
|
||||
expected: { backend: 'primary', descriptorProfile: null, scopePath: false }
|
||||
},
|
||||
{
|
||||
name: 'a renamed primary profile still owns the window backend',
|
||||
profile: ' coder ',
|
||||
opts: { primaryProfile: 'coder', globalRemote: true },
|
||||
expected: { backend: 'primary', descriptorProfile: null, scopePath: false }
|
||||
},
|
||||
{
|
||||
name: 'an unset profile resolves to the primary',
|
||||
profile: '',
|
||||
opts: { primaryProfile: 'default', globalRemote: true },
|
||||
expected: { backend: 'primary', descriptorProfile: null, scopePath: false }
|
||||
},
|
||||
{
|
||||
name: 'a profile inheriting the app-global remote shares the primary backend, scoped per request',
|
||||
profile: 'coder',
|
||||
opts: { primaryProfile: 'default', globalRemote: true, profileRemoteOverride: false },
|
||||
expected: { backend: 'primary', descriptorProfile: 'coder', scopePath: true }
|
||||
},
|
||||
{
|
||||
name: 'a profile with its own remote override gets a pooled descriptor for that host',
|
||||
profile: 'coder',
|
||||
opts: { primaryProfile: 'default', globalRemote: true, profileRemoteOverride: true },
|
||||
expected: { backend: 'pool', descriptorProfile: null, scopePath: false }
|
||||
},
|
||||
{
|
||||
name: 'a local non-primary profile gets its own pooled backend',
|
||||
profile: 'coder',
|
||||
opts: { primaryProfile: 'default', globalRemote: false, profileRemoteOverride: false },
|
||||
expected: { backend: 'pool', descriptorProfile: null, scopePath: false }
|
||||
}
|
||||
]
|
||||
|
||||
for (const route of ROUTES) {
|
||||
test(`resolveProfileBackendRoute: ${route.name}`, () => {
|
||||
assert.deepEqual(resolveProfileBackendRoute(route.profile, route.opts), route.expected)
|
||||
})
|
||||
}
|
||||
|
||||
test('resolveProfileBackendRoute only tags a descriptor when the backend is shared', () => {
|
||||
// A pooled backend is already scoped to its profile, so tagging it would
|
||||
// imply a second scope the caller must reconcile. Only the shared
|
||||
// global-remote route carries one.
|
||||
for (const route of ROUTES) {
|
||||
const resolved = resolveProfileBackendRoute(route.profile, route.opts)
|
||||
|
||||
assert.equal(Boolean(resolved.descriptorProfile), resolved.scopePath)
|
||||
assert.ok(!resolved.descriptorProfile || resolved.backend === 'primary')
|
||||
}
|
||||
})
|
||||
|
||||
// --- pathWithGlobalRemoteProfile ---
|
||||
|
||||
test('pathWithGlobalRemoteProfile appends profile in global remote mode', () => {
|
||||
|
|
@ -199,6 +259,17 @@ test('pathWithGlobalRemoteProfile appends profile in global remote mode', () =>
|
|||
)
|
||||
})
|
||||
|
||||
test('pathWithGlobalRemoteProfile skips the primary profile, which the remote already serves', () => {
|
||||
assert.equal(
|
||||
pathWithGlobalRemoteProfile('/api/model/info', 'coder', {
|
||||
globalRemote: true,
|
||||
primaryProfile: 'coder',
|
||||
profileRemoteOverride: false
|
||||
}),
|
||||
'/api/model/info'
|
||||
)
|
||||
})
|
||||
|
||||
test('pathWithGlobalRemoteProfile preserves existing query params', () => {
|
||||
assert.equal(
|
||||
pathWithGlobalRemoteProfile('/api/model/options?force=1', 'iris', {
|
||||
|
|
|
|||
|
|
@ -350,16 +350,68 @@ function profileRemoteOverride(config, profile) {
|
|||
return { url, authMode: normAuthMode(entry.authMode), token: entry.token }
|
||||
}
|
||||
|
||||
export interface ProfileRouteOptions {
|
||||
globalRemote?: boolean
|
||||
primaryProfile?: null | string
|
||||
profileRemoteOverride?: boolean
|
||||
}
|
||||
|
||||
export interface ProfileBackendRoute {
|
||||
/** Which backend serves this profile: the window backend, or a pooled one. */
|
||||
backend: 'pool' | 'primary'
|
||||
/**
|
||||
* Profile to tag on the returned descriptor when the backend is shared and
|
||||
* therefore not itself scoped to that profile. Null when the backend already
|
||||
* belongs to the profile.
|
||||
*/
|
||||
descriptorProfile: null | string
|
||||
/** Whether REST paths on this route must carry `?profile=` to be scoped. */
|
||||
scopePath: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* In global-remote mode one backend serves every Desktop profile, so REST calls
|
||||
* that are scoped by renderer-side `request.profile` must carry that scope as a
|
||||
* query parameter. Local pooled backends and per-profile remote overrides do not
|
||||
* need this: they already run against a backend scoped to the target profile.
|
||||
* The one place that answers "which backend serves profile P, and does its
|
||||
* REST path need a profile scope?". Four routes, in precedence order:
|
||||
*
|
||||
* 1. The primary profile owns the window backend outright.
|
||||
* 2. A profile with its own remote override gets a pooled descriptor for that
|
||||
* host, which is already scoped to it.
|
||||
* 3. A profile inheriting the app-global remote shares the primary backend —
|
||||
* one host serves every profile — so it is scoped per request instead.
|
||||
* 4. Any other local profile gets its own pooled backend, spawned with
|
||||
* `--profile`, so its `HERMES_HOME` scopes it.
|
||||
*
|
||||
* Routing used to be spread across three overlapping predicates that each
|
||||
* re-derived part of this table, which is how case 3 ended up registering
|
||||
* reapable pool entries for backends it never owned.
|
||||
*/
|
||||
function pathWithGlobalRemoteProfile(path, profile, opts: any = {}) {
|
||||
function resolveProfileBackendRoute(profile, opts: ProfileRouteOptions = {}): ProfileBackendRoute {
|
||||
const scopedProfile = connectionScopeKey(profile)
|
||||
const primaryProfile = connectionScopeKey(opts.primaryProfile) || 'default'
|
||||
|
||||
if (!scopedProfile || scopedProfile === primaryProfile) {
|
||||
return { backend: 'primary', descriptorProfile: null, scopePath: false }
|
||||
}
|
||||
|
||||
if (opts.profileRemoteOverride) {
|
||||
return { backend: 'pool', descriptorProfile: null, scopePath: false }
|
||||
}
|
||||
|
||||
if (opts.globalRemote) {
|
||||
return { backend: 'primary', descriptorProfile: scopedProfile, scopePath: true }
|
||||
}
|
||||
|
||||
return { backend: 'pool', descriptorProfile: null, scopePath: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* Add renderer-side `request.profile` to a REST path when the route says the
|
||||
* serving backend is not already scoped to that profile.
|
||||
*/
|
||||
function pathWithGlobalRemoteProfile(path, profile, opts: ProfileRouteOptions = {}) {
|
||||
const scopedProfile = connectionScopeKey(profile)
|
||||
|
||||
if (!scopedProfile || !opts.globalRemote || opts.profileRemoteOverride) {
|
||||
if (!resolveProfileBackendRoute(profile, opts).scopePath) {
|
||||
return path
|
||||
}
|
||||
|
||||
|
|
@ -506,6 +558,7 @@ export {
|
|||
profileRemoteOverride,
|
||||
profileSshOverride,
|
||||
resolveAuthMode,
|
||||
resolveProfileBackendRoute,
|
||||
resolveTestWsUrl,
|
||||
RT_COOKIE_VARIANTS,
|
||||
savedProfileSsh,
|
||||
|
|
|
|||
70
apps/desktop/electron/crash-forensics.test.ts
Normal file
70
apps/desktop/electron/crash-forensics.test.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { describeCrashReason, installCrashForensics } from './crash-forensics'
|
||||
|
||||
const harness = () => {
|
||||
const listeners = new Map<string, (value: unknown) => void>()
|
||||
const flush = vi.fn()
|
||||
const log = vi.fn()
|
||||
|
||||
installCrashForensics({
|
||||
flush,
|
||||
log,
|
||||
target: { on: (event, listener) => listeners.set(event, listener) }
|
||||
})
|
||||
|
||||
return { flush, listeners, log }
|
||||
}
|
||||
|
||||
describe('describeCrashReason', () => {
|
||||
it('prefers a stack, then a message, for thrown errors', () => {
|
||||
const withStack = new Error('boom')
|
||||
withStack.stack = 'Error: boom\n at somewhere'
|
||||
|
||||
expect(describeCrashReason(withStack)).toBe('Error: boom\n at somewhere')
|
||||
|
||||
const withoutStack = new Error('boom')
|
||||
withoutStack.stack = ''
|
||||
|
||||
expect(describeCrashReason(withoutStack)).toBe('boom')
|
||||
})
|
||||
|
||||
it('renders non-error rejections without throwing', () => {
|
||||
expect(describeCrashReason('plain string')).toBe('plain string')
|
||||
expect(describeCrashReason({ code: 'ECONNRESET' })).toBe('{"code":"ECONNRESET"}')
|
||||
expect(describeCrashReason(undefined)).toBe('undefined')
|
||||
|
||||
const circular: Record<string, unknown> = {}
|
||||
circular.self = circular
|
||||
|
||||
expect(describeCrashReason(circular)).toBe('[object Object]')
|
||||
})
|
||||
})
|
||||
|
||||
describe('installCrashForensics', () => {
|
||||
it('records and synchronously flushes an uncaught exception', () => {
|
||||
const { flush, listeners, log } = harness()
|
||||
const error = new Error('renderer gone')
|
||||
error.stack = 'Error: renderer gone\n at main'
|
||||
|
||||
listeners.get('uncaughtException')?.(error)
|
||||
|
||||
expect(log).toHaveBeenCalledWith('[main] Uncaught exception: Error: renderer gone\n at main')
|
||||
expect(flush).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('records and synchronously flushes an unhandled rejection', () => {
|
||||
const { flush, listeners, log } = harness()
|
||||
|
||||
listeners.get('unhandledRejection')?.('gateway ticket mint failed')
|
||||
|
||||
expect(log).toHaveBeenCalledWith('[main] Unhandled rejection: gateway ticket mint failed')
|
||||
expect(flush).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('registers both handlers', () => {
|
||||
const { listeners } = harness()
|
||||
|
||||
expect([...listeners.keys()].sort()).toEqual(['uncaughtException', 'unhandledRejection'])
|
||||
})
|
||||
})
|
||||
51
apps/desktop/electron/crash-forensics.ts
Normal file
51
apps/desktop/electron/crash-forensics.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/**
|
||||
* Last-chance forensics for the Electron main process.
|
||||
*
|
||||
* Electron installs its own `uncaughtException` listener and only warns on
|
||||
* unhandled rejections, so the app usually survives — but the reason lands on
|
||||
* stderr alone, which is discarded entirely when the app is launched from
|
||||
* Finder or the Start menu. Without a record in desktop.log, a main-process
|
||||
* fault is invisible in a `hermes debug share` bundle and the user is left
|
||||
* describing symptoms instead of showing a stack.
|
||||
*/
|
||||
|
||||
export interface CrashForensicsTarget {
|
||||
on: (event: 'uncaughtException' | 'unhandledRejection', listener: (value: unknown) => void) => unknown
|
||||
}
|
||||
|
||||
export interface CrashForensicsOptions {
|
||||
flush: () => void
|
||||
log: (message: string) => void
|
||||
target?: CrashForensicsTarget
|
||||
}
|
||||
|
||||
/** Render a thrown value for the log, preferring a stack over a bare message. */
|
||||
export function describeCrashReason(reason: unknown): string {
|
||||
if (reason instanceof Error) {
|
||||
return reason.stack || reason.message || reason.name || 'Error'
|
||||
}
|
||||
|
||||
if (typeof reason === 'string') {
|
||||
return reason
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.stringify(reason) ?? String(reason)
|
||||
} catch {
|
||||
return String(reason)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record main-process faults to desktop.log and flush synchronously, since a
|
||||
* fault that does prove fatal leaves no chance for the batched async flush.
|
||||
*/
|
||||
export function installCrashForensics({ flush, log, target = process }: CrashForensicsOptions): void {
|
||||
const record = (label: string) => (reason: unknown) => {
|
||||
log(`[main] ${label}: ${describeCrashReason(reason)}`)
|
||||
flush()
|
||||
}
|
||||
|
||||
target.on('uncaughtException', record('Uncaught exception'))
|
||||
target.on('unhandledRejection', record('Unhandled rejection'))
|
||||
}
|
||||
222
apps/desktop/electron/find-in-page.test.ts
Normal file
222
apps/desktop/electron/find-in-page.test.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
/**
|
||||
* Unit tests for the pure find-in-page helpers. The IPC handlers in
|
||||
* main.ts are the only consumer — the helpers below must keep the wire
|
||||
* shape stable (match counter shape, defaults, no-throw-on-destroyed).
|
||||
*/
|
||||
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
|
||||
import { describe, test } from 'vitest'
|
||||
|
||||
import { formatFoundInPage, installFoundInPageForwarder, performFind, stopFind } from './find-in-page'
|
||||
|
||||
// Minimal webContents stub. The Electron.WebContents type is huge, so we
|
||||
// model just the slice the helpers touch (`isDestroyed`, `findInPage`,
|
||||
// `stopFindInPage`, `on`/`off`, `send`, `destroyed`, `emit`) and cast through
|
||||
// `asWC()` at call sites.
|
||||
interface FakeWebContents {
|
||||
calls: {
|
||||
find: Array<{ query: string; options: { forward: boolean; findNext: boolean } }>
|
||||
stop: Array<'clearSelection' | 'keepSelection' | 'activateSelection'>
|
||||
send: Array<{ channel: string; payload: unknown }>
|
||||
}
|
||||
isDestroyed: () => boolean
|
||||
destroy: () => void
|
||||
findInPage: (query: string, options: { forward: boolean; findNext: boolean }) => void
|
||||
stopFindInPage: (action: 'clearSelection' | 'keepSelection' | 'activateSelection') => void
|
||||
send: (channel: string, payload: unknown) => void
|
||||
on: typeof EventEmitter.prototype.on
|
||||
off: typeof EventEmitter.prototype.off
|
||||
emit: (event: string | symbol, ...args: unknown[]) => boolean
|
||||
}
|
||||
|
||||
function makeFakeWebContents(): FakeWebContents {
|
||||
const emitter = new EventEmitter()
|
||||
|
||||
const calls = {
|
||||
find: [] as Array<{ query: string; options: { forward: boolean; findNext: boolean } }>,
|
||||
stop: [] as Array<'clearSelection' | 'keepSelection' | 'activateSelection'>,
|
||||
send: [] as Array<{ channel: string; payload: unknown }>
|
||||
}
|
||||
|
||||
let destroyed = false
|
||||
|
||||
return {
|
||||
calls,
|
||||
isDestroyed: () => destroyed,
|
||||
destroy() {
|
||||
destroyed = true
|
||||
emitter.emit('destroyed')
|
||||
},
|
||||
findInPage(query: string, options: { forward: boolean; findNext: boolean }) {
|
||||
calls.find.push({ query, options })
|
||||
},
|
||||
stopFindInPage(action: 'clearSelection' | 'keepSelection' | 'activateSelection') {
|
||||
calls.stop.push(action)
|
||||
},
|
||||
send(channel: string, payload: unknown) {
|
||||
calls.send.push({ channel, payload })
|
||||
},
|
||||
on: emitter.on.bind(emitter),
|
||||
off: emitter.off.bind(emitter),
|
||||
emit: emitter.emit.bind(emitter)
|
||||
}
|
||||
}
|
||||
|
||||
function asWC(fake: FakeWebContents): Electron.WebContents {
|
||||
return fake as unknown as Electron.WebContents
|
||||
}
|
||||
|
||||
describe('formatFoundInPage', () => {
|
||||
test('maps activeMatchOrdinal + matches onto the wire payload', () => {
|
||||
assert.deepEqual(formatFoundInPage({ activeMatchOrdinal: 3, matches: 12 }), {
|
||||
activeMatchOrdinal: 3,
|
||||
count: 12
|
||||
})
|
||||
})
|
||||
|
||||
test('coerces missing fields to zero so the renderer never sees NaN', () => {
|
||||
assert.deepEqual(formatFoundInPage({}), { activeMatchOrdinal: 0, count: 0 })
|
||||
assert.deepEqual(formatFoundInPage({ activeMatchOrdinal: 0, matches: 0 }), {
|
||||
activeMatchOrdinal: 0,
|
||||
count: 0
|
||||
})
|
||||
})
|
||||
|
||||
test('null / undefined inputs still produce a well-formed payload', () => {
|
||||
assert.deepEqual(formatFoundInPage(null as unknown as { activeMatchOrdinal?: number; matches?: number }), {
|
||||
activeMatchOrdinal: 0,
|
||||
count: 0
|
||||
})
|
||||
assert.deepEqual(formatFoundInPage(undefined), { activeMatchOrdinal: 0, count: 0 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('performFind', () => {
|
||||
test('forwards the query and options to webContents.findInPage', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
performFind(asWC(wc), 'hello', { forward: true, findNext: false })
|
||||
assert.deepEqual(wc.calls.find, [{ query: 'hello', options: { forward: true, findNext: false } }])
|
||||
})
|
||||
|
||||
test('defaults forward to true when omitted', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
performFind(asWC(wc), 'x', { findNext: true })
|
||||
assert.deepEqual(wc.calls.find, [{ query: 'x', options: { forward: true, findNext: true } }])
|
||||
})
|
||||
|
||||
test('defaults findNext to false when omitted', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
performFind(asWC(wc), 'x', { forward: false })
|
||||
assert.deepEqual(wc.calls.find, [{ query: 'x', options: { forward: false, findNext: false } }])
|
||||
})
|
||||
|
||||
test('treats null / non-object options as "all defaults"', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
performFind(asWC(wc), 'x', null)
|
||||
assert.deepEqual(wc.calls.find, [{ query: 'x', options: { forward: true, findNext: false } }])
|
||||
})
|
||||
|
||||
test('coerces a non-string query to string (defensive against bad renderer payloads)', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
performFind(asWC(wc), 42 as unknown as string, null)
|
||||
assert.equal(wc.calls.find[0].query, '42')
|
||||
})
|
||||
|
||||
test('is a no-op when webContents is null', () => {
|
||||
assert.doesNotThrow(() => performFind(null, 'q', null))
|
||||
})
|
||||
|
||||
test('is a no-op when webContents is destroyed (does not throw across IPC)', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
wc.destroy()
|
||||
performFind(asWC(wc), 'q', null)
|
||||
assert.equal(wc.calls.find.length, 0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('stopFind', () => {
|
||||
test('calls stopFindInPage with the default action (clearSelection)', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
stopFind(asWC(wc))
|
||||
assert.deepEqual(wc.calls.stop, ['clearSelection'])
|
||||
})
|
||||
|
||||
test('honors an explicit action argument', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
stopFind(asWC(wc), 'keepSelection')
|
||||
assert.deepEqual(wc.calls.stop, ['keepSelection'])
|
||||
})
|
||||
|
||||
test('is a no-op when webContents is null or destroyed', () => {
|
||||
assert.doesNotThrow(() => stopFind(null))
|
||||
const wc = makeFakeWebContents()
|
||||
wc.destroy()
|
||||
stopFind(asWC(wc))
|
||||
assert.equal(wc.calls.stop.length, 0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('installFoundInPageForwarder', () => {
|
||||
test('forwards found-in-page to the sender as a formatted payload', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
installFoundInPageForwarder(asWC(wc))
|
||||
// Drive the fake's emit directly — this exercises the same code path
|
||||
// as Electron's actual `webContents.emit('found-in-page', …)`.
|
||||
wc.emit('found-in-page', {}, { activeMatchOrdinal: 2, matches: 5 })
|
||||
assert.deepEqual(wc.calls.send, [{ channel: 'hermes:found-in-page', payload: { activeMatchOrdinal: 2, count: 5 } }])
|
||||
})
|
||||
|
||||
test('handles missing fields without throwing', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
installFoundInPageForwarder(asWC(wc))
|
||||
wc.emit('found-in-page', {}, {})
|
||||
assert.deepEqual(wc.calls.send, [{ channel: 'hermes:found-in-page', payload: { activeMatchOrdinal: 0, count: 0 } }])
|
||||
})
|
||||
|
||||
test('skips send when webContents is destroyed at fire time', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
installFoundInPageForwarder(asWC(wc))
|
||||
wc.destroy()
|
||||
wc.emit('found-in-page', {}, { activeMatchOrdinal: 1, matches: 1 })
|
||||
assert.equal(wc.calls.send.length, 0, 'destroyed webContents must not be sent to')
|
||||
})
|
||||
|
||||
test('returned uninstall removes the listener', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
const uninstall = installFoundInPageForwarder(asWC(wc))
|
||||
uninstall()
|
||||
wc.emit('found-in-page', {}, { activeMatchOrdinal: 9, matches: 9 })
|
||||
assert.equal(wc.calls.send.length, 0, 'uninstalled listener must not fire')
|
||||
})
|
||||
|
||||
test('returned uninstall on a null webContents is a safe no-op', () => {
|
||||
const uninstall = installFoundInPageForwarder(null)
|
||||
assert.doesNotThrow(() => uninstall())
|
||||
})
|
||||
|
||||
test('returned uninstall on a destroyed webContents is a safe no-op', () => {
|
||||
const wc = makeFakeWebContents()
|
||||
wc.destroy()
|
||||
const uninstall = installFoundInPageForwarder(asWC(wc))
|
||||
assert.doesNotThrow(() => uninstall())
|
||||
})
|
||||
|
||||
// Regression: the original PR scoped the forwarder to the global mainWindow,
|
||||
// so Cmd+F pressed in a secondary session window routed results back to the
|
||||
// primary. Pin that the helper does NOT close over any window other than the
|
||||
// webContents it was given — two forwarders installed on two distinct fakes
|
||||
// must each send only to their own sender.
|
||||
test('two forwarders installed on distinct webContents do not cross-fire', () => {
|
||||
const wcA = makeFakeWebContents()
|
||||
const wcB = makeFakeWebContents()
|
||||
installFoundInPageForwarder(asWC(wcA))
|
||||
installFoundInPageForwarder(asWC(wcB))
|
||||
wcA.emit('found-in-page', {}, { activeMatchOrdinal: 1, matches: 1 })
|
||||
assert.deepEqual(wcA.calls.send, [
|
||||
{ channel: 'hermes:found-in-page', payload: { activeMatchOrdinal: 1, count: 1 } }
|
||||
])
|
||||
assert.equal(wcB.calls.send.length, 0, 'wcB must not receive wcA results')
|
||||
})
|
||||
})
|
||||
120
apps/desktop/electron/find-in-page.ts
Normal file
120
apps/desktop/electron/find-in-page.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
/**
|
||||
* Pure helpers for the desktop find-in-page bridge (Ctrl/Cmd+F).
|
||||
*
|
||||
* The renderer drives an Electron `webContents.findInPage` over IPC so it can
|
||||
* reuse the native "find-in-page" experience (incremental search, match
|
||||
* highlight, Enter to step, Shift+Enter to step backwards, Escape to clear)
|
||||
* across chat transcripts and editor panels. Everything in this module is
|
||||
* pure with respect to its inputs so the routing + payload shaping can be
|
||||
* unit-tested without booting a BrowserWindow.
|
||||
*
|
||||
* Multi-window correctness: the IPC handlers in main.ts resolve the
|
||||
* requesting window via `BrowserWindow.fromWebContents(event.sender)` so a
|
||||
* Cmd+F pressed in a secondary session window searches THAT window, not the
|
||||
* primary. The `found-in-page` results are forwarded back to the same sender
|
||||
* — see {@link installFoundInPageForwarder}.
|
||||
*/
|
||||
|
||||
/** Match options accepted by the renderer's `findInPage` bridge call. */
|
||||
export interface FindInPageOptions {
|
||||
/** Step direction. Defaults to `true` (forward). */
|
||||
forward?: boolean
|
||||
/**
|
||||
* `true` to advance to the next/previous match using the previous query;
|
||||
* `false` to (re)search the current `query` from scratch. The renderer
|
||||
* passes `false` on a fresh query and `true` on Enter / Shift+Enter.
|
||||
*/
|
||||
findNext?: boolean
|
||||
}
|
||||
|
||||
/** Payload shape sent back to the renderer on every `found-in-page` event. */
|
||||
export interface FoundInPagePayload {
|
||||
/** 1-indexed ordinal of the active match, or 0 when none. */
|
||||
activeMatchOrdinal: number
|
||||
/** Total matches in the document for the current query. */
|
||||
count: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Defensive projection of Electron's `found-in-page` event result. Electron
|
||||
* exposes more fields (finalUpdate, selectionArea, etc.) that we don't need;
|
||||
* keeping the projection explicit makes the wire shape auditable and keeps
|
||||
* tests independent of the runtime type.
|
||||
*/
|
||||
export function formatFoundInPage(result: { activeMatchOrdinal?: number; matches?: number }): FoundInPagePayload {
|
||||
return {
|
||||
activeMatchOrdinal: Number(result?.activeMatchOrdinal ?? 0),
|
||||
count: Number(result?.matches ?? 0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Issue a `findInPage` against the given `webContents`. No-op when the
|
||||
* webContents is missing or destroyed — surfaces as a silent miss rather
|
||||
* than throwing across the IPC boundary, matching Electron's own semantics
|
||||
* for a destroyed renderer.
|
||||
*/
|
||||
export function performFind(
|
||||
webContents: Electron.WebContents | null | undefined,
|
||||
query: string,
|
||||
options: FindInPageOptions | null | undefined
|
||||
): void {
|
||||
if (!webContents || webContents.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
|
||||
const opts = options && typeof options === 'object' ? options : {}
|
||||
|
||||
webContents.findInPage(String(query ?? ''), {
|
||||
forward: opts.forward !== false,
|
||||
findNext: Boolean(opts.findNext)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the current find and clear highlights. The default `action` matches
|
||||
* what the renderer sends on Escape / close.
|
||||
*/
|
||||
export function stopFind(
|
||||
webContents: Electron.WebContents | null | undefined,
|
||||
action: 'clearSelection' | 'keepSelection' | 'activateSelection' = 'clearSelection'
|
||||
): void {
|
||||
if (!webContents || webContents.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
|
||||
webContents.stopFindInPage(action)
|
||||
}
|
||||
|
||||
/**
|
||||
* Install a `found-in-page` listener on the given sender `webContents` and
|
||||
* forward each result back to the SAME renderer (via `webContents.send`).
|
||||
*
|
||||
* Returns an uninstall function. Call it from `webContents.on('destroyed', …)`
|
||||
* to avoid leaking the listener when the window goes away — Electron does
|
||||
* not auto-detach webContents listeners on close.
|
||||
*
|
||||
* The forwarder is intentionally bound to a single sender rather than the
|
||||
* primary window: a Cmd+F pressed in a secondary session window must
|
||||
* highlight matches in THAT window, and the match counter must reflect
|
||||
* THAT window's DOM, not the primary's.
|
||||
*/
|
||||
export function installFoundInPageForwarder(webContents: Electron.WebContents | null | undefined): () => void {
|
||||
if (!webContents || webContents.isDestroyed()) {
|
||||
return () => {}
|
||||
}
|
||||
|
||||
const handler = (_event: Electron.Event, result: Parameters<typeof formatFoundInPage>[0]) => {
|
||||
if (webContents.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
|
||||
webContents.send('hermes:found-in-page', formatFoundInPage(result))
|
||||
}
|
||||
|
||||
webContents.on('found-in-page', handler)
|
||||
|
||||
return () => {
|
||||
webContents.off('found-in-page', handler)
|
||||
}
|
||||
}
|
||||
133
apps/desktop/electron/first-run-setup-gate.test.ts
Normal file
133
apps/desktop/electron/first-run-setup-gate.test.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
function delay(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function settledState(promise: Promise<unknown>) {
|
||||
return Promise.race([promise.then(() => 'resolved'), delay(10).then(() => 'pending')])
|
||||
}
|
||||
|
||||
test('first-run setup gate skips non-bootstrap backends', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
await gate.wait({ kind: 'remote' })
|
||||
await gate.wait(null)
|
||||
|
||||
assert.deepEqual(prompts, [])
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
})
|
||||
|
||||
test('first-run setup gate prompts once for concurrent waits', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const first = gate.wait(bootstrapBackend)
|
||||
const second = gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.hasWaiter(), true)
|
||||
assert.equal(prompts.length, 1)
|
||||
assert.equal(await settledState(first), 'pending')
|
||||
|
||||
gate.continueLocal()
|
||||
|
||||
assert.deepEqual(await Promise.all([first, second]), ['continue-local', 'continue-local'])
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
|
||||
test('continueLocal keeps the setup choice visible until bootstrap owns the overlay', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.continueLocal()
|
||||
|
||||
assert.equal(await pending, 'continue-local')
|
||||
assert.equal(hidden, 0)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
|
||||
test('retry reset preserves the local install confirmation', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
gate.resetForRetry()
|
||||
await gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
assert.equal(prompts.length, 1)
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
})
|
||||
|
||||
test('retry reset explicitly settles an active waiter without allowing local bootstrap', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.resetForRetry()
|
||||
|
||||
assert.equal(await pending, 'reset')
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
})
|
||||
|
||||
test('repair reset clears the local install confirmation and shows the gate again', async () => {
|
||||
const prompts = []
|
||||
const gate = createFirstRunSetupGate({ promptChoice: backend => prompts.push(backend), stuckAfterMs: 0 })
|
||||
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
gate.resetForRepair()
|
||||
const next = gate.wait(bootstrapBackend)
|
||||
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
assert.equal(prompts.length, 2)
|
||||
assert.equal(gate.hasWaiter(), true)
|
||||
|
||||
gate.continueLocal()
|
||||
await next
|
||||
})
|
||||
|
||||
test('remote apply settles the gated boot for remote re-resolution and hides the choice', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
const resumedWaiter = gate.abandonForRemoteApply()
|
||||
|
||||
assert.equal(resumedWaiter, true)
|
||||
assert.equal(hidden, 1)
|
||||
assert.equal(gate.hasWaiter(), false)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), false)
|
||||
assert.equal(await pending, 'remote-applied')
|
||||
})
|
||||
|
||||
test('remote apply without a waiter has no first-run side effects', async () => {
|
||||
let hidden = 0
|
||||
const gate = createFirstRunSetupGate({ hideChoice: () => hidden++, stuckAfterMs: 0 })
|
||||
const pending = gate.wait(bootstrapBackend)
|
||||
|
||||
gate.continueLocal()
|
||||
await pending
|
||||
|
||||
assert.equal(gate.abandonForRemoteApply(), false)
|
||||
assert.equal(hidden, 0)
|
||||
assert.equal(gate.isLocalBootstrapConfirmed(), true)
|
||||
})
|
||||
146
apps/desktop/electron/first-run-setup-gate.ts
Normal file
146
apps/desktop/electron/first-run-setup-gate.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
interface FirstRunSetupBackend {
|
||||
activeRoot?: string
|
||||
kind?: string
|
||||
platform?: string
|
||||
}
|
||||
|
||||
interface FirstRunSetupGateOptions {
|
||||
hideChoice?: () => void
|
||||
log?: (message: string) => void
|
||||
onStuck?: (backend: FirstRunSetupBackend, stuckAfterMs: number) => void
|
||||
promptChoice?: (backend: FirstRunSetupBackend) => void
|
||||
stuckAfterMs?: number
|
||||
}
|
||||
|
||||
export type FirstRunSetupDecision = 'continue-local' | 'remote-applied' | 'reset'
|
||||
|
||||
export function createFirstRunSetupGate({
|
||||
hideChoice,
|
||||
log,
|
||||
onStuck,
|
||||
promptChoice,
|
||||
stuckAfterMs = 120000
|
||||
}: FirstRunSetupGateOptions = {}) {
|
||||
let localBootstrapConfirmed = false
|
||||
|
||||
let waiter: {
|
||||
promise: Promise<FirstRunSetupDecision>
|
||||
resolve: (decision: FirstRunSetupDecision) => void
|
||||
} | null = null
|
||||
|
||||
let stuckTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const clearStuckTimer = () => {
|
||||
if (stuckTimer) {
|
||||
clearTimeout(stuckTimer)
|
||||
stuckTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
const armStuckTimer = (backend: FirstRunSetupBackend) => {
|
||||
clearStuckTimer()
|
||||
|
||||
if (!Number.isFinite(stuckAfterMs) || stuckAfterMs <= 0 || typeof log !== 'function') {
|
||||
return
|
||||
}
|
||||
|
||||
stuckTimer = setTimeout(() => {
|
||||
onStuck?.(backend, stuckAfterMs)
|
||||
log(
|
||||
`[bootstrap] still waiting for first-run setup choice after ${Math.round(stuckAfterMs / 1000)}s ` +
|
||||
`(platform=${backend?.platform || 'unknown'})`
|
||||
)
|
||||
}, stuckAfterMs)
|
||||
|
||||
if (typeof stuckTimer.unref === 'function') {
|
||||
stuckTimer.unref()
|
||||
}
|
||||
}
|
||||
|
||||
const shouldGate = (backend?: FirstRunSetupBackend | null) =>
|
||||
Boolean(backend && backend.kind === 'bootstrap-needed' && !localBootstrapConfirmed)
|
||||
|
||||
const wait = async (backend?: FirstRunSetupBackend | null) => {
|
||||
if (!shouldGate(backend)) {
|
||||
return 'continue-local' as const
|
||||
}
|
||||
|
||||
if (waiter) {
|
||||
return waiter.promise
|
||||
}
|
||||
|
||||
promptChoice?.(backend)
|
||||
armStuckTimer(backend)
|
||||
|
||||
let resolveWaiter: (decision: FirstRunSetupDecision) => void = () => {}
|
||||
|
||||
const promise = new Promise<FirstRunSetupDecision>(resolve => {
|
||||
resolveWaiter = resolve
|
||||
})
|
||||
|
||||
waiter = { promise, resolve: resolveWaiter }
|
||||
|
||||
return promise
|
||||
}
|
||||
|
||||
const settleWaiter = (decision: FirstRunSetupDecision) => {
|
||||
clearStuckTimer()
|
||||
|
||||
if (!waiter) {
|
||||
return false
|
||||
}
|
||||
|
||||
const activeWaiter = waiter
|
||||
waiter = null
|
||||
activeWaiter.resolve(decision)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const continueLocal = () => {
|
||||
localBootstrapConfirmed = true
|
||||
settleWaiter('continue-local')
|
||||
}
|
||||
|
||||
const resetForRetry = () => {
|
||||
// Reset paths are followed by a renderer reload / fresh startHermes() call.
|
||||
// Settle the old boot explicitly so it cannot fall through into local
|
||||
// bootstrap and cannot leak a forever-pending connection promise.
|
||||
settleWaiter('reset')
|
||||
}
|
||||
|
||||
const resetForRepair = () => {
|
||||
resetForRetry()
|
||||
localBootstrapConfirmed = false
|
||||
}
|
||||
|
||||
const abandonForRemoteApply = () => {
|
||||
// Resume the gated startHermes() with an explicit remote decision. The
|
||||
// caller re-resolves the newly-persisted remote config instead of falling
|
||||
// through into local bootstrap or leaking the original connection promise.
|
||||
const resumedWaiter = settleWaiter('remote-applied')
|
||||
|
||||
if (!resumedWaiter) {
|
||||
return false
|
||||
}
|
||||
|
||||
localBootstrapConfirmed = false
|
||||
hideChoice?.()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const isLocalBootstrapConfirmed = () => localBootstrapConfirmed
|
||||
const hasWaiter = () => Boolean(waiter)
|
||||
|
||||
return {
|
||||
abandonForRemoteApply,
|
||||
continueLocal,
|
||||
hasWaiter,
|
||||
isLocalBootstrapConfirmed,
|
||||
resetForRepair,
|
||||
resetForRetry,
|
||||
shouldGate,
|
||||
wait
|
||||
}
|
||||
}
|
||||
119
apps/desktop/electron/first-run-setup-main-process.test.ts
Normal file
119
apps/desktop/electron/first-run-setup-main-process.test.ts
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test, vi } from 'vitest'
|
||||
|
||||
import { applyConnectionChange } from './connection-apply'
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
import { runPrimaryBackendStartup } from './primary-backend-startup'
|
||||
import { rehomePrimaryConnection } from './primary-connection-rehome'
|
||||
|
||||
test('a first-run bootstrap-needed remote apply connects without ensuring or bootstrapping locally', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
const candidateRemote = {
|
||||
authMode: 'token',
|
||||
baseUrl: 'https://gateway.example.com/hermes',
|
||||
source: 'settings',
|
||||
token: 'secret',
|
||||
wsUrl: 'wss://gateway.example.com/hermes/api/ws?token=secret'
|
||||
}
|
||||
|
||||
let savedRemote: typeof candidateRemote | null = null
|
||||
|
||||
const resolveRemote = vi.fn(async () => savedRemote)
|
||||
const connectRemote = vi.fn(async remote => ({ ...remote, mode: 'remote' as const }))
|
||||
const runBootstrap = vi.fn()
|
||||
|
||||
const ensureLocalRuntime = vi.fn(async backend => {
|
||||
await runBootstrap()
|
||||
|
||||
return { ...backend, command: 'hermes' }
|
||||
})
|
||||
|
||||
const teardownPrimaryBackend = vi.fn(async () => {})
|
||||
const cancelSshBootstrap = vi.fn(async () => {})
|
||||
const teardownSsh = vi.fn(async () => {})
|
||||
const clearLocalBootstrapFailure = vi.fn()
|
||||
const notifyConnectionApplied = vi.fn()
|
||||
const waitForLocalStart = vi.fn(async () => {})
|
||||
const prepareLocalBackend = vi.fn(async () => bootstrapBackend)
|
||||
|
||||
const pendingConnection = runPrimaryBackendStartup({
|
||||
connectRemote,
|
||||
ensureLocalRuntime,
|
||||
prepareLocalBackend,
|
||||
resolveRemote,
|
||||
waitForDecision: gate.wait,
|
||||
waitForLocalStart
|
||||
})
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
|
||||
// Mirrors the IPC handler's production ordering: persist the tested config,
|
||||
// then re-home. The pending start must re-resolve this saved value.
|
||||
savedRemote = candidateRemote
|
||||
|
||||
await applyConnectionChange({
|
||||
cancelAndWait: cancelSshBootstrap,
|
||||
isPrimary: true,
|
||||
rehomePrimary: () =>
|
||||
rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode: 'remote',
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote: gate.abandonForRemoteApply,
|
||||
teardownPrimaryBackend
|
||||
}),
|
||||
scope: '',
|
||||
sendApplied: notifyConnectionApplied,
|
||||
stopPool: vi.fn(),
|
||||
teardownPrimary: teardownPrimaryBackend,
|
||||
teardownSsh
|
||||
})
|
||||
|
||||
assert.deepEqual(await pendingConnection, {
|
||||
kind: 'remote',
|
||||
connection: { ...candidateRemote, mode: 'remote' }
|
||||
})
|
||||
assert.deepEqual(resolveRemote.mock.calls, [[], []])
|
||||
assert.deepEqual(connectRemote.mock.calls, [[candidateRemote]])
|
||||
assert.deepEqual(waitForLocalStart.mock.calls, [[]])
|
||||
assert.deepEqual(prepareLocalBackend.mock.calls, [[]])
|
||||
assert.equal(ensureLocalRuntime.mock.calls.length, 0)
|
||||
assert.equal(runBootstrap.mock.calls.length, 0)
|
||||
assert.deepEqual(cancelSshBootstrap.mock.calls, [['']])
|
||||
assert.deepEqual(teardownSsh.mock.calls, [['']])
|
||||
assert.equal(teardownPrimaryBackend.mock.calls.length, 0)
|
||||
assert.equal(clearLocalBootstrapFailure.mock.calls.length, 1)
|
||||
assert.equal(notifyConnectionApplied.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('a primary apply without an active first-run gate tears down before reconnect notification', async () => {
|
||||
const order: string[] = []
|
||||
const clearLocalBootstrapFailure = vi.fn(() => order.push('clear-failure'))
|
||||
|
||||
const teardownPrimaryBackend = vi.fn(async () => {
|
||||
order.push('teardown')
|
||||
})
|
||||
|
||||
const notifyConnectionApplied = vi.fn(() => order.push('notify'))
|
||||
|
||||
assert.deepEqual(
|
||||
await rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode: 'remote',
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote: () => false,
|
||||
teardownPrimaryBackend
|
||||
}),
|
||||
{ resumedFirstRunRemote: false }
|
||||
)
|
||||
assert.deepEqual(teardownPrimaryBackend.mock.calls, [[{ soft: true }]])
|
||||
assert.deepEqual(order, ['clear-failure', 'teardown', 'notify'])
|
||||
})
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Regression tests for electron/native-auth-decisions.ts — the three pure
|
||||
* decision seams behind the RFC 8252 native-app auth flow, each of which was a
|
||||
* real runtime bug that the mocked flow tests could not catch.
|
||||
* Regression tests for electron/native-auth-decisions.ts — the pure decision
|
||||
* seams behind the RFC 8252 native-app auth flow, each of which was a real
|
||||
* runtime bug that the mocked flow tests could not catch.
|
||||
*
|
||||
* Run via the vitest `electron` project (electron/**\/*.test.ts).
|
||||
*/
|
||||
|
|
@ -10,7 +10,13 @@ import assert from 'node:assert/strict'
|
|||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { oauthSessionIsLive, resolveJsonBody, resolveOauthRestAuth } from './native-auth-decisions'
|
||||
import {
|
||||
oauthGuardMayHardFail,
|
||||
oauthSessionIsLive,
|
||||
resolveJsonBody,
|
||||
resolveOauthRestAuth,
|
||||
resolveReadinessProbeAuth
|
||||
} from './native-auth-decisions'
|
||||
|
||||
// --- 1. body encoding (guards the double-JSON.stringify 422) ---
|
||||
|
||||
|
|
@ -66,3 +72,61 @@ test('resolveOauthRestAuth falls back to cookie when there is no native token',
|
|||
// Empty string is not a usable bearer — must fall back, not send "Bearer ".
|
||||
assert.deepEqual(resolveOauthRestAuth(''), { kind: 'cookie' })
|
||||
})
|
||||
|
||||
// --- 4. readiness-probe auth (guards the credential-free 401 boot loop) ---
|
||||
|
||||
test('resolveReadinessProbeAuth reuses the oauth bearer-vs-cookie choice', () => {
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', 'native-at'), { kind: 'bearer', token: 'native-at' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', null), { kind: 'cookie' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('oauth', ''), { kind: 'cookie' })
|
||||
})
|
||||
|
||||
test('resolveReadinessProbeAuth sends the session token for a token gateway', () => {
|
||||
assert.deepEqual(resolveReadinessProbeAuth('token', null, 'session-token'), {
|
||||
kind: 'token',
|
||||
token: 'session-token'
|
||||
})
|
||||
assert.deepEqual(resolveReadinessProbeAuth('token', null, null), { kind: 'token', token: null })
|
||||
})
|
||||
|
||||
test('resolveReadinessProbeAuth stays public for local and unknown modes', () => {
|
||||
// A loopback backend has no gate; sending credentials it never issued is
|
||||
// meaningless, and an unknown mode must not invent a credential.
|
||||
assert.deepEqual(resolveReadinessProbeAuth('local', 'native-at', 'session-token'), { kind: 'public' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth(undefined, 'native-at', 'session-token'), { kind: 'public' })
|
||||
assert.deepEqual(resolveReadinessProbeAuth('something-new', null, null), { kind: 'public' })
|
||||
})
|
||||
|
||||
// --- 5. oauth guard vs password gateways (guards the false "not signed in") ---
|
||||
|
||||
test('oauthGuardMayHardFail is false only when EVERY provider is password-based', () => {
|
||||
assert.equal(oauthGuardMayHardFail([{ name: 'basic', supportsPassword: true }]), false)
|
||||
assert.equal(
|
||||
oauthGuardMayHardFail([
|
||||
{ name: 'basic', supportsPassword: true },
|
||||
{ name: 'ldap', supportsPassword: true }
|
||||
]),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('oauthGuardMayHardFail keeps the strict guard for oauth and mixed deployments', () => {
|
||||
assert.equal(oauthGuardMayHardFail([{ name: 'nous', supportsPassword: false }]), true)
|
||||
assert.equal(
|
||||
oauthGuardMayHardFail([
|
||||
{ name: 'nous', supportsPassword: false },
|
||||
{ name: 'basic', supportsPassword: true }
|
||||
]),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('oauthGuardMayHardFail keeps the strict guard when the list is unusable', () => {
|
||||
// Backends predating /api/auth/providers, or an unreachable probe, must not
|
||||
// silently weaken the guard.
|
||||
assert.equal(oauthGuardMayHardFail([]), true)
|
||||
assert.equal(oauthGuardMayHardFail(null), true)
|
||||
assert.equal(oauthGuardMayHardFail(undefined), true)
|
||||
assert.equal(oauthGuardMayHardFail('nonsense' as any), true)
|
||||
assert.equal(oauthGuardMayHardFail([{ supportsPassword: true }]), true)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -21,7 +21,17 @@
|
|||
* native bearer when present, else the cookie partition. Cookie-only
|
||||
* routing returns 401 no_cookie for a cookieless native session.
|
||||
*
|
||||
* All three are trivial once named; the value is the test that pins the
|
||||
* 4. resolveReadinessProbeAuth — the boot readiness probe must authenticate
|
||||
* the same way the rest of the connection does. A credential-free probe
|
||||
* against a gated gateway 401s forever; worse, it cannot tell a missing
|
||||
* route from a rejected session (see backend-health.ts).
|
||||
*
|
||||
* 5. oauthGuardMayHardFail — `auth_required: true` means "this gateway is
|
||||
* gated", NOT "this gateway speaks OAuth". A password-provider gateway
|
||||
* can satisfy neither the native-bearer nor the OAuth-partition-cookie
|
||||
* check by design, so the pre-flight guard must not hard-fail it.
|
||||
*
|
||||
* All five are trivial once named; the value is the test that pins the
|
||||
* contract so the god-file call sites can't drift back to the buggy shape.
|
||||
*/
|
||||
|
||||
|
|
@ -60,3 +70,75 @@ export function resolveOauthRestAuth(nativeAccessToken: string | null | undefine
|
|||
|
||||
return { kind: 'cookie' }
|
||||
}
|
||||
|
||||
export type ReadinessProbeAuth = OauthRestAuth | { kind: 'token'; token: string | null } | { kind: 'public' }
|
||||
|
||||
/**
|
||||
* Decide how the boot readiness probe authenticates.
|
||||
*
|
||||
* The probe must present the SAME credentials the rest of the connection
|
||||
* will use. A credential-free probe against a gated gateway 401s until the
|
||||
* boot deadline even though the session is perfectly valid — and because the
|
||||
* dashboard auth gate runs ahead of the SPA catch-all, an unknown `/api/*`
|
||||
* path answers 401 rather than 404, so the probe also cannot detect a backend
|
||||
* that predates `/api/health`. Sending credentials is what lets a missing
|
||||
* route surface as a real 404 (see `isMissingHealthEndpointError`).
|
||||
*
|
||||
* `oauth` reuses `resolveOauthRestAuth` so the probe and every other oauth
|
||||
* REST call make the identical bearer-vs-cookie choice. `token` presents the
|
||||
* connection's session token. `local` (and anything unrecognized) stays
|
||||
* public: a loopback backend has no gate, and sending credentials it never
|
||||
* issued would be meaningless.
|
||||
*/
|
||||
export function resolveReadinessProbeAuth(
|
||||
authMode: string | null | undefined,
|
||||
nativeAccessToken?: string | null,
|
||||
connectionToken?: string | null
|
||||
): ReadinessProbeAuth {
|
||||
if (authMode === 'oauth') {
|
||||
return resolveOauthRestAuth(nativeAccessToken)
|
||||
}
|
||||
|
||||
if (authMode === 'token') {
|
||||
return { kind: 'token', token: connectionToken ?? null }
|
||||
}
|
||||
|
||||
return { kind: 'public' }
|
||||
}
|
||||
|
||||
export interface AdvertisedAuthProvider {
|
||||
name?: string
|
||||
supportsPassword?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the oauth pre-flight guard may hard-fail a connection for "not
|
||||
* signed in".
|
||||
*
|
||||
* `authModeFromStatus` maps the gateway's `auth_required: true` onto
|
||||
* `'oauth'`, but that flag only means the dashboard is GATED — it says
|
||||
* nothing about how you authenticate. A gateway whose providers are all
|
||||
* username/password cannot satisfy the guard's checks by construction:
|
||||
* `start_login` raises NotImplementedError, `/auth/native/authorize` rejects
|
||||
* password providers, and its cookies are set by a plain password-login POST
|
||||
* rather than the `/auth/callback` redirect the OAuth partition is primed
|
||||
* for. Hard-failing there rejects a live session one line before the
|
||||
* ws-ticket mint that would have succeeded against that very partition.
|
||||
*
|
||||
* Returns false only when EVERY advertised provider is password-based. An
|
||||
* unknown or empty list keeps the strict guard, so backends that predate
|
||||
* `/api/auth/providers` are unaffected.
|
||||
*/
|
||||
export function oauthGuardMayHardFail(providers: AdvertisedAuthProvider[] | null | undefined): boolean {
|
||||
if (!Array.isArray(providers) || providers.length === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
const named = providers.filter(provider => provider && typeof provider === 'object' && provider.name)
|
||||
|
||||
if (named.length === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
return !named.every(provider => provider.supportsPassword)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,41 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
|
|||
return () => ipcRenderer.removeListener('hermes:pet-overlay:control', listener)
|
||||
}
|
||||
},
|
||||
// Quick Entry: the global-hotkey mini composer window. Main owns the OS
|
||||
// shortcut + the persisted preference; the quick window only captures text
|
||||
// and hands it back, and the primary renderer submits it through the normal
|
||||
// prompt path.
|
||||
quickEntry: {
|
||||
getSettings: () => ipcRenderer.invoke('hermes:quick-entry:settings:get'),
|
||||
setSettings: patch => ipcRenderer.invoke('hermes:quick-entry:settings:set', patch),
|
||||
submit: payload => ipcRenderer.send('hermes:quick-entry:submit', payload),
|
||||
dismiss: () => ipcRenderer.send('hermes:quick-entry:dismiss'),
|
||||
// Primary renderer → main → quick window: gateway connection state + the
|
||||
// recent-session options the target picker offers. Main caches the latest
|
||||
// payload so a freshly spawned quick window starts from truth.
|
||||
pushState: payload => ipcRenderer.send('hermes:quick-entry:state', payload),
|
||||
// Quick window subscribes to those pushes.
|
||||
onState: callback => {
|
||||
const listener = (_event, payload) => callback(payload)
|
||||
ipcRenderer.on('hermes:quick-entry:state', listener)
|
||||
|
||||
return () => ipcRenderer.removeListener('hermes:quick-entry:state', listener)
|
||||
},
|
||||
// Main → primary renderer: a submit captured by the quick window.
|
||||
onSubmit: callback => {
|
||||
const listener = (_event, payload) => callback(payload)
|
||||
ipcRenderer.on('hermes:quick-entry:submit', listener)
|
||||
|
||||
return () => ipcRenderer.removeListener('hermes:quick-entry:submit', listener)
|
||||
},
|
||||
// Main → quick window: you were just summoned (reset draft + refocus).
|
||||
onShown: callback => {
|
||||
const listener = () => callback()
|
||||
ipcRenderer.on('hermes:quick-entry:shown', listener)
|
||||
|
||||
return () => ipcRenderer.removeListener('hermes:quick-entry:shown', listener)
|
||||
}
|
||||
},
|
||||
getBootProgress: () => ipcRenderer.invoke('hermes:boot-progress:get'),
|
||||
getConnectionConfig: profile => ipcRenderer.invoke('hermes:connection-config:get', profile),
|
||||
saveConnectionConfig: payload => ipcRenderer.invoke('hermes:connection-config:save', payload),
|
||||
|
|
@ -237,6 +272,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
|
|||
// current snapshot via getBootstrapState() to recover after a devtools
|
||||
// reload mid-bootstrap.
|
||||
getBootstrapState: () => ipcRenderer.invoke('hermes:bootstrap:get'),
|
||||
continueBootstrapLocal: () => ipcRenderer.invoke('hermes:bootstrap:continue-local'),
|
||||
resetBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:reset'),
|
||||
repairBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:repair'),
|
||||
cancelBootstrap: () => ipcRenderer.invoke('hermes:bootstrap:cancel'),
|
||||
|
|
@ -267,5 +303,19 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
|
|||
themes: {
|
||||
fetchMarketplace: id => ipcRenderer.invoke('hermes:vscode-theme:fetch', id),
|
||||
searchMarketplace: query => ipcRenderer.invoke('hermes:vscode-theme:search', query)
|
||||
},
|
||||
// Find-in-page (Ctrl/Cmd+F): delegates to Electron's
|
||||
// webContents.findInPage on the IPC sender's window so a Cmd+F pressed
|
||||
// in a secondary session window searches THAT window, not the primary.
|
||||
// `onFoundInPage` returns the unsubscribe fn; the renderer wires it via
|
||||
// `initFindInPageListener` in store/find-in-page.ts and tears it down
|
||||
// when the FindBar unmounts.
|
||||
findInPage: (query, options) => ipcRenderer.invoke('hermes:find-in-page', query, options),
|
||||
stopFindInPage: () => ipcRenderer.invoke('hermes:stop-find-in-page'),
|
||||
onFoundInPage: callback => {
|
||||
const listener = (_event, result) => callback(result)
|
||||
ipcRenderer.on('hermes:found-in-page', listener)
|
||||
|
||||
return () => ipcRenderer.removeListener('hermes:found-in-page', listener)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
110
apps/desktop/electron/primary-backend-startup.test.ts
Normal file
110
apps/desktop/electron/primary-backend-startup.test.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test, vi } from 'vitest'
|
||||
|
||||
import { createFirstRunSetupGate } from './first-run-setup-gate'
|
||||
import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup'
|
||||
|
||||
const bootstrapBackend = {
|
||||
activeRoot: '/tmp/hermes-home/hermes-agent',
|
||||
kind: 'bootstrap-needed',
|
||||
platform: 'linux'
|
||||
}
|
||||
|
||||
function startupOptions(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
connectRemote: vi.fn(async remote => ({ baseUrl: remote.baseUrl, mode: 'remote' as const })),
|
||||
ensureLocalRuntime: vi.fn(async backend => ({ ...backend, command: 'hermes' })),
|
||||
prepareLocalBackend: vi.fn(async () => bootstrapBackend),
|
||||
resolveRemote: vi.fn(async () => null),
|
||||
waitForDecision: vi.fn(async () => 'continue-local' as const),
|
||||
waitForLocalStart: vi.fn(async () => {}),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('remote apply re-resolves the saved connection without ensuring a local runtime', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
|
||||
let configuredRemote: typeof savedRemote | null = null
|
||||
|
||||
const options = startupOptions({
|
||||
resolveRemote: vi.fn(async () => configuredRemote),
|
||||
waitForDecision: gate.wait
|
||||
})
|
||||
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
configuredRemote = savedRemote
|
||||
assert.equal(gate.abandonForRemoteApply(), true)
|
||||
|
||||
assert.deepEqual(await pending, {
|
||||
kind: 'remote',
|
||||
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
|
||||
})
|
||||
assert.deepEqual(options.resolveRemote.mock.calls, [[], []])
|
||||
assert.deepEqual(options.connectRemote.mock.calls, [[savedRemote]])
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('an already-saved remote bypasses every local startup step', async () => {
|
||||
const savedRemote = { baseUrl: 'https://gateway.example.com/hermes' }
|
||||
const options = startupOptions({ resolveRemote: vi.fn(async () => savedRemote) })
|
||||
|
||||
assert.deepEqual(await runPrimaryBackendStartup(options), {
|
||||
kind: 'remote',
|
||||
connection: { baseUrl: savedRemote.baseUrl, mode: 'remote' }
|
||||
})
|
||||
assert.equal(options.waitForLocalStart.mock.calls.length, 0)
|
||||
assert.equal(options.prepareLocalBackend.mock.calls.length, 0)
|
||||
assert.equal(options.waitForDecision.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('remote apply fails clearly when no saved remote can be resolved', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const options = startupOptions({ waitForDecision: gate.wait })
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.abandonForRemoteApply()
|
||||
|
||||
await assert.rejects(pending, /without a saved remote backend/)
|
||||
assert.equal(options.connectRemote.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
|
||||
test('continue local waits for update exclusion and ensures the prepared runtime exactly once', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const runtimeBackend = { ...bootstrapBackend, command: 'hermes' }
|
||||
|
||||
const options = startupOptions({
|
||||
ensureLocalRuntime: vi.fn(async () => runtimeBackend),
|
||||
waitForDecision: gate.wait
|
||||
})
|
||||
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.continueLocal()
|
||||
|
||||
assert.deepEqual(await pending, { kind: 'local', backend: runtimeBackend })
|
||||
assert.deepEqual(options.waitForLocalStart.mock.calls, [[]])
|
||||
assert.deepEqual(options.prepareLocalBackend.mock.calls, [[]])
|
||||
assert.deepEqual(options.ensureLocalRuntime.mock.calls, [[bootstrapBackend]])
|
||||
assert.deepEqual(options.resolveRemote.mock.calls, [[]])
|
||||
})
|
||||
|
||||
test('reset rejects with a typed error and never enters either backend', async () => {
|
||||
const gate = createFirstRunSetupGate({ stuckAfterMs: 0 })
|
||||
const options = startupOptions({ waitForDecision: gate.wait })
|
||||
const pending = runPrimaryBackendStartup(options)
|
||||
|
||||
await vi.waitFor(() => assert.equal(gate.hasWaiter(), true))
|
||||
gate.resetForRetry()
|
||||
|
||||
await assert.rejects(pending, error => error instanceof FirstRunSetupResetError && error.firstRunSetupReset)
|
||||
assert.equal(options.connectRemote.mock.calls.length, 0)
|
||||
assert.equal(options.ensureLocalRuntime.mock.calls.length, 0)
|
||||
})
|
||||
66
apps/desktop/electron/primary-backend-startup.ts
Normal file
66
apps/desktop/electron/primary-backend-startup.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import type { FirstRunSetupDecision } from './first-run-setup-gate'
|
||||
|
||||
export interface PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection> {
|
||||
connectRemote: (remote: Remote) => Promise<Connection>
|
||||
ensureLocalRuntime: (backend: Backend) => Promise<RuntimeBackend>
|
||||
prepareLocalBackend: () => Backend | Promise<Backend>
|
||||
resolveRemote: () => Promise<Remote | null>
|
||||
waitForDecision: (backend: Backend) => Promise<FirstRunSetupDecision>
|
||||
waitForLocalStart: () => Promise<unknown>
|
||||
}
|
||||
|
||||
export type PrimaryBackendStartupResult<RuntimeBackend, Connection> =
|
||||
| { kind: 'local'; backend: RuntimeBackend }
|
||||
| { kind: 'remote'; connection: Connection }
|
||||
|
||||
export class FirstRunSetupResetError extends Error {
|
||||
readonly firstRunSetupReset = true
|
||||
|
||||
constructor() {
|
||||
super('First-run setup was reset before a choice completed.')
|
||||
this.name = 'FirstRunSetupResetError'
|
||||
}
|
||||
}
|
||||
|
||||
// Owns the production startHermes path up to the local process spawn. Keeping
|
||||
// the full ordering here makes the first-run remote boundary executable in a
|
||||
// test: an already-saved remote wins immediately; otherwise update exclusion
|
||||
// and local backend resolution happen before the setup gate, and a remote Apply
|
||||
// re-resolves persisted config without ever entering ensureRuntime/bootstrap.
|
||||
export async function runPrimaryBackendStartup<Backend, RuntimeBackend, Remote, Connection>({
|
||||
connectRemote,
|
||||
ensureLocalRuntime,
|
||||
prepareLocalBackend,
|
||||
resolveRemote,
|
||||
waitForDecision,
|
||||
waitForLocalStart
|
||||
}: PrimaryBackendStartupOptions<Backend, RuntimeBackend, Remote, Connection>): Promise<
|
||||
PrimaryBackendStartupResult<RuntimeBackend, Connection>
|
||||
> {
|
||||
const savedRemote = await resolveRemote()
|
||||
|
||||
if (savedRemote) {
|
||||
return { kind: 'remote', connection: await connectRemote(savedRemote) }
|
||||
}
|
||||
|
||||
await waitForLocalStart()
|
||||
|
||||
const backend = await prepareLocalBackend()
|
||||
const decision = await waitForDecision(backend)
|
||||
|
||||
if (decision === 'remote-applied') {
|
||||
const appliedRemote = await resolveRemote()
|
||||
|
||||
if (!appliedRemote) {
|
||||
throw new Error('First-run remote setup completed without a saved remote backend.')
|
||||
}
|
||||
|
||||
return { kind: 'remote', connection: await connectRemote(appliedRemote) }
|
||||
}
|
||||
|
||||
if (decision === 'reset') {
|
||||
throw new FirstRunSetupResetError()
|
||||
}
|
||||
|
||||
return { kind: 'local', backend: await ensureLocalRuntime(backend) }
|
||||
}
|
||||
35
apps/desktop/electron/primary-connection-rehome.ts
Normal file
35
apps/desktop/electron/primary-connection-rehome.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
export interface PrimaryConnectionRehomeOptions {
|
||||
clearLocalBootstrapFailure: () => void
|
||||
mode: string
|
||||
notifyConnectionApplied: () => void
|
||||
resumeFirstRunRemote: () => boolean
|
||||
teardownPrimaryBackend: (options: { soft: boolean }) => Promise<void>
|
||||
}
|
||||
|
||||
// Production seam shared by the connection-config IPC handler and the
|
||||
// first-run integration test. A remote apply that resumes the active setup
|
||||
// gate must keep that connection attempt alive; ordinary mode changes tear the
|
||||
// current backend down before the renderer is told to reconnect.
|
||||
export async function rehomePrimaryConnection({
|
||||
clearLocalBootstrapFailure,
|
||||
mode,
|
||||
notifyConnectionApplied,
|
||||
resumeFirstRunRemote,
|
||||
teardownPrimaryBackend
|
||||
}: PrimaryConnectionRehomeOptions): Promise<{ resumedFirstRunRemote: boolean }> {
|
||||
let resumedFirstRunRemote = false
|
||||
|
||||
if (mode === 'remote') {
|
||||
resumedFirstRunRemote = resumeFirstRunRemote()
|
||||
clearLocalBootstrapFailure()
|
||||
}
|
||||
|
||||
if (resumedFirstRunRemote) {
|
||||
return { resumedFirstRunRemote: true }
|
||||
}
|
||||
|
||||
await teardownPrimaryBackend({ soft: true })
|
||||
notifyConnectionApplied()
|
||||
|
||||
return { resumedFirstRunRemote: false }
|
||||
}
|
||||
30
apps/desktop/electron/profile-session-routing.test.ts
Normal file
30
apps/desktop/electron/profile-session-routing.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { fetchPrimaryProfileSessions } from './profile-session-routing'
|
||||
|
||||
test('primary session reads use the profile-aware request path', async () => {
|
||||
const calls: Array<{ profile: string | null; path: string }> = []
|
||||
const expected = { sessions: [{ id: 'session-1' }], total: 1, profile_totals: { default: 1 } }
|
||||
|
||||
const result = await fetchPrimaryProfileSessions(
|
||||
new URLSearchParams({ profile: 'default', limit: '20' }),
|
||||
async (profile, path) => {
|
||||
calls.push({ profile, path })
|
||||
|
||||
return expected
|
||||
}
|
||||
)
|
||||
|
||||
assert.deepEqual(calls, [{ profile: null, path: '/api/profiles/sessions?profile=default&limit=20' }])
|
||||
assert.equal(result, expected)
|
||||
})
|
||||
|
||||
test('primary session reads preserve the empty-list fallback', async () => {
|
||||
const result = await fetchPrimaryProfileSessions(new URLSearchParams({ profile: 'all' }), async () => {
|
||||
throw new Error('remote unavailable')
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { sessions: [], total: 0, profile_totals: {} })
|
||||
})
|
||||
19
apps/desktop/electron/profile-session-routing.ts
Normal file
19
apps/desktop/electron/profile-session-routing.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
export interface ProfileSessionsResponse {
|
||||
sessions: unknown[]
|
||||
total: number
|
||||
profile_totals: Record<string, number>
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
type FetchJsonForProfile = (profile: string | null, path: string) => Promise<unknown>
|
||||
|
||||
export async function fetchPrimaryProfileSessions(
|
||||
searchParams: URLSearchParams,
|
||||
fetchJsonForProfile: FetchJsonForProfile
|
||||
): Promise<ProfileSessionsResponse> {
|
||||
try {
|
||||
return (await fetchJsonForProfile(null, `/api/profiles/sessions?${searchParams}`)) as ProfileSessionsResponse
|
||||
} catch {
|
||||
return { sessions: [], total: 0, profile_totals: {} }
|
||||
}
|
||||
}
|
||||
240
apps/desktop/electron/quick-entry.test.ts
Normal file
240
apps/desktop/electron/quick-entry.test.ts
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import {
|
||||
createQuickEntryShortcut,
|
||||
DEFAULT_QUICK_ENTRY_SHORTCUT,
|
||||
type GlobalShortcutLike,
|
||||
parseQuickEntryShortcut,
|
||||
quickEntryWindowBounds,
|
||||
sanitizeQuickEntrySettings
|
||||
} from './quick-entry'
|
||||
|
||||
function fakeGlobalShortcut(options: { register?: boolean; taken?: string[] } = {}) {
|
||||
const held = new Set(options.taken ?? [])
|
||||
|
||||
const globalShortcut: GlobalShortcutLike = {
|
||||
isRegistered: vi.fn((accelerator: string) => held.has(accelerator)),
|
||||
register: vi.fn((accelerator: string) => {
|
||||
if (options.register === false) {
|
||||
return false
|
||||
}
|
||||
|
||||
held.add(accelerator)
|
||||
|
||||
return true
|
||||
}),
|
||||
unregister: vi.fn((accelerator: string) => void held.delete(accelerator))
|
||||
}
|
||||
|
||||
return { globalShortcut, held }
|
||||
}
|
||||
|
||||
describe('parseQuickEntryShortcut', () => {
|
||||
it('normalizes casing, aliases, and modifier order', () => {
|
||||
expect(parseQuickEntryShortcut('cmdorctrl+shift+space')).toEqual({
|
||||
accelerator: 'CommandOrControl+Shift+Space',
|
||||
ok: true
|
||||
})
|
||||
expect(parseQuickEntryShortcut(' Shift + CTRL + k ')).toEqual({ accelerator: 'Control+Shift+K', ok: true })
|
||||
expect(parseQuickEntryShortcut('Alt+f5')).toEqual({ accelerator: 'Alt+F5', ok: true })
|
||||
expect(parseQuickEntryShortcut('Meta+/')).toEqual({ accelerator: 'Super+/', ok: true })
|
||||
})
|
||||
|
||||
it('collapses duplicate modifiers', () => {
|
||||
expect(parseQuickEntryShortcut('Ctrl+Control+Shift+J')).toEqual({ accelerator: 'Control+Shift+J', ok: true })
|
||||
})
|
||||
|
||||
it('requires a modifier so a global bind cannot swallow a bare key', () => {
|
||||
expect(parseQuickEntryShortcut('K')).toEqual({ ok: false, reason: 'no-modifier' })
|
||||
expect(parseQuickEntryShortcut('Space')).toEqual({ ok: false, reason: 'no-modifier' })
|
||||
})
|
||||
|
||||
it('requires exactly one non-modifier key', () => {
|
||||
expect(parseQuickEntryShortcut('Shift+Control')).toEqual({ ok: false, reason: 'no-key' })
|
||||
expect(parseQuickEntryShortcut('Shift+A+B')).toEqual({ ok: false, reason: 'invalid-key' })
|
||||
expect(parseQuickEntryShortcut('A+Shift')).toEqual({ ok: false, reason: 'invalid-modifier' })
|
||||
})
|
||||
|
||||
it('rejects empty, junk, and the reserved Escape key', () => {
|
||||
expect(parseQuickEntryShortcut('')).toEqual({ ok: false, reason: 'empty' })
|
||||
expect(parseQuickEntryShortcut(' ')).toEqual({ ok: false, reason: 'empty' })
|
||||
expect(parseQuickEntryShortcut(null)).toEqual({ ok: false, reason: 'empty' })
|
||||
expect(parseQuickEntryShortcut('Ctrl+NotAKey')).toEqual({ ok: false, reason: 'invalid-key' })
|
||||
// Escape hides the window; binding it globally would make it un-toggleable.
|
||||
expect(parseQuickEntryShortcut('Ctrl+Escape')).toEqual({ ok: false, reason: 'reserved' })
|
||||
})
|
||||
|
||||
it('accepts the shipped default unchanged', () => {
|
||||
expect(parseQuickEntryShortcut(DEFAULT_QUICK_ENTRY_SHORTCUT)).toEqual({
|
||||
accelerator: DEFAULT_QUICK_ENTRY_SHORTCUT,
|
||||
ok: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeQuickEntrySettings', () => {
|
||||
it('defaults to enabled with the default shortcut', () => {
|
||||
expect(sanitizeQuickEntrySettings(undefined)).toEqual({ enabled: true, shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT })
|
||||
expect(sanitizeQuickEntrySettings('not an object')).toEqual({
|
||||
enabled: true,
|
||||
shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps an explicit disable and normalizes a stored shortcut', () => {
|
||||
expect(sanitizeQuickEntrySettings({ enabled: false, shortcut: 'alt+j' })).toEqual({
|
||||
enabled: false,
|
||||
shortcut: 'Alt+J'
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to the default when the stored shortcut is unusable', () => {
|
||||
expect(sanitizeQuickEntrySettings({ enabled: true, shortcut: 'Q' })).toEqual({
|
||||
enabled: true,
|
||||
shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT
|
||||
})
|
||||
})
|
||||
|
||||
it('treats a non-boolean enabled as off (only `true` opts in once present)', () => {
|
||||
expect(sanitizeQuickEntrySettings({ enabled: 'yes' }).enabled).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createQuickEntryShortcut', () => {
|
||||
it('registers the normalized accelerator when enabled', () => {
|
||||
const { globalShortcut } = fakeGlobalShortcut()
|
||||
const onTrigger = vi.fn()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, onTrigger)
|
||||
|
||||
const state = controller.apply({ enabled: true, shortcut: 'cmdorctrl+shift+space' })
|
||||
|
||||
expect(state).toEqual({ error: null, registered: true, shortcut: 'CommandOrControl+Shift+Space' })
|
||||
expect(globalShortcut.register).toHaveBeenCalledWith('CommandOrControl+Shift+Space', onTrigger)
|
||||
expect(controller.current()).toEqual(state)
|
||||
})
|
||||
|
||||
it('never registers while the setting is disabled', () => {
|
||||
const { globalShortcut } = fakeGlobalShortcut()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
const state = controller.apply({ enabled: false, shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT })
|
||||
|
||||
expect(globalShortcut.register).not.toHaveBeenCalled()
|
||||
expect(state).toEqual({ error: null, registered: false, shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT })
|
||||
})
|
||||
|
||||
it('releases the old accelerator before registering a new one', () => {
|
||||
const { globalShortcut, held } = fakeGlobalShortcut()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
controller.apply({ enabled: true, shortcut: 'Alt+J' })
|
||||
controller.apply({ enabled: true, shortcut: 'Alt+K' })
|
||||
|
||||
expect(globalShortcut.unregister).toHaveBeenCalledWith('Alt+J')
|
||||
expect(held.has('Alt+J')).toBe(false)
|
||||
expect(held.has('Alt+K')).toBe(true)
|
||||
})
|
||||
|
||||
it('turning the feature off releases the live accelerator', () => {
|
||||
const { globalShortcut, held } = fakeGlobalShortcut()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
controller.apply({ enabled: true, shortcut: 'Alt+J' })
|
||||
const off = controller.apply({ enabled: false, shortcut: 'Alt+J' })
|
||||
|
||||
expect(globalShortcut.unregister).toHaveBeenCalledWith('Alt+J')
|
||||
expect(held.size).toBe(0)
|
||||
expect(off.registered).toBe(false)
|
||||
expect(off.error).toBeNull()
|
||||
})
|
||||
|
||||
it("surfaces 'taken' when another app already owns the chord", () => {
|
||||
const { globalShortcut } = fakeGlobalShortcut({ taken: ['Alt+J'] })
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
const state = controller.apply({ enabled: true, shortcut: 'alt+j' })
|
||||
|
||||
expect(globalShortcut.register).not.toHaveBeenCalled()
|
||||
expect(state).toEqual({ error: 'taken', registered: false, shortcut: 'Alt+J' })
|
||||
})
|
||||
|
||||
it("surfaces 'taken' when the OS refuses the registration", () => {
|
||||
const { globalShortcut } = fakeGlobalShortcut({ register: false })
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
expect(controller.apply({ enabled: true, shortcut: 'Alt+J' })).toEqual({
|
||||
error: 'taken',
|
||||
registered: false,
|
||||
shortcut: 'Alt+J'
|
||||
})
|
||||
})
|
||||
|
||||
it("surfaces 'invalid' for an unusable shortcut without asking the OS", () => {
|
||||
const { globalShortcut } = fakeGlobalShortcut()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
expect(controller.apply({ enabled: true, shortcut: 'J' })).toEqual({
|
||||
error: 'invalid',
|
||||
registered: false,
|
||||
shortcut: 'J'
|
||||
})
|
||||
expect(globalShortcut.register).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('survives a throwing globalShortcut', () => {
|
||||
const globalShortcut: GlobalShortcutLike = {
|
||||
isRegistered: () => false,
|
||||
register: () => {
|
||||
throw new Error('x11 grab failed')
|
||||
},
|
||||
unregister: () => {}
|
||||
}
|
||||
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
expect(controller.apply({ enabled: true, shortcut: 'Alt+J' }).error).toBe('taken')
|
||||
})
|
||||
|
||||
it('dispose releases the accelerator and is idempotent', () => {
|
||||
const { globalShortcut, held } = fakeGlobalShortcut()
|
||||
const controller = createQuickEntryShortcut(globalShortcut, vi.fn())
|
||||
|
||||
controller.apply({ enabled: true, shortcut: 'Alt+J' })
|
||||
controller.dispose()
|
||||
controller.dispose()
|
||||
|
||||
expect(globalShortcut.unregister).toHaveBeenCalledTimes(1)
|
||||
expect(held.size).toBe(0)
|
||||
expect(controller.current().registered).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('quickEntryWindowBounds', () => {
|
||||
it('centers horizontally and sits below the top edge of the work area', () => {
|
||||
const bounds = quickEntryWindowBounds({ height: 1000, width: 1600, x: 0, y: 0 })
|
||||
|
||||
expect(bounds.width).toBe(640)
|
||||
expect(bounds.x).toBe((1600 - 640) / 2)
|
||||
expect(bounds.y).toBeGreaterThan(0)
|
||||
expect(bounds.y + bounds.height).toBeLessThanOrEqual(1000)
|
||||
})
|
||||
|
||||
it('respects a display origin offset (second monitor)', () => {
|
||||
const bounds = quickEntryWindowBounds({ height: 900, width: 1440, x: 1600, y: -200 })
|
||||
|
||||
expect(bounds.x).toBe(1600 + (1440 - 640) / 2)
|
||||
expect(bounds.y).toBeGreaterThanOrEqual(-200)
|
||||
})
|
||||
|
||||
it('stays inside a tiny work area', () => {
|
||||
const bounds = quickEntryWindowBounds({ height: 120, width: 320, x: 0, y: 0 })
|
||||
|
||||
expect(bounds.width).toBeLessThanOrEqual(320)
|
||||
expect(bounds.height).toBeLessThanOrEqual(120)
|
||||
expect(bounds.y + bounds.height).toBeLessThanOrEqual(120)
|
||||
})
|
||||
|
||||
it('falls back to the origin without a work area', () => {
|
||||
expect(quickEntryWindowBounds()).toEqual({ height: 168, width: 640, x: 0, y: 0 })
|
||||
})
|
||||
})
|
||||
426
apps/desktop/electron/quick-entry.ts
Normal file
426
apps/desktop/electron/quick-entry.ts
Normal file
|
|
@ -0,0 +1,426 @@
|
|||
/**
|
||||
* Quick Entry — the global-hotkey mini composer.
|
||||
*
|
||||
* A small frameless always-on-top window that a global shortcut summons from
|
||||
* anywhere so the user can fire a prompt at Hermes without raising the whole
|
||||
* app. The window carries NO gateway connection of its own: it forwards the
|
||||
* text to the primary renderer, which sends it through the SAME prompt-submit
|
||||
* path the normal composer uses (see app/contrib/hooks/use-quick-entry-bridge).
|
||||
*
|
||||
* Everything Electron-free lives here so the parts that actually break a user —
|
||||
* accelerator validation, "disabled means never register", and surfacing a
|
||||
* shortcut another app already owns — are unit-testable without booting
|
||||
* Electron. main.ts owns the BrowserWindow, the file I/O, and the real
|
||||
* `globalShortcut`.
|
||||
*/
|
||||
|
||||
// Default matches the muscle memory of the apps this ports from (Claude
|
||||
// Desktop's quick entry / ChatGPT's Quick Chat sit on a Cmd+Shift chord).
|
||||
const DEFAULT_QUICK_ENTRY_SHORTCUT = 'CommandOrControl+Shift+Space'
|
||||
|
||||
// Compact capture surface: wide enough for a sentence, short enough to read as
|
||||
// a HUD rather than a second app window. Height covers the composer row plus
|
||||
// the session-target picker row; the renderer never grows the OS window in v1.
|
||||
const QUICK_ENTRY_WINDOW_WIDTH = 640
|
||||
const QUICK_ENTRY_WINDOW_HEIGHT = 168
|
||||
|
||||
// Spotlight-ish placement: horizontally centered on the active display, a
|
||||
// comfortable fraction down from the top rather than dead center.
|
||||
const QUICK_ENTRY_TOP_FRACTION = 0.22
|
||||
|
||||
// Electron accelerator vocabulary (electronjs.org/docs/latest/api/accelerator).
|
||||
// Kept as data so validation and the settings UI agree on one list.
|
||||
const ACCELERATOR_MODIFIERS = new Set([
|
||||
'alt',
|
||||
'altgr',
|
||||
'cmd',
|
||||
'cmdorctrl',
|
||||
'command',
|
||||
'commandorcontrol',
|
||||
'control',
|
||||
'ctrl',
|
||||
'meta',
|
||||
'option',
|
||||
'shift',
|
||||
'super'
|
||||
])
|
||||
|
||||
const ACCELERATOR_KEYS = new Set([
|
||||
'backspace',
|
||||
'delete',
|
||||
'down',
|
||||
'end',
|
||||
'enter',
|
||||
'escape',
|
||||
'home',
|
||||
'insert',
|
||||
'left',
|
||||
'medianexttrack',
|
||||
'mediaplaypause',
|
||||
'mediaprevioustrack',
|
||||
'mediastop',
|
||||
'pagedown',
|
||||
'pageup',
|
||||
'plus',
|
||||
'printscreen',
|
||||
'return',
|
||||
'right',
|
||||
'space',
|
||||
'tab',
|
||||
'up',
|
||||
'volumedown',
|
||||
'volumemute',
|
||||
'volumeup'
|
||||
])
|
||||
|
||||
// Single printable characters Electron accepts verbatim, plus 0-9 / A-Z below.
|
||||
const ACCELERATOR_PUNCTUATION = new Set([
|
||||
'!',
|
||||
'"',
|
||||
'#',
|
||||
'$',
|
||||
'%',
|
||||
'&',
|
||||
"'",
|
||||
'(',
|
||||
')',
|
||||
'*',
|
||||
'+',
|
||||
',',
|
||||
'-',
|
||||
'.',
|
||||
'/',
|
||||
':',
|
||||
';',
|
||||
'<',
|
||||
'=',
|
||||
'>',
|
||||
'?',
|
||||
'@',
|
||||
'[',
|
||||
'\\',
|
||||
']',
|
||||
'^',
|
||||
'_',
|
||||
'`',
|
||||
'{',
|
||||
'|',
|
||||
'}',
|
||||
'~'
|
||||
])
|
||||
|
||||
/** Why a shortcut string was rejected. The renderer maps these to copy. */
|
||||
export type QuickEntryShortcutError =
|
||||
| 'empty'
|
||||
| 'invalid-key'
|
||||
| 'invalid-modifier'
|
||||
| 'no-key'
|
||||
| 'no-modifier'
|
||||
| 'reserved'
|
||||
|
||||
export type QuickEntryShortcutParse = { ok: false; reason: QuickEntryShortcutError } | { accelerator: string; ok: true }
|
||||
|
||||
function isAcceleratorKey(token: string): boolean {
|
||||
if (ACCELERATOR_KEYS.has(token)) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (/^f([1-9]|1[0-9]|2[0-4])$/.test(token)) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (/^num(?:[0-9]|lock|dec|add|sub|mult|div)$/.test(token)) {
|
||||
return true
|
||||
}
|
||||
|
||||
return token.length === 1 && (/^[a-z0-9]$/.test(token) || ACCELERATOR_PUNCTUATION.has(token))
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate + normalize a user-typed accelerator.
|
||||
*
|
||||
* Rules beyond Electron's own grammar, both deliberate:
|
||||
* - At least one modifier. A bare global key steals that key from EVERY app.
|
||||
* - `Escape` can't be the key: inside the window Escape means "hide", so
|
||||
* binding it globally would make the shortcut un-toggleable.
|
||||
*/
|
||||
export function parseQuickEntryShortcut(raw: unknown): QuickEntryShortcutParse {
|
||||
if (typeof raw !== 'string' || !raw.trim()) {
|
||||
return { ok: false, reason: 'empty' }
|
||||
}
|
||||
|
||||
const parts = raw
|
||||
.split('+')
|
||||
.map(part => part.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
if (parts.length === 0) {
|
||||
return { ok: false, reason: 'empty' }
|
||||
}
|
||||
|
||||
const modifiers: string[] = []
|
||||
let key: null | string = null
|
||||
|
||||
for (const part of parts) {
|
||||
const lower = part.toLowerCase()
|
||||
|
||||
if (ACCELERATOR_MODIFIERS.has(lower)) {
|
||||
if (key) {
|
||||
// A modifier after the key ("A+Shift") is not a valid accelerator.
|
||||
return { ok: false, reason: 'invalid-modifier' }
|
||||
}
|
||||
|
||||
modifiers.push(lower)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if (key) {
|
||||
// Two non-modifier keys ("Shift+A+B").
|
||||
return { ok: false, reason: 'invalid-key' }
|
||||
}
|
||||
|
||||
if (!isAcceleratorKey(lower)) {
|
||||
return { ok: false, reason: 'invalid-key' }
|
||||
}
|
||||
|
||||
key = lower
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
return { ok: false, reason: 'no-key' }
|
||||
}
|
||||
|
||||
if (modifiers.length === 0) {
|
||||
return { ok: false, reason: 'no-modifier' }
|
||||
}
|
||||
|
||||
if (key === 'escape') {
|
||||
return { ok: false, reason: 'reserved' }
|
||||
}
|
||||
|
||||
// Canonical casing so a saved shortcut round-trips identically no matter how
|
||||
// the user typed it, and duplicate modifiers collapse.
|
||||
const seen = new Set<string>()
|
||||
|
||||
const normalizedModifiers = modifiers
|
||||
.map(modifier => CANONICAL_MODIFIER[modifier] ?? modifier)
|
||||
.filter(modifier => (seen.has(modifier) ? false : (seen.add(modifier), true)))
|
||||
// Stable display order (Electron itself is order-insensitive).
|
||||
.sort((left, right) => MODIFIER_ORDER.indexOf(left) - MODIFIER_ORDER.indexOf(right))
|
||||
|
||||
return { accelerator: [...normalizedModifiers, canonicalKey(key)].join('+'), ok: true }
|
||||
}
|
||||
|
||||
const CANONICAL_MODIFIER: Record<string, string> = {
|
||||
alt: 'Alt',
|
||||
altgr: 'AltGr',
|
||||
cmd: 'Command',
|
||||
cmdorctrl: 'CommandOrControl',
|
||||
command: 'Command',
|
||||
commandorcontrol: 'CommandOrControl',
|
||||
control: 'Control',
|
||||
ctrl: 'Control',
|
||||
meta: 'Super',
|
||||
option: 'Option',
|
||||
shift: 'Shift',
|
||||
super: 'Super'
|
||||
}
|
||||
|
||||
const MODIFIER_ORDER = ['CommandOrControl', 'Command', 'Control', 'Super', 'Alt', 'Option', 'AltGr', 'Shift']
|
||||
|
||||
const CANONICAL_KEY: Record<string, string> = {
|
||||
backspace: 'Backspace',
|
||||
delete: 'Delete',
|
||||
down: 'Down',
|
||||
end: 'End',
|
||||
enter: 'Enter',
|
||||
escape: 'Escape',
|
||||
home: 'Home',
|
||||
insert: 'Insert',
|
||||
medianexttrack: 'MediaNextTrack',
|
||||
mediaplaypause: 'MediaPlayPause',
|
||||
mediaprevioustrack: 'MediaPreviousTrack',
|
||||
mediastop: 'MediaStop',
|
||||
pagedown: 'PageDown',
|
||||
pageup: 'PageUp',
|
||||
plus: 'Plus',
|
||||
printscreen: 'PrintScreen',
|
||||
return: 'Return',
|
||||
right: 'Right',
|
||||
space: 'Space',
|
||||
tab: 'Tab',
|
||||
up: 'Up',
|
||||
volumedown: 'VolumeDown',
|
||||
volumemute: 'VolumeMute',
|
||||
volumeup: 'VolumeUp',
|
||||
left: 'Left'
|
||||
}
|
||||
|
||||
function canonicalKey(key: string): string {
|
||||
if (CANONICAL_KEY[key]) {
|
||||
return CANONICAL_KEY[key]
|
||||
}
|
||||
|
||||
if (/^f([1-9]|1[0-9]|2[0-4])$/.test(key)) {
|
||||
return key.toUpperCase()
|
||||
}
|
||||
|
||||
if (key.length === 1 && /^[a-z]$/.test(key)) {
|
||||
return key.toUpperCase()
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
/** The persisted shape of `quick-entry.json` (main-process owned). */
|
||||
export interface QuickEntrySettings {
|
||||
enabled: boolean
|
||||
shortcut: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw persisted JSON → usable settings. A malformed/absent file, or a shortcut
|
||||
* that no longer validates (hand-edited, or from a future build), falls back to
|
||||
* the default shortcut rather than leaving the feature un-summonable.
|
||||
*/
|
||||
export function sanitizeQuickEntrySettings(raw: unknown): QuickEntrySettings {
|
||||
const record = raw && typeof raw === 'object' ? (raw as Record<string, unknown>) : {}
|
||||
const parsed = parseQuickEntryShortcut(record.shortcut)
|
||||
|
||||
return {
|
||||
// Default ON: the feature is inert until the shortcut is pressed.
|
||||
enabled: record.enabled === undefined ? true : record.enabled === true,
|
||||
shortcut: parsed.ok ? parsed.accelerator : DEFAULT_QUICK_ENTRY_SHORTCUT
|
||||
}
|
||||
}
|
||||
|
||||
/** The slice of Electron's `globalShortcut` we use (injected for testing). */
|
||||
export interface GlobalShortcutLike {
|
||||
isRegistered(accelerator: string): boolean
|
||||
register(accelerator: string, callback: () => void): boolean
|
||||
unregister(accelerator: string): void
|
||||
}
|
||||
|
||||
/**
|
||||
* What Settings shows. `registered` is the ground truth (we asked the OS);
|
||||
* `error` distinguishes "you turned it off" from "another app owns that chord",
|
||||
* which is the failure this feature must never swallow.
|
||||
*/
|
||||
export interface QuickEntryRegistration {
|
||||
error: null | QuickEntryRegistrationError
|
||||
registered: boolean
|
||||
shortcut: string
|
||||
}
|
||||
|
||||
export type QuickEntryRegistrationError = 'invalid' | 'taken'
|
||||
|
||||
export interface QuickEntryShortcutController {
|
||||
/** Registration state as of the last apply. */
|
||||
current(): QuickEntryRegistration
|
||||
/** Release the shortcut (quit / feature off). Idempotent. */
|
||||
dispose(): void
|
||||
/** Re-register to match `settings`. Returns the resulting state. */
|
||||
apply(settings: QuickEntrySettings): QuickEntryRegistration
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the one live global accelerator. Single resolver so every caller — boot,
|
||||
* the settings write, quit — gets the same answer and we can never leak two
|
||||
* registrations for one feature.
|
||||
*
|
||||
* Disabled settings never touch `register()` at all: a user who turned Quick
|
||||
* Entry off must not have their chord silently held hostage.
|
||||
*/
|
||||
export function createQuickEntryShortcut(
|
||||
globalShortcut: GlobalShortcutLike,
|
||||
onTrigger: () => void
|
||||
): QuickEntryShortcutController {
|
||||
let active: null | string = null
|
||||
let state: QuickEntryRegistration = { error: null, registered: false, shortcut: DEFAULT_QUICK_ENTRY_SHORTCUT }
|
||||
|
||||
const release = () => {
|
||||
if (active) {
|
||||
try {
|
||||
globalShortcut.unregister(active)
|
||||
} catch {
|
||||
// Best effort — a dead accelerator must not block a re-register.
|
||||
}
|
||||
|
||||
active = null
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
apply(settings) {
|
||||
const parsed = parseQuickEntryShortcut(settings.shortcut)
|
||||
const shortcut = parsed.ok ? parsed.accelerator : settings.shortcut
|
||||
|
||||
release()
|
||||
|
||||
if (!settings.enabled) {
|
||||
state = { error: null, registered: false, shortcut }
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
if (!parsed.ok) {
|
||||
state = { error: 'invalid', registered: false, shortcut }
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
// `isRegistered` catches the common conflict before we ask, and
|
||||
// `register()` returning false catches the rest (another process owns it
|
||||
// OS-wide). Both land in the same surfaced 'taken' state.
|
||||
let ok = false
|
||||
|
||||
try {
|
||||
ok = globalShortcut.isRegistered(parsed.accelerator)
|
||||
? false
|
||||
: globalShortcut.register(parsed.accelerator, onTrigger)
|
||||
} catch {
|
||||
ok = false
|
||||
}
|
||||
|
||||
active = ok ? parsed.accelerator : null
|
||||
state = { error: ok ? null : 'taken', registered: ok, shortcut: parsed.accelerator }
|
||||
|
||||
return state
|
||||
},
|
||||
current() {
|
||||
return state
|
||||
},
|
||||
dispose() {
|
||||
release()
|
||||
state = { ...state, error: null, registered: false }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the quick window opens on a given display work area. Centered
|
||||
* horizontally, a fraction down from the top, and clamped so it stays fully
|
||||
* inside the work area on small/odd displays.
|
||||
*/
|
||||
export function quickEntryWindowBounds(workArea?: { height: number; width: number; x: number; y: number }): {
|
||||
height: number
|
||||
width: number
|
||||
x: number
|
||||
y: number
|
||||
} {
|
||||
const width = Math.min(QUICK_ENTRY_WINDOW_WIDTH, workArea?.width ?? QUICK_ENTRY_WINDOW_WIDTH)
|
||||
const height = Math.min(QUICK_ENTRY_WINDOW_HEIGHT, workArea?.height ?? QUICK_ENTRY_WINDOW_HEIGHT)
|
||||
|
||||
if (!workArea) {
|
||||
return { height, width, x: 0, y: 0 }
|
||||
}
|
||||
|
||||
const x = Math.round(workArea.x + (workArea.width - width) / 2)
|
||||
const maxY = workArea.y + workArea.height - height
|
||||
const y = Math.round(Math.min(Math.max(workArea.y, workArea.y + workArea.height * QUICK_ENTRY_TOP_FRACTION), maxY))
|
||||
|
||||
return { height, width, x, y }
|
||||
}
|
||||
|
||||
export { DEFAULT_QUICK_ENTRY_SHORTCUT, QUICK_ENTRY_TOP_FRACTION, QUICK_ENTRY_WINDOW_HEIGHT, QUICK_ENTRY_WINDOW_WIDTH }
|
||||
|
|
@ -6,6 +6,7 @@ import {
|
|||
REMOTE_LIVENESS_TIMEOUT_MS,
|
||||
RemoteLivenessTracker,
|
||||
RemoteRevalidationCoordinator,
|
||||
revalidatePooledRemoteBackends,
|
||||
revalidateRemoteConnection
|
||||
} from './remote-liveness'
|
||||
|
||||
|
|
@ -251,3 +252,102 @@ describe('revalidateRemoteConnection', () => {
|
|||
expect(rejected.probe).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('revalidatePooledRemoteBackends', () => {
|
||||
const harness = (entries: Array<[string, { process?: unknown; remoteBaseUrl?: null | string }]>) => {
|
||||
const unreachable = new Set<string>()
|
||||
const log = vi.fn()
|
||||
const stopBackend = vi.fn()
|
||||
|
||||
const probe = vi.fn(async (url: string) => {
|
||||
if ([...unreachable].some(base => url.startsWith(base))) {
|
||||
throw new Error('unreachable')
|
||||
}
|
||||
|
||||
return {}
|
||||
})
|
||||
|
||||
return {
|
||||
log,
|
||||
probe,
|
||||
stopBackend,
|
||||
unreachable,
|
||||
run: (tracker: RemoteLivenessTracker) =>
|
||||
revalidatePooledRemoteBackends({ entries, log, probe, stopBackend, tracker })
|
||||
}
|
||||
}
|
||||
|
||||
it('probes only pooled entries backed by a remote host', async () => {
|
||||
const local = { process: {}, remoteBaseUrl: null }
|
||||
const spawning = { process: null, remoteBaseUrl: null }
|
||||
const remote = { process: null, remoteBaseUrl: 'https://remote.example.com' }
|
||||
|
||||
const pool = harness([
|
||||
['local', local],
|
||||
['spawning', spawning],
|
||||
['remote', remote]
|
||||
])
|
||||
|
||||
await pool.run(new RemoteLivenessTracker())
|
||||
|
||||
expect(pool.probe).toHaveBeenCalledTimes(1)
|
||||
expect(pool.probe).toHaveBeenCalledWith('https://remote.example.com/api/status', {
|
||||
timeoutMs: REMOTE_LIVENESS_TIMEOUT_MS
|
||||
})
|
||||
expect(pool.stopBackend).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('drops a descriptor only after the shared failure limit', async () => {
|
||||
const pool = harness([['coder', { process: null, remoteBaseUrl: 'https://remote.example.com/' }]])
|
||||
pool.unreachable.add('https://remote.example.com')
|
||||
|
||||
const tracker = new RemoteLivenessTracker()
|
||||
|
||||
for (let attempt = 1; attempt < REMOTE_LIVENESS_FAILURE_LIMIT; attempt += 1) {
|
||||
await expect(pool.run(tracker)).resolves.toEqual({ dropped: [] })
|
||||
expect(pool.stopBackend).not.toHaveBeenCalled()
|
||||
}
|
||||
|
||||
await expect(pool.run(tracker)).resolves.toEqual({ dropped: ['coder'] })
|
||||
expect(pool.stopBackend).toHaveBeenCalledWith('coder')
|
||||
})
|
||||
|
||||
it('clears the streak when the host answers again', async () => {
|
||||
const pool = harness([['coder', { process: null, remoteBaseUrl: 'https://remote.example.com' }]])
|
||||
const tracker = new RemoteLivenessTracker()
|
||||
|
||||
pool.unreachable.add('https://remote.example.com')
|
||||
await pool.run(tracker)
|
||||
|
||||
pool.unreachable.clear()
|
||||
await pool.run(tracker)
|
||||
|
||||
pool.unreachable.add('https://remote.example.com')
|
||||
|
||||
for (let attempt = 1; attempt < REMOTE_LIVENESS_FAILURE_LIMIT; attempt += 1) {
|
||||
await expect(pool.run(tracker)).resolves.toEqual({ dropped: [] })
|
||||
}
|
||||
|
||||
expect(pool.stopBackend).not.toHaveBeenCalled()
|
||||
await expect(pool.run(tracker)).resolves.toEqual({ dropped: ['coder'] })
|
||||
})
|
||||
|
||||
it('keeps a healthy sibling when another profile on a different host dies', async () => {
|
||||
const pool = harness([
|
||||
['coder', { process: null, remoteBaseUrl: 'https://dead.example.com' }],
|
||||
['writer', { process: null, remoteBaseUrl: 'https://live.example.com' }]
|
||||
])
|
||||
|
||||
pool.unreachable.add('https://dead.example.com')
|
||||
|
||||
const tracker = new RemoteLivenessTracker()
|
||||
|
||||
for (let attempt = 1; attempt < REMOTE_LIVENESS_FAILURE_LIMIT; attempt += 1) {
|
||||
await pool.run(tracker)
|
||||
}
|
||||
|
||||
await expect(pool.run(tracker)).resolves.toEqual({ dropped: ['coder'] })
|
||||
expect(pool.stopBackend).toHaveBeenCalledTimes(1)
|
||||
expect(pool.stopBackend).toHaveBeenCalledWith('coder')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue