Merge remote-tracking branch 'origin/main' into merge/relay-metrics-upstream-20260723

This commit is contained in:
Alex Fournier 2026-07-23 14:10:49 -07:00
commit fb1e417367
281 changed files with 26317 additions and 1673 deletions

View file

@ -1936,8 +1936,8 @@ class HermesACPAgent(acp.Agent):
return "No tools available."
lines = [f"Available tools ({len(tools)}):"]
for t in tools:
name = t.get("function", {}).get("name", "?")
desc = t.get("function", {}).get("description", "")
name = (t.get("function") or {}).get("name", "?")
desc = (t.get("function") or {}).get("description", "")
# Truncate long descriptions
if len(desc) > 80:
desc = desc[:77] + "..."

View file

@ -922,12 +922,17 @@ def recover_with_credential_pool(
)
return False, has_retried_429
# Capture the current API key before any rotation — needed to
# identify which credential actually failed when
# mark_exhausted_and_rotate is called. Without this hint the
# pool falls back to current() or _select_unlocked(), which may
# return the NEXT (healthy) entry after a prior rotation, marking
# the wrong credential as exhausted (#43747).
# Attribute the failure to the API key the agent actually dispatched the
# request with, not to pool.current(). The current() pointer is shared,
# mutable state — round-robin select() advances it on every call, and
# concurrent turns or a second process (gateway/dashboard) reloading the
# pool reset it to None — so by the time recovery runs it routinely points
# at a DIFFERENT, healthy entry. Marking that entry exhausted copies this
# request's error/reset time onto it and can take the whole pool offline
# from a single rate-limited key (#43747). ``_swap_credential`` keeps
# ``agent.api_key`` in sync with the entry in use, so it identifies the
# failing entry exactly; fall back to current()'s key only when the agent
# carries no key at all.
_api_key_hint = getattr(agent, "api_key", None) or None
if not _api_key_hint:
_cur = pool.current()
@ -987,7 +992,16 @@ def recover_with_credential_pool(
# rotate immediately. This prevents the "cancel-between-429s" trap
# where has_retried_429 (a local var) gets reset on each new prompt,
# causing the pool to retry the same exhausted credential forever.
current_entry = pool.current()
# Prefer the entry matching the failing key over the shared current()
# pointer, for the same attribution reason as above.
current_entry = None
if _api_key_hint:
current_entry = next(
(e for e in pool.entries() if e.runtime_api_key == _api_key_hint),
None,
)
if current_entry is None:
current_entry = pool.current()
current_last_status = getattr(current_entry, "last_status", None) if current_entry else None
if current_last_status == STATUS_EXHAUSTED:
_ra().logger.info(
@ -995,7 +1009,11 @@ def recover_with_credential_pool(
current_last_status,
)
rotate_status = status_code if status_code is not None else 429
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
if next_entry is not None:
_ra().logger.info(
"Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s",
@ -1019,7 +1037,11 @@ def recover_with_credential_pool(
if not has_retried_429 and not usage_limit_reached:
return False, True
rotate_status = status_code if status_code is not None else 429
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
if next_entry is not None:
_ra().logger.info(
"Credential %s (rate limit) — rotated to pool entry %s",
@ -1034,7 +1056,7 @@ def recover_with_credential_pool(
# Subscription/entitlement 403s look like auth failures on the wire
# but refresh cannot fix them — the OAuth token is already valid,
# the account simply lacks the entitlement. Without this guard,
# ``try_refresh_current()`` keeps minting fresh tokens against the
# the refresh path keeps minting fresh tokens against the
# same unsubscribed account and the main agent loop spins re-issuing
# the same 403 until the user Ctrl+C's.
#
@ -1087,9 +1109,13 @@ def recover_with_credential_pool(
agent.provider or "provider",
)
return False, has_retried_429
refreshed = pool.try_refresh_current()
# Refresh the entry that supplied the failing key, not current():
# the shared pointer can reference a different, healthy entry, and
# refreshing it would consume that entry's single-use refresh token
# (or mark it exhausted on failure) for a failure it never had.
refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint)
if refreshed is not None:
# ``try_refresh_current()`` re-mints a fresh OAuth token and reports
# ``try_refresh_matching()`` re-mints a fresh OAuth token and reports
# success even when the upstream keeps rejecting it — a single-entry
# pool (common for OAuth/Max subscribers) has nothing to rotate to,
# so a bare "refreshed → retry" loop spins forever on the same dead
@ -1117,9 +1143,13 @@ def recover_with_credential_pool(
agent._swap_credential(refreshed)
return True, has_retried_429
# Refresh failed — rotate to next credential instead of giving up.
# The failed entry is already marked exhausted by try_refresh_current().
# The failed entry is already marked exhausted by the refresh attempt.
rotate_status = status_code if status_code is not None else 401
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
next_entry = pool.mark_exhausted_and_rotate(
status_code=rotate_status,
error_context=error_context,
api_key_hint=_api_key_hint,
)
if next_entry is not None:
_ra().logger.info(
"Credential %s (auth refresh failed) — rotated to pool entry %s",

View file

@ -1881,6 +1881,28 @@ def _content_parts_to_anthropic_blocks(parts: Any) -> List[Dict[str, Any]]:
return out
_EMPTY_TEXT_PLACEHOLDER = "(empty)"
def _safe_text(text: Any) -> str:
"""Return ``text`` if it's non-whitespace, else a non-whitespace placeholder.
The Anthropic Messages API rejects requests where a text content block is
empty or whitespace-only (HTTP 400 "text content blocks must contain
non-whitespace text"). When such a block gets stored in session history —
e.g. produced by context compression it is replayed verbatim on every
subsequent turn, permanently wedging the session. Coercing to a
non-whitespace placeholder is self-healing: the next API call recovers.
Mirrors ``bedrock_adapter._safe_text`` (#9486); ref #69512.
"""
if text is None:
return _EMPTY_TEXT_PLACEHOLDER
if not isinstance(text, str):
text = str(text)
return text if text.strip() else _EMPTY_TEXT_PLACEHOLDER
def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Strip output-only fields from a stored Anthropic content block so it is
valid as REQUEST input on replay.
@ -1898,7 +1920,10 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
return None
btype = b.get("type")
if btype == "text":
out: Dict[str, Any] = {"type": "text", "text": b.get("text", "")}
# Coerce empty/whitespace-only text to a non-whitespace placeholder;
# the Messages input schema rejects blank text blocks (#69512), and a
# blank block stored in history replays on every turn → permanent 400.
out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))}
# citations is input-valid ONLY when it's a non-empty list; the SDK
# emits citations=None on responses, which the input schema rejects.
cits = b.get("citations")
@ -2058,7 +2083,16 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
# Anthropic rejects empty assistant content
effective = blocks or content
if not effective or effective == "":
effective = [{"type": "text", "text": "(empty)"}]
effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
elif isinstance(effective, list):
# The all-empty guard above misses a list that still contains a
# whitespace-only text block (e.g. from a content array of blank parts,
# or compression). Those also trip "text content blocks must contain
# non-whitespace text" (#69512). Coerce text blocks in place; other
# block types (thinking/tool_use/image) are left untouched.
for blk in effective:
if isinstance(blk, dict) and blk.get("type") == "text":
blk["text"] = _safe_text(blk.get("text", ""))
return {"role": "assistant", "content": effective}

View file

@ -2305,6 +2305,62 @@ def _read_main_base_url() -> str:
return ""
def _resolve_moa_aggregator(preset_name: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
"""Resolve a MoA preset to its aggregator (provider, model) pair.
"moa" is a virtual provider the acting model of a preset is its
aggregator slot, and there is no real "moa" HTTP endpoint. Auxiliary
tasks (title generation, compression, vision, commit messages, ) don't
need the reference fan-out, so every aux resolution layer maps
provider="moa"/model=<preset> to the aggregator's real provider+model
through this single helper (shared by ``_resolve_auto``,
``_resolve_task_provider_model``, and ``resolve_provider_client`` so the
preset lookup and validation cannot drift between paths).
Args:
preset_name: The MoA preset name (usually carried in the "model"
field), or None/"" to resolve the user's default preset.
Returns:
(aggregator_provider, aggregator_model), or (None, None) when the
preset cannot be resolved (missing config, renamed/deleted preset,
or a malformed aggregator slot).
"""
try:
from hermes_cli.config import load_config
from hermes_cli.moa_config import resolve_moa_preset
preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None)
agg = preset.get("aggregator") or {}
agg_provider = str(agg.get("provider") or "").strip()
agg_model = str(agg.get("model") or "").strip()
if agg_provider and agg_model and agg_provider.lower() != "moa":
return agg_provider, agg_model
except Exception:
logger.debug(
"MoA aggregator resolution failed for preset %r", preset_name, exc_info=True
)
return None, None
def _read_main_model_for_aux() -> str:
"""Main model with MoA presets unwrapped to the aggregator's model.
When the main provider is ``moa``, ``_read_main_model()`` returns a MoA
*preset name* (e.g. "opus-gpt") never a valid wire model id on any
provider. Auxiliary fallback chains that pre-fill a missing model from
the main model must use this reader instead, so unset aux models default
to the preset's acting (aggregator) model. Returns "" when the main
provider is moa but the preset cannot be resolved sending nothing is
strictly better than sending a preset name that 400s.
"""
model = _read_main_model()
if (_read_main_provider() or "").strip().lower() == "moa":
_, agg_model = _resolve_moa_aggregator(model)
return agg_model or ""
return model
def _read_main_api_key_if_same_host(aux_base_url: str) -> str:
"""Return the main api_key only when *aux_base_url* points at the same
host as the main model's base_url.
@ -2734,7 +2790,7 @@ def _try_custom_endpoint() -> Tuple[Optional[Any], Optional[str]]:
return None, None
if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()):
return None, None
model = _read_main_model() or "gpt-4o-mini"
model = _read_main_model_for_aux() or "gpt-4o-mini"
logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions")
_clean_base, _dq = _extract_url_query_params(custom_base)
_extra = {"default_query": _dq} if _dq else {}
@ -4234,6 +4290,13 @@ def _try_main_agent_model_fallback(
"""
main_provider = (_read_main_provider() or "").strip()
main_model = (_read_main_model() or "").strip()
if main_provider.lower() == "moa":
# MoA virtual provider: fall back to the preset's aggregator — the
# acting model — instead of the unreachable "moa"/<preset-name> pair.
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
if not _agg_provider or not _agg_model:
return None, None, ""
main_provider, main_model = _agg_provider, _agg_model
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
return None, None, ""
@ -4634,26 +4697,17 @@ def _resolve_auto(
# model. Resolve the MoA preset to its aggregator slot and continue Step 1
# with that real provider+model. Mirrors the MoA context-length resolution.
if main_provider == "moa":
try:
from hermes_cli.config import load_config
from hermes_cli.moa_config import resolve_moa_preset
_preset = resolve_moa_preset(load_config().get("moa") or {}, main_model)
_agg = _preset.get("aggregator") or {}
_agg_provider = str(_agg.get("provider") or "").strip()
_agg_model = str(_agg.get("model") or "").strip()
if _agg_provider and _agg_model and _agg_provider.lower() != "moa":
main_provider = _agg_provider
main_model = _agg_model
# The MoA virtual runtime carries a non-HTTP base_url
# ("moa://local") and a placeholder api_key; they belong to the
# facade, not the aggregator's real provider. Drop them so the
# aggregator resolves through its own provider credentials.
runtime_base_url = ""
runtime_api_key = ""
runtime_api_mode = ""
except Exception:
logger.debug("MoA aux resolution to aggregator failed", exc_info=True)
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
if _agg_provider and _agg_model:
main_provider = _agg_provider
main_model = _agg_model
# The MoA virtual runtime carries a non-HTTP base_url
# ("moa://local") and a placeholder api_key; they belong to the
# facade, not the aggregator's real provider. Drop them so the
# aggregator resolves through its own provider credentials.
runtime_base_url = ""
runtime_api_key = ""
runtime_api_mode = ""
if (main_provider and main_model
and main_provider not in {"auto", ""}):
@ -4908,6 +4962,27 @@ def resolve_provider_client(
# Normalise aliases
provider = _normalize_aux_provider(provider)
# MoA virtual provider chokepoint: "moa" is not a real HTTP provider —
# its acting model is the preset's aggregator slot. The two resolver
# layers above (_resolve_auto, _resolve_task_provider_model) already
# unwrap their own paths, but callers that route here directly (vision
# auto-detect, _try_main_agent_model_fallback, get_available_vision_backends,
# plugin code) would otherwise dead-end in the unknown-provider branch.
# ``model`` carries the preset name for moa calls; when the preset can't
# be resolved we leave the call untouched and let the normal
# missing-provider handling produce its diagnostic.
if provider == "moa":
_agg_provider, _agg_model = _resolve_moa_aggregator(model)
if _agg_provider and _agg_model:
original_provider = _agg_provider.strip().lower()
provider = _normalize_aux_provider(_agg_provider)
model = _agg_model
# The moa:// facade endpoint and placeholder key belong to the
# virtual runtime, not the aggregator's real provider.
if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"):
explicit_base_url = None
explicit_api_key = None
# Universal model-resolution fallback for concrete providers. ``auto`` is
# intentionally excluded: `_resolve_auto(main_runtime=...)` returns the
# model paired with the provider it actually selected. Pre-filling an auto
@ -4928,6 +5003,10 @@ def resolve_provider_client(
# the load-bearing step for OAuth providers: an xai-oauth user
# with grok-4.3 configured gets grok-4.3 for title generation
# instead of silently dropping to whatever Step-2 fallback (#31845).
# When the main provider is MoA, ``_read_main_model_for_aux()``
# substitutes the preset's aggregator model — the preset NAME is
# never a valid wire model id, so unset aux models default to the
# preset's acting model instead.
#
# Each provider branch below sees a non-empty ``model`` whenever the
# user has *anything* configured — no provider-specific empty-model
@ -4944,7 +5023,7 @@ def resolve_provider_client(
# return the actual current runtime model when the caller did not explicitly
# request one. (# compression-current-model)
if not model and provider != "auto":
model = _get_aux_model_for_provider(provider) or _read_main_model() or model
model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model
def _needs_codex_wrap(client_obj, base_url_str: str, model_str: str) -> bool:
"""Decide if a plain OpenAI client should be wrapped for Responses API.
@ -5218,7 +5297,7 @@ def resolve_provider_client(
model
or custom_entry.get("model")
or (main_runtime.get("model") if main_runtime else None)
or _read_main_model()
or _read_main_model_for_aux()
or "gpt-4o-mini",
provider,
)
@ -5453,7 +5532,7 @@ def resolve_provider_client(
final_model = _normalize_resolved_model(
model
or (main_runtime.get("model") if main_runtime else None)
or _read_main_model(),
or _read_main_model_for_aux(),
provider,
)
if provider == "copilot-acp":
@ -5821,7 +5900,24 @@ def resolve_vision_provider_client(
# 5. Stop
main_provider = str(runtime.get("provider") or _read_main_provider())
main_model = str(runtime.get("model") or _read_main_model())
if main_provider and main_provider not in {"auto", ""}:
if main_provider.strip().lower() == "moa":
# MoA virtual provider: main_model is a preset NAME, and every
# capability probe below (_PROVIDERS_WITHOUT_VISION,
# _main_model_supports_vision, _resolve_provider_vision_default)
# would run against a provider/model pair that doesn't exist on
# any wire. Unwrap to the preset's aggregator slot first so the
# checks and the eventual client target the real acting model.
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
if _agg_provider and _agg_model:
main_provider, main_model = _agg_provider, _agg_model
# Drop the moa:// facade endpoint from the runtime view used
# below — it belongs to the virtual provider, not the
# aggregator's real provider.
runtime = dict(runtime)
runtime["base_url"] = ""
runtime["api_key"] = ""
runtime["api_mode"] = ""
if main_provider and main_provider not in {"auto", "", "moa"}:
# A provider-specific vision default wins over the user's chat model:
# static overrides (xiaomi/zai) and catalog-backed discovery (the
# DeepInfra profile hook) both yield a *known* vision-capable model,
@ -6418,8 +6514,8 @@ def _resolve_task_provider_model(
task: str = None,
provider: str = None,
model: str = None,
base_url: str = None,
api_key: str = None,
base_url: Optional[str] = None,
api_key: Optional[str] = None,
) -> Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]:
"""Determine provider + model for a call.
@ -6462,12 +6558,57 @@ def _resolve_task_provider_model(
# which downstream consumers like ContextCompressor accept as the task output.
# The provider-side 'auto' is handled in _resolve_auto() via main_runtime
# fallback, so dropping cfg_model to None here lets that path do its job.
#
# The explicit `model` kwarg needs the identical normalization: MoA slots
# (agent/moa_loop.py's _slot_runtime) forward a preset's `model:` field as
# this explicit argument rather than through auxiliary.<task> config, so a
# user-configured `model: auto` on a MoA reference/aggregator slot reaches
# this function here, not as cfg_model. Only normalizing cfg_model let that
# literal "auto" slip through via `model or cfg_model` below.
if model and model.lower() == "auto":
model = None
if cfg_model and cfg_model.lower() == "auto":
cfg_model = None
resolved_model = model or cfg_model
resolved_api_mode = cfg_api_mode
# MoA virtual provider: an *explicit* `provider: moa` override (either the
# caller-passed `provider` arg or `auxiliary.<task>.provider` in
# config.yaml) reaches this function directly — it never goes through
# _resolve_auto(), which only unwraps the *implicit* "main provider is
# moa" case (#53827). Left as-is, "moa" is returned verbatim and
# resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has
# no "moa" entry — it's not a real HTTP provider), falls to the
# unknown-provider dead end, and call_llm surfaces a nonsensical
# "MOA_API_KEY environment variable" error for a provider that was never
# meant to be reached over the wire. Auxiliary tasks don't need the
# reference fan-out — resolve to the preset's aggregator slot instead,
# exactly like the implicit path does (shared helper: _resolve_moa_aggregator).
def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> Tuple[str, Optional[str]]:
if prov.strip().lower() != "moa":
return prov, mdl
agg_provider, agg_model = _resolve_moa_aggregator(mdl)
if agg_provider and agg_model:
return agg_provider, agg_model
return prov, mdl
if provider and str(provider).strip().lower() == "moa":
provider, resolved_model = _unwrap_moa_provider(provider, resolved_model)
# The moa:// virtual endpoint (if any explicit base_url/api_key was
# passed alongside provider="moa") belongs to the facade, not the
# aggregator's real provider — drop it so the aggregator resolves
# through its own provider credentials, mirroring _resolve_auto().
if provider and provider.lower() != "moa":
base_url = None
api_key = None
elif cfg_provider and str(cfg_provider).strip().lower() == "moa":
cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model)
if cfg_provider and cfg_provider.lower() != "moa":
resolved_model = cfg_model
cfg_base_url = None
cfg_api_key = None
# Convenience aliases for direct API-key endpoints that aren't first-class
# providers (e.g. ``provider: openai`` → custom + api.openai.com/v1).
# Applied to both explicit args and config-derived values. When the user

View file

@ -433,6 +433,29 @@ def _model_supports_tool_use(model_id: str) -> bool:
return not any(pattern in model_lower for pattern in _NON_TOOL_CALLING_PATTERNS)
# ---------------------------------------------------------------------------
# Prompt-cache capability detection (Converse API cachePoint)
# ---------------------------------------------------------------------------
# Claude on Bedrock already gets prompt caching through the AnthropicBedrock
# SDK path (see is_anthropic_bedrock_model / runtime_provider.py's dual-path
# routing) — it never reaches build_converse_kwargs unless bearer-token auth
# forces the Converse path (#28156). This allowlist covers the Converse API
# itself: sending an unsupported model a cachePoint block raises a
# ValidationException, so — like _model_supports_tool_use but inverted —
# unknown models default to NOT receiving cache markers until confirmed.
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
_CACHE_POINT_PATTERNS = [
"anthropic.claude", # bearer-token fallback path
"amazon.nova",
]
def _model_supports_prompt_cache(model_id: str) -> bool:
"""Return True if the model accepts a Converse API cachePoint block."""
model_lower = model_id.lower()
return any(pattern in model_lower for pattern in _CACHE_POINT_PATTERNS)
def is_anthropic_bedrock_model(model_id: str) -> bool:
"""Return True if the model is an Anthropic Claude model on Bedrock.
@ -764,14 +787,22 @@ def normalize_converse_response(response: Dict) -> SimpleNamespace:
reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None,
)
# Build usage stats
# Build usage stats. Converse's inputTokens excludes cache read/write
# tokens (unlike OpenAI's prompt_tokens, which includes them) — restore
# the OpenAI-style "total includes cache" convention here so downstream
# normalize_usage() can subtract them back out consistently, and surface
# the Anthropic-named fields it already falls back to for cache reads.
usage_data = response.get("usage", {})
input_tokens = usage_data.get("inputTokens", 0)
cache_read_tokens = usage_data.get("cacheReadInputTokens", 0)
cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0)
output_tokens = usage_data.get("outputTokens", 0)
usage = SimpleNamespace(
prompt_tokens=usage_data.get("inputTokens", 0),
completion_tokens=usage_data.get("outputTokens", 0),
total_tokens=(
usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0)
),
prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens,
completion_tokens=output_tokens,
total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens,
cache_read_input_tokens=cache_read_tokens,
cache_creation_input_tokens=cache_write_tokens,
)
finish_reason = _converse_stop_reason_to_openai(stop_reason)
@ -936,6 +967,8 @@ def stream_converse_with_callbacks(
usage_data = {
"inputTokens": meta_usage.get("inputTokens", 0),
"outputTokens": meta_usage.get("outputTokens", 0),
"cacheReadInputTokens": meta_usage.get("cacheReadInputTokens", 0),
"cacheWriteInputTokens": meta_usage.get("cacheWriteInputTokens", 0),
}
# Flush remaining text
@ -949,12 +982,16 @@ def stream_converse_with_callbacks(
reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None,
)
input_tokens = usage_data.get("inputTokens", 0)
cache_read_tokens = usage_data.get("cacheReadInputTokens", 0)
cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0)
output_tokens = usage_data.get("outputTokens", 0)
usage = SimpleNamespace(
prompt_tokens=usage_data.get("inputTokens", 0),
completion_tokens=usage_data.get("outputTokens", 0),
total_tokens=(
usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0)
),
prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens,
completion_tokens=output_tokens,
total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens,
cache_read_input_tokens=cache_read_tokens,
cache_creation_input_tokens=cache_write_tokens,
)
finish_reason = _converse_stop_reason_to_openai(stop_reason)
@ -993,6 +1030,7 @@ def build_converse_kwargs(
Converts OpenAI-format inputs to Converse API parameters.
"""
system_prompt, converse_messages = convert_messages_to_converse(messages)
cache_enabled = _model_supports_prompt_cache(model)
kwargs: Dict[str, Any] = {
"modelId": model,
@ -1003,6 +1041,8 @@ def build_converse_kwargs(
}
if system_prompt:
if cache_enabled:
system_prompt = system_prompt + [{"cachePoint": {"type": "default"}}]
kwargs["system"] = system_prompt
from agent.anthropic_adapter import _forbids_sampling_params
@ -1026,6 +1066,8 @@ def build_converse_kwargs(
# Strip tools for known non-tool-calling models and warn the user.
# Ref: PR #7920 feedback from @ptlally, pattern from PR #4346.
if _model_supports_tool_use(model):
if cache_enabled:
converse_tools = converse_tools + [{"cachePoint": {"type": "default"}}]
kwargs["toolConfig"] = {"tools": converse_tools}
else:
logger.warning(
@ -1033,6 +1075,14 @@ def build_converse_kwargs(
"The agent will operate in text-only mode.", model
)
if cache_enabled and len(converse_messages) >= 2:
# Checkpoint everything up to (not including) the newest turn, so the
# marker survives unchanged across requests as only the tail grows —
# mirroring the Anthropic system_and_3 strategy in prompt_caching.py.
content = converse_messages[-2].get("content")
if isinstance(content, list) and content:
content.append({"cachePoint": {"type": "default"}})
if guardrail_config:
kwargs["guardrailConfig"] = guardrail_config

View file

@ -335,6 +335,11 @@ _ACTIVE_TASK_MAX_CHARS = 1400
# high for small/light tails, but using all 20 as a hard floor here would bring
# back the old large-tool-output case where nothing can be compacted.
_MAX_TAIL_MESSAGE_FLOOR = 8
# Under context pressure (protected-tail tool bodies alone exceed the soft
# tail budget), demote large completed tool/file outputs even inside the
# protected region — but always keep this many trailing messages verbatim so
# the active user ask / latest tool pair remain readable. Issue #61932.
_PRESSURE_KEEP_RECENT_MESSAGES = 3
# Models with context windows below this get their compression threshold
# floored at ``_SMALL_CTX_THRESHOLD_PERCENT`` (raise-only — an explicitly
@ -1129,8 +1134,10 @@ class ContextCompressor(ContextEngine):
self._last_summary_error = None
self._consecutive_timeout_failures = 0
self._fallback_compression_streak = 0
self._ineffective_compression_count = 0
self.get_active_compression_failure_cooldown()
self._load_fallback_compression_streak()
self._load_ineffective_compression_count()
def on_session_start(self, session_id: str, **kwargs) -> None:
"""Bind session-scoped compression state for a new or resumed session."""
@ -1139,6 +1146,7 @@ class ContextCompressor(ContextEngine):
old_session_id = kwargs.get("old_session_id")
session_db = kwargs.get("session_db", getattr(self, "_session_db", None))
previous_fallback_streak = self._fallback_compression_streak
previous_ineffective_count = self._ineffective_compression_count
if boundary_reason == "compression" and old_session_id:
getter = getattr(session_db, "get_compression_fallback_streak", None)
if callable(getter):
@ -1153,12 +1161,37 @@ class ContextCompressor(ContextEngine):
"compression parent fallback streak lookup failed (non-sqlite): %s",
exc,
)
count_getter = getattr(
session_db, "get_compression_ineffective_count", None,
)
if callable(count_getter):
try:
stored_count = count_getter(old_session_id)
if isinstance(stored_count, (int, float, str)):
previous_ineffective_count = max(0, int(stored_count))
except (TypeError, ValueError, sqlite3.Error) as exc:
logger.debug(
"compression parent ineffective count lookup failed: %s", exc,
)
except Exception as exc:
logger.debug(
"compression parent ineffective count lookup failed (non-sqlite): %s",
exc,
)
self.bind_session_state(session_db, session_id)
if boundary_reason == "compression":
# Rotation creates a fresh child row before this callback. Preserve
# the logical conversation's streak until boundary bookkeeping
# persists the updated value onto the child row.
self._fallback_compression_streak = previous_fallback_streak
# Same for the anti-thrash strike counter — but unlike the streak,
# no later boundary bookkeeping writes it, so persist the carried
# value onto the (fresh) child row now. Otherwise a restart between
# rotation and the next real-usage verdict would silently disarm
# an armed guard (#54923).
if self._ineffective_compression_count != previous_ineffective_count:
self._ineffective_compression_count = previous_ineffective_count
self._persist_ineffective_compression_count()
def _load_fallback_compression_streak(self) -> None:
session_db = getattr(self, "_session_db", None)
@ -1192,6 +1225,59 @@ class ContextCompressor(ContextEngine):
except Exception as exc:
logger.debug("compression fallback streak persist failed (non-sqlite): %s", exc)
def _load_ineffective_compression_count(self) -> None:
"""Load the durable anti-thrash strike count for the bound session.
A fresh compressor on a resumed session starts with
``compression_count == 0`` and, historically, an in-memory-only
ineffective counter so a guard armed (1 strike) or tripped
(2 strikes) before a process restart silently disarmed, and a
near-threshold session could re-compact once per restart forever
(#54923). The counter now round-trips through the session row like
the failure cooldown and the fallback streak.
"""
session_db = getattr(self, "_session_db", None)
session_id = getattr(self, "_session_id", "")
getter = getattr(session_db, "get_compression_ineffective_count", None)
if not session_id or not callable(getter):
return
try:
stored_count = getter(session_id)
self._ineffective_compression_count = max(
0,
int(stored_count)
if isinstance(stored_count, (int, float, str))
else 0,
)
except (TypeError, ValueError, sqlite3.Error) as exc:
logger.debug("compression ineffective count lookup failed: %s", exc)
except Exception as exc:
logger.debug("compression ineffective count lookup failed (non-sqlite): %s", exc)
def _persist_ineffective_compression_count(self) -> None:
session_db = getattr(self, "_session_db", None)
session_id = getattr(self, "_session_id", "")
setter = getattr(session_db, "set_compression_ineffective_count", None)
if not session_id or not callable(setter):
return
try:
setter(session_id, self._ineffective_compression_count)
except sqlite3.Error as exc:
logger.debug("compression ineffective count persist failed: %s", exc)
except Exception as exc:
logger.debug("compression ineffective count persist failed (non-sqlite): %s", exc)
def _record_ineffective_compression_verdict(self, count: int) -> None:
"""Set the anti-thrash strike counter, keeping the durable copy in sync.
Persists only on change so the reset issued by every ordinary fitting
response (already-zero -> zero) never costs a DB write.
"""
if count == self._ineffective_compression_count:
return
self._ineffective_compression_count = count
self._persist_ineffective_compression_count()
def record_completed_compaction(self, *, used_fallback: bool = False) -> None:
"""Record one completed boundary and its summary quality."""
self._verify_compaction_cleared_threshold = True
@ -1400,7 +1486,10 @@ class ContextCompressor(ContextEngine):
self.last_rough_tokens_when_real_prompt_fit = 0
self.last_compression_rough_tokens = 0
self.awaiting_real_usage_after_compression = False
self._ineffective_compression_count = 0
# Strikes were judged against the PREVIOUS threshold; a recomputed
# trigger invalidates them. Keep the durable copy in sync so a
# restart doesn't resurrect strikes this recalibration just voided.
self._record_ineffective_compression_verdict(0)
if runtime_changed:
self._fallback_compression_streak = 0
self._persist_fallback_compression_streak()
@ -1724,7 +1813,7 @@ class ContextCompressor(ContextEngine):
# when this response was not immediately after compaction. The
# independent fallback streak is boundary-scoped and survives
# ordinary fitting responses during context regrowth.
self._ineffective_compression_count = 0
self._record_ineffective_compression_verdict(0)
else:
self.last_rough_tokens_when_real_prompt_fit = 0
@ -1746,7 +1835,9 @@ class ContextCompressor(ContextEngine):
# per compaction.
if self._verify_compaction_cleared_threshold:
if self.last_prompt_tokens >= self.threshold_tokens:
self._ineffective_compression_count += 1
self._record_ineffective_compression_verdict(
self._ineffective_compression_count + 1,
)
if not self.quiet_mode:
logger.warning(
"Compaction did not clear the threshold: %d real "
@ -1758,7 +1849,7 @@ class ContextCompressor(ContextEngine):
self._ineffective_compression_count,
)
else:
self._ineffective_compression_count = 0
self._record_ineffective_compression_verdict(0)
# Consume the pending-verification flag once real usage arrives, whether
# or not prompt_tokens was reported, so a usage-less response can't leave
# it armed for a later, unrelated reading.
@ -1810,23 +1901,83 @@ class ContextCompressor(ContextEngine):
def should_compress(self, prompt_tokens: int = None) -> bool:
"""Check if context exceeds the compression threshold.
Returns ``True`` when compression should run now. For the caller-facing
*reason* (e.g. why compression is skipped while still over threshold),
see :meth:`should_compress_info`, which returns a ``(bool, reason)``
tuple without changing the decision logic here.
Includes anti-thrashing protection: if the last two compressions
each saved less than 10%, skip compression to avoid infinite loops
where each pass removes only 1-2 messages.
"""
decision, _reason = self.should_compress_info(prompt_tokens)
return decision
def should_compress_info(
self, prompt_tokens: int = None
) -> "tuple[bool, str | None]":
"""Check if context exceeds the compression threshold.
Returns a ``(should_compress, reason)`` tuple instead of a bare bool so
callers can tell *why* compression is skipped when it is skipped while
the context is already over threshold. ``reason`` is ``None`` unless
compression is needed but blocked:
* ``"cooldown:<seconds>"`` the summary LLM is recovering from a
recent 429/transient failure; compression is deferred to avoid the
freeze loop described in #11529.
* ``"ineffective"`` anti-thrashing has backed off because the last
two compressions each saved <10%.
When ``reason`` is non-``None`` the session is over its compression
threshold yet cannot shrink callers should surface a warning so the
user knows the model may silently stop answering (the context keeps
growing until it hits the hard provider limit). Without this signal an
over-threshold session fails opaquely.
Includes anti-thrashing protection: if the last two compressions
each saved less than 10%, skip compression to avoid infinite loops
where each pass removes only 1-2 messages.
"""
tokens = prompt_tokens if prompt_tokens is not None else self.last_prompt_tokens
if tokens < self.threshold_tokens:
return False
return not self._automatic_compression_blocked()
return False, None
if self._automatic_compression_blocked():
return False, self._compression_block_reason() or "blocked"
return True, None
def _compression_block_reason(self) -> "str | None":
"""Return a human-readable reason for the current automatic-compaction
block, derived from the same in-memory state that
:meth:`_automatic_compression_blocked_locally` evaluates.
* ``"cooldown:<seconds>"`` the summary LLM is recovering from a
recent 429/transient failure; compression is deferred to avoid the
freeze loop described in #11529.
* ``"ineffective"`` anti-thrashing has backed off (the last two
compressions each saved <10%, or the fallback streak tripped).
* ``None`` no block active.
"""
_cooldown_remaining = self._summary_failure_cooldown_until - time.monotonic()
if _cooldown_remaining > 0:
return f"cooldown:{_cooldown_remaining:.0f}"
if (
self._ineffective_compression_count >= 2
or self._fallback_compression_streak >= 2
):
return "ineffective"
return None
def _refresh_durable_guards(self) -> None:
"""Re-read durable cooldown + fallback-streak state from the DB.
"""Re-read durable cooldown + breaker state from the DB.
Cheap, best-effort, and only called when a gate is about to say
"blocked": another agent on the same session may have cleared the
durable rows (successful boundary, forced retry) after this
compressor was bound, and a fallback streak has no timer without
a re-read the stale in-memory snapshot blocks forever.
durable rows (successful boundary, forced retry, a real usage
reading that dipped below the threshold) after this compressor was
bound, and neither the fallback streak nor the ineffective-strike
counter has a timer without a re-read the stale in-memory
snapshot blocks forever.
"""
try:
self.get_active_compression_failure_cooldown(refresh=True)
@ -1836,26 +1987,22 @@ class ContextCompressor(ContextEngine):
self._load_fallback_compression_streak()
except Exception as exc:
logger.debug("compression fallback-streak refresh failed: %s", exc)
try:
self._load_ineffective_compression_count()
except Exception as exc:
logger.debug("compression ineffective-count refresh failed: %s", exc)
def _automatic_compression_blocked(self) -> bool:
"""Return whether automatic compaction is in cooldown or tripped."""
if not self._automatic_compression_blocked_locally():
return False
# Blocked on the in-memory snapshot. Durable guard rows may have
# been cleared by another agent since bind_session_state(); refresh
# and re-evaluate so a stale local block cannot outlive the durable
# state that justified it. The unblocked hot path above never pays
# for the DB reads.
if (
self._summary_failure_cooldown_until <= time.monotonic()
and self._fallback_compression_streak < 2
):
# Blocked solely by the in-memory ineffective-compression
# counter, which is not durable — there is nothing in the DB
# that could unblock it, so skip the refresh (otherwise this
# branch would re-read the DB on every gate check for the rest
# of the session).
return True
# been cleared by another agent since bind_session_state() — a
# successful boundary, a forced retry, or a real usage reading
# below the threshold (which zeroes the durable ineffective
# counter) — so refresh and re-evaluate before letting a stale
# local block outlive the durable state that justified it. The
# unblocked hot path above never pays for the DB reads.
self._refresh_durable_guards()
return self._automatic_compression_blocked_locally()
@ -1918,7 +2065,14 @@ class ContextCompressor(ContextEngine):
fall within ``protect_tail_tokens`` (when provided) OR the last
``protect_tail_count`` messages (backward-compatible default).
When both are given, the token budget takes priority and the message
count acts as a hard minimum floor.
count acts as a hard minimum floor capped at
``_MAX_TAIL_MESSAGE_FLOOR`` so a default ``protect_last_n=20`` cannot
freeze a whole run of bulky tool outputs against pruning.
When the protected region itself still exceeds the soft tail budget
(``protect_tail_tokens * 1.5``), a pressure pass demotes large
completed tool/file outputs *inside* that region while keeping a
short recent floor verbatim (issue #61932).
Returns (pruned_messages, pruned_count).
"""
@ -1946,10 +2100,17 @@ class ContextCompressor(ContextEngine):
# Determine the prune boundary
if protect_tail_tokens is not None and protect_tail_tokens > 0:
# Token-budget approach: walk backward accumulating tokens
# Token-budget approach: walk backward accumulating tokens.
# Cap the message-count floor the same way tail-cut does so a
# default protect_last_n=20 cannot lock a bulky recent tool run
# outside the compressible / prunable window (#61932).
accumulated = 0
boundary = len(result)
min_protect = min(protect_tail_count, len(result))
min_protect = min(
protect_tail_count,
len(result),
_MAX_TAIL_MESSAGE_FLOOR,
)
for i in range(len(result) - 1, -1, -1):
msg = result[i]
msg_tokens = _estimate_msg_budget_tokens(msg)
@ -1997,54 +2158,53 @@ class ContextCompressor(ContextEngine):
else:
content_hashes[h] = (i, msg.get("tool_call_id", "?"))
# Pass 2: Replace old tool results with informative summaries
for i in range(prune_boundary):
msg = result[i]
def _demote_tool_result_at(idx: int) -> bool:
"""Replace a bulky tool result at ``idx`` with a 1-line summary.
Returns True when the message was modified.
"""
nonlocal pruned
msg = result[idx]
if msg.get("role") != "tool":
continue
return False
content = msg.get("content", "")
# Multimodal content (base64 screenshots etc.): strip the image
# payload — keep a lightweight text placeholder in its place.
# Without this, an old computer_use screenshot (~1MB base64 +
# ~1500 real tokens) survives every compression pass forever.
if isinstance(content, list):
stripped = _strip_image_parts_from_parts(content)
if stripped is not None:
result[i] = {**msg, "content": stripped}
result[idx] = {**msg, "content": stripped}
pruned += 1
continue
return True
return False
if isinstance(content, dict) and content.get("_multimodal"):
summary = content.get("text_summary") or "[screenshot removed to save context]"
result[i] = {**msg, "content": f"[screenshot removed] {summary[:200]}"}
result[idx] = {**msg, "content": f"[screenshot removed] {summary[:200]}"}
pruned += 1
continue
return True
if not isinstance(content, str):
continue
return False
if not content or content == _PRUNED_TOOL_PLACEHOLDER:
continue
# Skip already-deduplicated or previously-summarized results
return False
if content.startswith("[Duplicate tool output"):
continue
# Only prune if the content is substantial (>200 chars)
if len(content) > 200:
call_id = msg.get("tool_call_id", "")
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
summary = _summarize_tool_result(tool_name, tool_args, content)
result[i] = {**msg, "content": summary}
pruned += 1
return False
# Already replaced by a prior prune/pressure pass (1-line summary).
if content.startswith("[") and " chars)" in content and len(content) < 400:
return False
if content.startswith("[screenshot removed"):
return False
if len(content) <= 200:
return False
call_id = msg.get("tool_call_id", "")
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
summary = _summarize_tool_result(tool_name, tool_args, content)
result[idx] = {**msg, "content": summary}
pruned += 1
return True
# Pass 3: Truncate large tool_call arguments in assistant messages
# outside the protected tail. write_file with 50KB content, for
# example, survives pruning entirely without this.
#
# The shrinking is done inside the parsed JSON structure so the
# result remains valid JSON — otherwise downstream providers 400
# on every subsequent turn until the broken call falls out of
# the window. See ``_truncate_tool_call_args_json`` docstring.
for i in range(prune_boundary):
msg = result[i]
def _truncate_tool_call_args_at(idx: int) -> bool:
"""Shrink large tool_call argument payloads at ``idx``."""
msg = result[idx]
if msg.get("role") != "assistant" or not msg.get("tool_calls"):
continue
return False
new_tcs = []
modified = False
for tc in msg["tool_calls"]:
@ -2057,7 +2217,93 @@ class ContextCompressor(ContextEngine):
modified = True
new_tcs.append(tc)
if modified:
result[i] = {**msg, "tool_calls": new_tcs}
result[idx] = {**msg, "tool_calls": new_tcs}
return modified
# Pass 2: Replace old tool results with informative summaries
for i in range(max(0, prune_boundary)):
_demote_tool_result_at(i)
# Pass 3: Truncate large tool_call arguments in assistant messages
# outside the protected tail. write_file with 50KB content, for
# example, survives pruning entirely without this.
#
# The shrinking is done inside the parsed JSON structure so the
# result remains valid JSON — otherwise downstream providers 400
# on every subsequent turn until the broken call falls out of
# the window. See ``_truncate_tool_call_args_json`` docstring.
for i in range(max(0, prune_boundary)):
_truncate_tool_call_args_at(i)
# Pass 4 (issue #61932): protected-tail pressure demotion.
# After multiple in-place compactions the transcript can be short
# enough that nearly every remaining message sits inside the
# protected floor, yet those messages are huge completed tool /
# file outputs. Summarizing the (empty) middle does nothing and
# preflight ends in "Cannot compress further". Demote bulky tool
# bodies *inside* the protected region until the protected tail
# fits the soft budget, always keeping a short recent floor
# verbatim so the active ask stays readable.
if protect_tail_tokens is not None and protect_tail_tokens > 0 and result:
soft_ceiling = int(protect_tail_tokens * 1.5)
keep_recent = min(_PRESSURE_KEEP_RECENT_MESSAGES, len(result))
demote_end = len(result) - keep_recent
def _protected_region_tokens() -> int:
start = max(0, prune_boundary)
return sum(
_estimate_msg_budget_tokens(result[i])
for i in range(start, len(result))
)
if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling:
pressure_hits = 0
for i in range(max(0, prune_boundary), demote_end):
if _demote_tool_result_at(i):
pressure_hits += 1
if _truncate_tool_call_args_at(i):
pressure_hits += 1
if _protected_region_tokens() <= soft_ceiling:
break
# If the short recent floor itself is still dominated by a
# stack of huge tool bodies, demote every protected tool
# result except the single most recent one. The active
# user message (usually the last row) stays untouched.
if _protected_region_tokens() > soft_ceiling:
last_tool_idx = None
for i in range(len(result) - 1, -1, -1):
if result[i].get("role") == "tool":
last_tool_idx = i
break
for i in range(max(0, prune_boundary), len(result)):
if last_tool_idx is not None and i == last_tool_idx:
continue
if result[i].get("role") == "tool":
if _demote_tool_result_at(i):
pressure_hits += 1
elif result[i].get("role") == "assistant":
if _truncate_tool_call_args_at(i):
pressure_hits += 1
# Absolute last resort: even the newest tool body can
# be larger than the soft budget alone (one 200KB file
# read). Summarize it so compression can still reclaim
# enough headroom to continue the session.
if (
last_tool_idx is not None
and last_tool_idx >= prune_boundary
and _protected_region_tokens() > soft_ceiling
):
if _demote_tool_result_at(last_tool_idx):
pressure_hits += 1
if pressure_hits and not self.quiet_mode:
logger.info(
"Pre-compression pressure demotion: reclaimed protected-tail "
"tool output (%d change(s); protected region now ~%s tokens, "
"soft ceiling %s)",
pressure_hits,
f"{_protected_region_tokens():,}",
f"{soft_ceiling:,}",
)
return result, pruned
@ -4083,7 +4329,9 @@ This compaction should PRIORITISE preserving all information related to the focu
# threshold because of the incompressible floor (system prompt +
# tool schemas), every subsequent turn re-fires a compaction that
# returns here unchanged, and the CLI appears frozen.
self._ineffective_compression_count += 1
self._record_ineffective_compression_verdict(
self._ineffective_compression_count + 1,
)
self._last_compression_savings_pct = 0.0
telemetry["failure_class"] = "insufficient_messages"
if not self.quiet_mode:
@ -4151,7 +4399,9 @@ This compaction should PRIORITISE preserving all information related to the focu
# an ineffective compression the anti-thrashing guard in
# should_compress() never fires and every subsequent turn
# re-triggers a no-op compression loop. (#40803)
self._ineffective_compression_count += 1
self._record_ineffective_compression_verdict(
self._ineffective_compression_count + 1,
)
self._last_compression_savings_pct = 0.0
if not self.quiet_mode:
logger.warning(

View file

@ -53,6 +53,39 @@ def sanitize_memory_context(memory_context: str) -> str:
)
def automatic_compaction_status_message(
engine: Any,
*,
phase: str,
default_message: str,
**context: Any,
) -> str | None:
"""Resolve host-visible status for an automatic compaction event.
Engines can suppress routine automatic status with
``emit_automatic_compaction_status = False`` or customize it by defining
``get_automatic_compaction_status_message(...)``. Empty strings and
``None`` mean "do not emit a lifecycle status".
"""
if not getattr(engine, "emit_automatic_compaction_status", True):
return None
formatter = getattr(engine, "get_automatic_compaction_status_message", None)
if callable(formatter):
message = formatter(
phase=phase,
default_message=default_message,
**context,
)
else:
message = default_message
if message is None:
return None
message = str(message).strip()
return message or None
class ContextEngine(ABC):
"""Base class all context engines must implement."""
@ -89,6 +122,12 @@ class ContextEngine(ABC):
protect_first_n: int = 3
protect_last_n: int = 6
# User-visible lifecycle status for automatic host-triggered compaction.
# Alternative engines that treat compaction as routine background
# maintenance can set this false to keep successful automatic passes silent;
# warnings, errors, and explicit manual commands should still surface.
emit_automatic_compaction_status: bool = True
# -- Core interface ----------------------------------------------------
@abstractmethod
@ -107,6 +146,19 @@ class ContextEngine(ABC):
def should_compress(self, prompt_tokens: int = None) -> bool:
"""Return True if compaction should fire this turn."""
def should_compress_info(self, prompt_tokens: int = None) -> "tuple[bool, str | None]":
"""Return ``(should_compress, reason)``.
The base implementation is backward-compatible: engines that only
implement ``should_compress`` get ``(should_compress(prompt_tokens),
None)``. Concrete engines with richer block reasons (e.g. a
summary-LLM cooldown or an anti-thrashing guard) override this to
surface a human-readable reason so callers can warn the user instead
of silently skipping compression. Added for the silent-overflow
warning fix (#62625) so plugin engines don't raise AttributeError.
"""
return self.should_compress(prompt_tokens), None
@abstractmethod
def compress(
self,
@ -156,6 +208,27 @@ class ContextEngine(ABC):
"""
return False
def get_automatic_compaction_status_message(
self,
*,
phase: str,
default_message: str,
**context: Any,
) -> str | None:
"""Return user-visible status for automatic host-triggered compaction.
Return ``None`` to suppress successful automatic lifecycle status for
this compaction event. ``phase`` identifies the host call site (for
example ``"preflight"`` or ``"compress"``). ``context`` contains
best-effort fields such as ``approx_tokens`` and ``threshold_tokens``.
This hook does not control warning/error messages or explicit manual
commands such as ``/compress``.
"""
if not self.emit_automatic_compaction_status:
return None
return default_message
# -- Optional: manual /compress preflight ------------------------------
def has_content_to_compress(self, messages: List[Dict[str, Any]]) -> bool:

View file

@ -42,7 +42,10 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Optional, Tuple
from agent.context_engine import sanitize_memory_context
from agent.context_engine import (
automatic_compaction_status_message,
sanitize_memory_context,
)
from agent.model_metadata import estimate_request_tokens_rough
logger = logging.getLogger(__name__)
@ -106,6 +109,22 @@ COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE = (
"🗜️ Context reduced to {new_ctx:,} tokens (was {old_ctx:,}), retrying..."
)
# FAILURE-CLASS notice — a deliberate carve-out from routine-compression
# silence (#16775 class): the context is over the compression threshold but
# compression is blocked (summary-LLM cooldown / anti-thrash breaker), so the
# session will keep growing until the hard provider token limit kills it.
# This MUST stay visible on chat gateways. Do NOT add it to
# ROUTINE_COMPRESSION_STATUS_SAMPLES or the gateway noise regex
# (_TELEGRAM_NOISY_STATUS_RE); it is pinned un-swallowed in
# tests/gateway/test_telegram_noise_filter.py::VISIBLE_COMPRESSION_MESSAGES.
CONTEXT_OVERFLOW_BLOCKED_WARNING_TEMPLATE = (
"⚠ Context is over the compression threshold "
"(~{tokens:,} tokens >= {threshold:,}) "
"but compression is currently blocked ({reason}). "
"The model may stop responding. Run /new to start a fresh "
"session or /compress to retry immediately."
)
# Sample-formatted instances of every routine compression status line, for
# behavioral tests that iterate the ACTUAL emitted wording (formatted from the
# same constants the emission sites use) through the gateway noise filter.
@ -188,6 +207,64 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo
return True
class CompressionCommitFence:
"""Fence timeout cancellation against post-summary session mutation.
Compression itself is synchronous and may be running in an executor thread.
A caller can stop waiting for the summary, but it cannot kill that thread.
This fence makes the commit boundary deterministic: cancellation either wins
before session mutation starts, or waits until an already-started commit is
fully complete before the caller proceeds.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._cancelled = False
self._commit_started = False
def cancel_before_commit(self) -> bool:
"""Cancel a pending commit, or wait for an active commit to finish.
Returns ``True`` when cancellation won before the commit boundary.
Returns ``False`` when the worker had already entered the boundary; in
that case acquiring this lock waits until all session mutation finishes.
"""
with self._lock:
if self._commit_started:
return False
self._cancelled = True
return True
def try_cancel_before_commit(self) -> Optional[bool]:
"""Non-blocking form of :meth:`cancel_before_commit`.
Returns ``None`` while an active commit owns the fence, allowing an
async caller to yield instead of blocking its event loop.
"""
if not self._lock.acquire(blocking=False):
return None
try:
if self._commit_started:
return False
self._cancelled = True
return True
finally:
self._lock.release()
def begin_commit(self) -> bool:
"""Enter the commit boundary unless cancellation already won."""
self._lock.acquire()
if self._cancelled:
self._lock.release()
return False
self._commit_started = True
return True
def finish_commit(self) -> None:
"""Leave a commit boundary entered by :meth:`begin_commit`."""
self._lock.release()
def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool:
"""Whether the live in-memory SessionDB class structurally predates locks.
@ -216,6 +293,7 @@ def _refresh_persisted_compression_guards(compressor: Any) -> None:
method_calls = (
("get_active_compression_failure_cooldown", {"refresh": True}),
("_load_fallback_compression_streak", {}),
("_load_ineffective_compression_count", {}),
)
for method_name, kwargs in method_calls:
method = getattr(type(compressor), method_name, None)
@ -730,7 +808,11 @@ def replay_compression_warning(agent: Any) -> None:
pass
def conversation_history_after_compression(agent: Any, messages: list) -> Optional[list]:
def conversation_history_after_compression(
agent: Any,
messages: list,
previous_history: Optional[list] = None,
) -> Optional[list]:
"""Return the correct flush baseline after a compression boundary.
Legacy compression rotates to a fresh child session. That child has not
@ -747,7 +829,19 @@ def conversation_history_after_compression(agent: Any, messages: list) -> Option
A shallow copy is intentional: it captures the current compacted dict
identities as history while allowing later same-turn appends to remain new.
An aborted or no-op attempt after an earlier in-place compaction must retain
the pre-attempt baseline. Treating all current messages as persisted would
drop any later, unflushed turns on restart; clearing the baseline would
append the already-persisted compacted rows a second time.
"""
if bool(getattr(agent, "_last_compression_attempt_recorded", False)):
attempt_in_place = getattr(agent, "_last_compression_attempt_in_place", None)
if attempt_in_place is True:
return list(messages)
if attempt_in_place is False:
return None
return previous_history
if bool(getattr(agent, "_last_compaction_in_place", False)):
return list(messages)
return None
@ -806,6 +900,37 @@ def _is_real_user_message(message: Any) -> bool:
return not ContextCompressor._is_synthetic_compression_user_turn(message)
def _strip_stale_todo_snapshot(content: Any) -> Any:
"""Remove a previously merged todo-snapshot block from message content.
Snapshot merges (see the injection site in ``compress_context``) always
append the block at the end of the trailing user turn, so a surviving
header marks stale todo state from an earlier compaction boundary.
Stripping before re-injection keeps repeated boundaries from
accumulating outdated snapshots (#26981).
"""
from tools.todo_tool import TODO_INJECTION_HEADER
if isinstance(content, str):
idx = content.find(TODO_INJECTION_HEADER)
if idx == -1:
return content
return content[:idx].rstrip()
if isinstance(content, list):
return [
part
for part in content
if not (
isinstance(part, dict)
and part.get("type") == "text"
and str(part.get("text") or "")
.lstrip()
.startswith(TODO_INJECTION_HEADER)
)
]
return content
def _merge_anchor_into_user_message(target: dict, anchor: dict) -> None:
"""Fold the human anchor into an existing user-role scaffolding turn.
@ -975,6 +1100,7 @@ def compress_context(
focus_topic: Optional[str] = None,
force: bool = False,
defer_context_engine_notification: bool = False,
commit_fence: Optional[CompressionCommitFence] = None,
) -> Tuple[list, str]:
"""Compress conversation context and split the session in SQLite.
@ -994,6 +1120,9 @@ def compress_context(
callers use the default ``False``.
defer_context_engine_notification: Delay the existing context-engine
hook until a manual host commits its outer history transaction.
commit_fence: Optional cooperative fence for executor callers that
may time out. It prevents a late worker from mutating session state
after its caller has moved on.
Returns:
``(compressed_messages, new_system_prompt)`` tuple. When
@ -1008,6 +1137,13 @@ def compress_context(
):
raise RuntimeError("a compression notification is already pending")
# ``conversation_history_after_compression()`` needs the latest attempt's
# outcome, while ``_last_compaction_in_place`` remains the run-level signal
# read by gateway callers. ``None`` means this attempt aborted or made no
# boundary, so the previous flush baseline remains authoritative.
agent._last_compression_attempt_recorded = True
agent._last_compression_attempt_in_place = None
_attempt_started_at = time.monotonic()
_attempt_id = uuid.uuid4().hex
_trigger_source = "manual" if force else "auto"
@ -1030,14 +1166,26 @@ def compress_context(
# the app server does not expose its native summary prompt, so there is no
# truthful injection point for ``on_pre_compress()`` return text here.
if getattr(agent, "api_mode", None) == "codex_app_server":
return _compress_context_via_codex_app_server(
agent,
messages,
system_message,
approx_tokens=approx_tokens,
task_id=task_id,
force=force,
)
_codex_fence_entered = False
if commit_fence is not None:
_codex_fence_entered = commit_fence.begin_commit()
if not _codex_fence_entered:
existing_prompt = getattr(agent, "_cached_system_prompt", None)
if not existing_prompt:
existing_prompt = agent._build_system_prompt(system_message)
return messages, existing_prompt
try:
return _compress_context_via_codex_app_server(
agent,
messages,
system_message,
approx_tokens=approx_tokens,
task_id=task_id,
force=force,
)
finally:
if _codex_fence_entered:
commit_fence.finish_commit()
# Every automatic entrypoint must honor compressor-owned cooldown and
# breaker state. Gateway hygiene constructs a fresh AIAgent, so the
@ -1089,7 +1237,20 @@ def compress_context(
f"{approx_tokens:,}" if approx_tokens else "unknown", agent.model,
focus_topic,
)
agent._emit_status(COMPACTION_STATUS)
_compaction_status = COMPACTION_STATUS
if not force:
_compaction_status = automatic_compaction_status_message(
agent.context_compressor,
phase="compress",
default_message=_compaction_status,
approx_tokens=approx_tokens,
message_count=_pre_msg_count,
model=agent.model,
focus_topic=focus_topic,
)
_compaction_status_emitted = bool(_compaction_status)
if _compaction_status:
agent._emit_status(_compaction_status)
_compaction_done_emitted = False
def _complete_compaction_lifecycle() -> None:
@ -1097,7 +1258,11 @@ def compress_context(
if _compaction_done_emitted:
return
_compaction_done_emitted = True
_emit_compaction_done(agent)
# A suppressed start (quiet context engine) opened no visible
# compaction phase — emit no terminal edge either. Failure warnings
# go through agent._emit_warning and are never suppressed here.
if _compaction_status_emitted:
_emit_compaction_done(agent)
# ── Compression lock ────────────────────────────────────────────────
# Atomic, state.db-backed lock per session_id. Without this, two
@ -1134,6 +1299,12 @@ def compress_context(
_try_acquire_lock = None
_lock_lookup_error: Optional[Exception] = None
_legacy_session_db_without_lock_api = False
# Clear any stale lock-skip signal from a prior call so this call's
# outcome alone determines what callers see. Without this an
# auto-compress lock-skip followed by a successful manual /compress
# would falsely report "Compression already in progress" and discard
# the compression results.
agent._compression_skipped_due_to_lock = None
if _lock_db is not None:
try:
_legacy_session_db_without_lock_api = _lock_api_is_absent_on_session_db(
@ -1221,6 +1392,11 @@ def compress_context(
_lock_sid, existing,
)
_lock_holder = None # don't release a lock we don't own
# Signal to callers that this no-op is due to a concurrent lock,
# not a genuine "nothing to compress" or aux-model failure.
# Manual /compress callers can surface a clear status message
# instead of the misleading "No changes from compression" text.
agent._compression_skipped_due_to_lock = existing or True
# Surface to the user once — quiet for downstream auto-compress loops
if getattr(agent, "_last_compression_lock_warning_sid", None) != _lock_sid:
agent._last_compression_lock_warning_sid = _lock_sid
@ -1394,6 +1570,7 @@ def compress_context(
if _activity_heartbeat is not None:
_activity_heartbeat.stop("context compression completed")
_commit_fence_entered = False
try:
# Capture boundary quality before session-rotation callbacks run. Built-in
# and plugin lifecycle hooks may reset per-session compressor fields while
@ -1481,6 +1658,28 @@ def compress_context(
_release_lock()
return messages, _existing_sp
if commit_fence is not None:
_commit_fence_entered = commit_fence.begin_commit()
if not _commit_fence_entered:
logger.info(
"Compression commit cancelled before session mutation "
"(session=%s).",
agent.session_id or "none",
)
agent._last_compaction_in_place = False
_existing_sp = getattr(agent, "_cached_system_prompt", None)
if not _existing_sp:
_existing_sp = agent._build_system_prompt(system_message)
_emit_compression_attempt_telemetry(
agent,
started_at=_attempt_started_at,
commit_status="aborted",
split_status="aborted",
failure_class="commit_fence_cancelled",
)
_release_lock()
return messages, _existing_sp
summary_error = getattr(agent.context_compressor, "_last_summary_error", None)
if summary_error:
if getattr(agent, "_last_compression_summary_warning", None) != summary_error:
@ -1509,11 +1708,54 @@ def compress_context(
todo_snapshot = agent._todo_store.format_for_injection()
if todo_snapshot:
compressed.append({
"role": "user",
"content": todo_snapshot,
"_todo_snapshot_synthetic": True,
})
# Fold the snapshot into a trailing REAL user message so
# compression never introduces a synthetic user/user pair. Any
# snapshot merged at an earlier boundary is stripped first so
# repeated compactions refresh rather than accumulate todo state
# (#26981). Scaffolding tails (continuation marker, summary
# handoff, a bare stale snapshot row) must never absorb the
# snapshot: merging would upgrade them to "real user" evidence
# and break zero-user provenance (#69292), so those keep the
# flagged standalone append and the real-user preservation pass
# continues to see todo scaffolding, not human intent.
from agent.context_compressor import _append_text_to_content
merged = False
_tail = (
compressed[-1]
if compressed and isinstance(compressed[-1], dict)
else None
)
if _tail is not None and _tail.get("role") == "user":
_stripped = _strip_stale_todo_snapshot(_tail.get("content"))
_probe = {
key: value for key, value in _tail.items() if key != "content"
}
_probe["content"] = _stripped
if _is_real_user_message(_probe):
_snapshot_text = (
f"\n\n{todo_snapshot}"
if isinstance(_stripped, str) and _stripped
else todo_snapshot
)
_tail["content"] = _append_text_to_content(
_stripped, _snapshot_text
)
merged = True
elif _stripped != _tail.get("content") and not _message_text(
{"role": "user", "content": _stripped}
).strip():
# The tail was nothing but an earlier snapshot row —
# refresh it in place instead of stacking a duplicate.
_tail["content"] = todo_snapshot
_tail["_todo_snapshot_synthetic"] = True
merged = True
if not merged:
compressed.append({
"role": "user",
"content": todo_snapshot,
"_todo_snapshot_synthetic": True,
})
_ensure_compressed_has_user_turn(messages, compressed)
cached_system_prompt = agent._cached_system_prompt
@ -1823,6 +2065,7 @@ def compress_context(
# via a rotation-independent flag. The gateway uses this — NOT an
# id-change diff — to re-baseline transcript handling (history_offset=0 +
# rewrite on the same id) when compaction happened in place. See #38763.
agent._last_compression_attempt_in_place = compacted_in_place
agent._last_compaction_in_place = compacted_in_place
# Keep the post-compression rough estimate for diagnostics, but do not
@ -1888,7 +2131,11 @@ def compress_context(
# file dedup) ran. A concurrent path that wakes up the moment we
# release will see the NEW session_id in state.db / SessionEntry and
# acquire on that — no race against our just-finished work.
_release_lock()
try:
_release_lock()
finally:
if _commit_fence_entered:
commit_fence.finish_commit()
def _compress_context_via_codex_app_server(

View file

@ -36,6 +36,7 @@ from agent.conversation_compression import (
PRE_API_COMPRESSION_STATUS_TEMPLATE,
conversation_history_after_compression,
)
from agent.context_engine import automatic_compaction_status_message
from agent.display import KawaiiSpinner
from agent.error_classifier import FailoverReason, classify_api_error
from agent.iteration_budget import IterationBudget
@ -710,6 +711,13 @@ def run_conversation(
except Exception:
pass
# The gateway caches agents across user turns. Compression state is
# per-turn: carrying a prior in-place boundary forward would make a later
# uncompressed result look like a compacted transcript to gateway writers.
agent._last_compaction_in_place = False
agent._last_compression_attempt_recorded = False
agent._last_compression_attempt_in_place = None
# ── Per-turn setup (the prologue) ──
# All once-per-turn setup — stdio guarding, retry-counter resets, user
# message sanitization, todo/nudge hydration, system-prompt restore-or-
@ -981,6 +989,13 @@ def run_conversation(
# outgoing copy.
_api_content = api_msg.pop("api_content", None)
# Display-only timeline metadata. Never a provider field — strip
# from every outgoing copy so strict OpenAI-compatible backends
# don't reject the request after a model switch or resumed typed
# event row enters the live history.
api_msg.pop("display_kind", None)
api_msg.pop("display_metadata", None)
# Inject ephemeral context into the current turn's user message.
# Sources: memory manager prefetch + plugin pre_llm_call hooks
# with target="user_message" (the default). Both are
@ -1300,6 +1315,15 @@ def run_conversation(
if _moa_prepared_request is not None:
pending_moa_prepared_request = _moa_prepared_request
compression_attempts += 1
# Compression is actually running (block cleared / was never
# blocked) — reset the blocked-overflow warning dedup so a future
# blocked-over-threshold turn can warn again. Mirrors the
# turn-context preflight reset (silent-overflow fix #62625).
# getattr guard: test doubles built via object.__new__ lack the
# method (gateway test-double pitfall) — treat absence as no-op.
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
if callable(_clear_warn):
_clear_warn()
logger.info(
"Pre-API compression: ~%s request tokens >= %s threshold "
"(context=%s, attempt=%s/%s)",
@ -1310,11 +1334,25 @@ def run_conversation(
compression_attempts,
max_compression_attempts,
)
agent._emit_status(
PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
_pre_api_status = automatic_compaction_status_message(
_compressor,
phase="pre_api",
default_message=PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
tokens=request_pressure_tokens
)
),
approx_tokens=request_pressure_tokens,
threshold_tokens=int(
getattr(_compressor, "threshold_tokens", 0) or 0
),
context_length=int(
getattr(_compressor, "context_length", 0) or 0
),
model=agent.model,
attempt=compression_attempts,
max_attempts=max_compression_attempts,
)
if _pre_api_status:
agent._emit_status(_pre_api_status)
_last_preflight_pressure = request_pressure_tokens
messages, active_system_prompt = agent._compress_context(
messages,
@ -1340,12 +1378,38 @@ def run_conversation(
# and preflight compaction sites; see
# conversation_history_after_compression().
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
api_call_count -= 1
agent._api_call_count = api_call_count
agent.iteration_budget.refund()
continue
elif (
agent.compression_enabled
and len(messages) > 1
and compression_attempts < max_compression_attempts
and not _defer_preflight(request_pressure_tokens)
and _compression_cooldown
):
# Blocked by the summary-LLM cooldown. Surface a deduped warning
# (only when actually over threshold — should_compress_info
# returns a None reason below threshold) so the user isn't left
# with a silently growing context. Mirrors the turn-context
# preflight and the loop-compaction guards (silent-overflow fix
# #62625).
_block_reason = None
try:
_block_reason = _compressor.should_compress_info(
request_pressure_tokens
)[1]
except Exception:
_block_reason = None
if _block_reason:
agent._warn_context_overflow_blocked(
_block_reason,
request_pressure_tokens,
int(getattr(_compressor, "threshold_tokens", 0) or 0),
)
# Thinking spinner for quiet mode (animated during API call)
thinking_spinner = None
@ -3555,7 +3619,7 @@ def run_conversation(
task_id=effective_task_id,
)
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
if len(messages) < original_len or old_ctx > _reduced_ctx:
agent._buffer_status(
@ -3810,7 +3874,7 @@ def run_conversation(
task_id=effective_task_id,
)
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
# Re-estimate tokens after compression. Same-message-count
@ -4051,7 +4115,7 @@ def run_conversation(
task_id=effective_task_id,
)
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
# Re-estimate tokens after compression. Same-message-count
@ -5442,6 +5506,15 @@ def run_conversation(
and _compressor.should_compress(_real_tokens)
):
compression_attempts += 1
# Compression is actually running (block cleared / was
# never blocked) — reset the blocked-overflow warning
# dedup so a future blocked-over-threshold turn can warn
# again (silent-overflow fix #62625).
# getattr guard: test doubles built via object.__new__ lack the
# method (gateway test-double pitfall) — treat absence as no-op.
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
if callable(_clear_warn):
_clear_warn()
agent._safe_print(" ⟳ compacting context…")
messages, active_system_prompt = agent._compress_context(
messages, system_message,
@ -5449,8 +5522,27 @@ def run_conversation(
task_id=effective_task_id,
)
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
elif agent.compression_enabled:
# Over threshold but compression is blocked (summary-LLM
# cooldown or anti-thrashing). Surface a deduped warning so
# the user isn't left with a silently growing context that
# eventually hits the hard provider limit. Mirrors the
# turn-context preflight guard (silent-overflow fix #62625).
_block_reason = None
_info = getattr(_compressor, "should_compress_info", None)
if _info is not None:
try:
_block_reason = _info(_real_tokens)[1]
except Exception:
_block_reason = None
if _block_reason:
agent._warn_context_overflow_blocked(
_block_reason,
_real_tokens,
int(getattr(_compressor, "threshold_tokens", 0) or 0),
)
# Save session log incrementally (so progress is visible even if interrupted)
agent._session_messages = messages

View file

@ -503,6 +503,10 @@ class CopilotACPClient:
def _run_prompt(self, prompt_text: str, *, timeout_seconds: float) -> tuple[str, str]:
try:
# Hide the console the CLI child would otherwise flash on Windows
# (#56747). Hide-only — stdio pipes stay intact for the ACP wire.
from hermes_cli._subprocess_compat import windows_hide_flags
proc = subprocess.Popen(
[self._acp_command] + self._acp_args,
stdin=subprocess.PIPE,
@ -512,6 +516,7 @@ class CopilotACPClient:
bufsize=1,
cwd=self._acp_cwd,
env=_build_subprocess_env(),
creationflags=windows_hide_flags(),
)
except FileNotFoundError as exc:
raise RuntimeError(

View file

@ -596,20 +596,32 @@ class CredentialPool:
self._last_no_entries_log_at: Optional[float] = None
def has_credentials(self) -> bool:
return bool(self._entries)
with self._lock:
return bool(self._entries)
def has_available(self) -> bool:
"""True if at least one entry is not currently in exhaustion cooldown."""
return bool(self._available_entries())
# ``_available_entries`` is not read-only: it prunes aged-out DEAD
# manual entries (rebinding ``self._entries``) and persists. It must
# run under ``self._lock`` like every other caller (``select`` etc.),
# otherwise a status probe here can race a concurrent ``select`` /
# rotation and tear ``self._entries`` or double-write auth.json.
with self._lock:
return bool(self._available_entries())
def entries(self) -> List[PooledCredential]:
return list(self._entries)
with self._lock:
return list(self._entries)
def current(self) -> Optional[PooledCredential]:
def _current_unlocked(self) -> Optional[PooledCredential]:
if not self._current_id:
return None
return next((entry for entry in self._entries if entry.id == self._current_id), None)
def current(self) -> Optional[PooledCredential]:
with self._lock:
return self._current_unlocked()
def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
"""Swap an entry in-place by id, preserving sort order."""
for idx, entry in enumerate(self._entries):
@ -652,6 +664,8 @@ class CredentialPool:
entry: PooledCredential,
status_code: Optional[int],
error_context: Optional[Dict[str, Any]] = None,
*,
persist: bool = True,
) -> PooledCredential:
normalized_error = _normalize_error_context(error_context)
# Permanent OAuth failures (token_invalidated, token_revoked, etc.)
@ -675,7 +689,8 @@ class CredentialPool:
last_error_reset_at=normalized_error.get("reset_at"),
)
self._replace_entry(entry, updated)
self._persist()
if persist:
self._persist()
return updated
def _sync_anthropic_entry_from_credentials_file(self, entry: PooledCredential) -> PooledCredential:
@ -1726,18 +1741,21 @@ class CredentialPool:
self._entries = [replace(candidate, priority=idx) for idx, candidate in enumerate(rotated)]
self._persist()
self._current_id = entry.id
return self.current() or entry
return self._current_unlocked() or entry
entry = available[0]
self._current_id = entry.id
return entry
def peek(self) -> Optional[PooledCredential]:
current = self.current()
if current is not None:
return current
available = self._available_entries()
return available[0] if available else None
# Single lock acquisition for the whole read; call the unlocked
# helpers so we don't re-enter the non-reentrant ``self._lock``.
with self._lock:
current = self._current_unlocked()
if current is not None:
return current
available = self._available_entries()
return available[0] if available else None
def mark_exhausted_and_rotate(
self,
@ -1757,12 +1775,49 @@ class CredentialPool:
(e for e in self._entries if e.runtime_api_key == api_key_hint),
None,
)
if entry is None:
# The failed key is identifiable but matches no entry
# (rotated away, or a wrapper whose runtime key differs).
# Falling through to current()/_select_unlocked() would
# mark an INNOCENT healthy key exhausted for the full
# cooldown TTL. Don't guess — just hand back a fresh
# selection so the caller can retry.
logger.info(
"credential pool: failed key hint matched no %s entry; "
"rotating without marking any credential exhausted",
self.provider,
)
self._current_id = None
return self._select_unlocked()
if entry is None:
entry = self.current() or self._select_unlocked()
entry = self._current_unlocked() or self._select_unlocked()
if entry is None:
return None
_label = entry.label or entry.id[:8]
self._mark_exhausted(entry, status_code, error_context)
# A 402/429/401 is an API-keylevel failure: the account is out of
# balance, rate-limited, or its key is rejected. The same key can
# back more than one pool entry (e.g. an explicit pool entry plus a
# ``model_config`` entry auto-seeded from ``model.api_key`` — both
# carry the identical ``runtime_api_key``). Marking only the first
# match leaves the sibling entries OK, so ``_select_unlocked()``
# keeps handing back the same depleted key and rotation never
# converges — the caller ``continue``s forever until the client
# disconnects (a ~2.5min hang with no error surfaced to the user).
# Mark every entry sharing the failed key so the pool can reach the
# "no available entries" state and let the error propagate.
if api_key_hint:
siblings_marked = False
for sibling in self._entries:
if sibling.id == entry.id:
continue
if sibling.runtime_api_key == api_key_hint:
self._mark_exhausted(
sibling, status_code, error_context, persist=False
)
siblings_marked = True
if siblings_marked:
self._persist()
# Re-read the updated entry to log the correct terminal state.
updated_entry = next(
(e for e in self._entries if e.id == entry.id), entry,
@ -1852,14 +1907,14 @@ class CredentialPool:
None,
)
else:
entry = self.current() or self._select_unlocked(refresh=False)
entry = self._current_unlocked() or self._select_unlocked(refresh=False)
if entry is None:
return None
self._current_id = entry.id
return self._try_refresh_current_unlocked()
def _try_refresh_current_unlocked(self) -> Optional[PooledCredential]:
entry = self.current()
entry = self._current_unlocked()
if entry is None:
return None
refreshed = self._refresh_entry(entry, force=True)
@ -1868,76 +1923,80 @@ class CredentialPool:
return refreshed
def reset_statuses(self) -> int:
count = 0
new_entries = []
for entry in self._entries:
if entry.last_status or entry.last_status_at or entry.last_error_code:
new_entries.append(
replace(
entry,
last_status=None,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
with self._lock:
count = 0
new_entries = []
for entry in self._entries:
if entry.last_status or entry.last_status_at or entry.last_error_code:
new_entries.append(
replace(
entry,
last_status=None,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
)
)
)
count += 1
else:
new_entries.append(entry)
if count:
self._entries = new_entries
self._persist()
return count
count += 1
else:
new_entries.append(entry)
if count:
self._entries = new_entries
self._persist()
return count
def remove_index(self, index: int) -> Optional[PooledCredential]:
if index < 1 or index > len(self._entries):
return None
removed = self._entries.pop(index - 1)
self._entries = [
replace(entry, priority=new_priority)
for new_priority, entry in enumerate(self._entries)
]
write_credential_pool(
self.provider,
[entry.to_dict() for entry in self._entries],
removed_ids=[removed.id],
)
if self._current_id == removed.id:
self._current_id = None
return removed
with self._lock:
if index < 1 or index > len(self._entries):
return None
removed = self._entries.pop(index - 1)
self._entries = [
replace(entry, priority=new_priority)
for new_priority, entry in enumerate(self._entries)
]
write_credential_pool(
self.provider,
[entry.to_dict() for entry in self._entries],
removed_ids=[removed.id],
)
if self._current_id == removed.id:
self._current_id = None
return removed
def resolve_target(self, target: Any) -> Tuple[Optional[int], Optional[PooledCredential], Optional[str]]:
raw = str(target or "").strip()
if not raw:
return None, None, "No credential target provided."
for idx, entry in enumerate(self._entries, start=1):
if entry.id == raw:
return idx, entry, None
with self._lock:
for idx, entry in enumerate(self._entries, start=1):
if entry.id == raw:
return idx, entry, None
label_matches = [
(idx, entry)
for idx, entry in enumerate(self._entries, start=1)
if entry.label.strip().lower() == raw.lower()
]
if len(label_matches) == 1:
return label_matches[0][0], label_matches[0][1], None
if len(label_matches) > 1:
return None, None, f'Ambiguous credential label "{raw}". Use the numeric index or entry id instead.'
if raw.isdigit():
index = int(raw)
if 1 <= index <= len(self._entries):
return index, self._entries[index - 1], None
return None, None, f"No credential #{index}."
return None, None, f'No credential matching "{raw}".'
label_matches = [
(idx, entry)
for idx, entry in enumerate(self._entries, start=1)
if entry.label.strip().lower() == raw.lower()
]
if len(label_matches) == 1:
return label_matches[0][0], label_matches[0][1], None
if len(label_matches) > 1:
return None, None, f'Ambiguous credential label "{raw}". Use the numeric index or entry id instead.'
if raw.isdigit():
index = int(raw)
if 1 <= index <= len(self._entries):
return index, self._entries[index - 1], None
return None, None, f"No credential #{index}."
return None, None, f'No credential matching "{raw}".'
def add_entry(self, entry: PooledCredential) -> PooledCredential:
entry = replace(entry, priority=_next_priority(self._entries))
self._entries.append(entry)
self._persist()
return entry
with self._lock:
entry = replace(entry, priority=_next_priority(self._entries))
self._entries.append(entry)
self._persist()
return entry
def _upsert_entry(entries: List[PooledCredential], provider: str, source: str, payload: Dict[str, Any]) -> bool:

View file

@ -0,0 +1,85 @@
"""Context-local state for delegate_task child execution.
The parent Hermes process may itself be a Kanban dispatcher worker with
HERMES_KANBAN_* variables in process env. delegate_task children run inside the
same Python process, but they are not dispatcher-owned Kanban workers. This
module lets code paths that resolve tool schemas or spawn subprocesses fail
closed for delegated children without mutating global os.environ for the parent.
"""
from __future__ import annotations
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Iterator, Mapping, MutableMapping
_DELEGATED_CHILD_CONTEXT: ContextVar[bool] = ContextVar(
"hermes_delegated_child_context",
default=False,
)
DELEGATED_CHILD_ENV_MARKER = "HERMES_DELEGATED_CHILD_CONTEXT"
KANBAN_ENV_KEYS: tuple[str, ...] = (
"HERMES_KANBAN_TASK",
"HERMES_KANBAN_RUN_ID",
"HERMES_KANBAN_WORKSPACE",
"HERMES_KANBAN_WORKSPACES_ROOT",
"HERMES_KANBAN_CLAIM_LOCK",
"HERMES_KANBAN_BOARD",
"HERMES_KANBAN_DB",
)
@contextmanager
def delegated_child_context() -> Iterator[None]:
"""Mark the current execution context as a delegate_task child."""
token = _DELEGATED_CHILD_CONTEXT.set(True)
try:
yield
finally:
_DELEGATED_CHILD_CONTEXT.reset(token)
def is_delegated_child_context() -> bool:
"""Return True while code is running for a delegate_task child."""
return bool(_DELEGATED_CHILD_CONTEXT.get())
def is_delegated_child_process_context() -> bool:
"""Return True in this process or a subprocess spawned by a child."""
import os
return bool(_DELEGATED_CHILD_CONTEXT.get()) or bool(
os.environ.get(DELEGATED_CHILD_ENV_MARKER)
)
def scrub_kanban_env(env: Mapping[str, str] | MutableMapping[str, str]) -> dict[str, str]:
"""Return *env* with dispatcher-only Kanban variables removed."""
cleaned = dict(env)
for key in KANBAN_ENV_KEYS:
cleaned.pop(key, None)
cleaned[DELEGATED_CHILD_ENV_MARKER] = "1"
return cleaned
def delegated_child_subprocess_env(
env: Mapping[str, str] | MutableMapping[str, str] | None = None,
) -> dict[str, str] | None:
"""Return an env override only when delegated-child lineage must cross fork.
Most subprocess call sites historically used ``env=None`` to inherit the
process environment. In a ``delegate_task`` child, inheriting as-is leaks
parent dispatcher ``HERMES_KANBAN_*`` vars while losing the ContextVar in
the new process. This helper preserves normal ``env=None`` semantics for
non-delegated calls, and only materializes a scrubbed env when the lineage
marker must be propagated across a child-process boundary.
"""
if not is_delegated_child_process_context():
return None if env is None else dict(env)
if env is None:
import os
env = os.environ
return scrub_kanban_env(env)

View file

@ -413,6 +413,7 @@ _CONTENT_POLICY_BLOCKED_PATTERNS = [
_AUTH_PATTERNS = [
"invalid api key",
"invalid_api_key",
"gateway_auth_failed",
"authentication",
"unauthorized",
"forbidden",

View file

@ -194,6 +194,10 @@ def _supports_vision_override(
and/or the user-declared name under ``model.provider``; all are
tried. For ``custom:<name>`` syntax, the stripped ``<name>`` is also
tried as a provider key.)
2b. ``custom_providers`` (legacy list form) ``.models.<model>``
Under (2) and (2b), the per-model capability key may be written as
either ``supports_vision`` or the shorter ``vision`` alias; both work.
Returns None when no override is set, so the caller falls through to
models.dev. Returns False explicitly only when the user wrote a
@ -234,7 +238,9 @@ def _supports_vision_override(
models_cfg: Dict[str, Any] = models_raw if isinstance(models_raw, dict) else {}
per_model_raw = models_cfg.get(model)
per_model: Dict[str, Any] = per_model_raw if isinstance(per_model_raw, dict) else {}
coerced = _coerce_capability_bool(per_model.get("supports_vision"))
coerced = _coerce_capability_bool(
per_model.get("supports_vision", per_model.get("vision"))
)
if coerced is not None:
return coerced
@ -258,7 +264,9 @@ def _supports_vision_override(
models_cfg = models_raw if isinstance(models_raw, dict) else {}
per_model_raw = models_cfg.get(model)
per_model = per_model_raw if isinstance(per_model_raw, dict) else {}
coerced = _coerce_capability_bool(per_model.get("supports_vision"))
coerced = _coerce_capability_bool(
per_model.get("supports_vision", per_model.get("vision"))
)
if coerced is not None:
return coerced

View file

@ -7,6 +7,36 @@ from typing import Any, Sequence
from agent.redact import redact_sensitive_text
def describe_compression_lock_skip(lock_signal: Any) -> str:
"""User-facing text for a manual /compress skipped by the compression lock.
``lock_signal`` is ``agent._compression_skipped_due_to_lock`` (or the
``holder`` carried by the TUI's ``CompressionLockHeld``): a descriptive
holder string when another compressor CONFIRMED holds the lock, or
``True``/``None`` when acquisition failed without a confirmed holder
(``hermes_state.try_acquire_compression_lock`` catches ``sqlite3.Error``
internally and returns ``False``, so a failed acquire is NOT proof that
another compression is running). The two cases must be worded
differently: claiming "already in progress" on an unconfirmed failure
misdirects the user when the real problem is a broken lock subsystem.
"""
holder = (
lock_signal
if isinstance(lock_signal, str) and lock_signal.strip()
else None
)
if holder:
return (
f"⏳ Compression already in progress for this session "
f"(holder: {holder}). Please wait for it to finish."
)
return (
"⏳ Compression skipped: could not acquire this session's "
"compression lock. Another compression may still be running, or "
"the lock check failed — try again shortly."
)
def summarize_manual_compression(
before_messages: Sequence[dict[str, Any]],
after_messages: Sequence[dict[str, Any]],

View file

@ -212,11 +212,30 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]:
out["api_key"] = rt["api_key"]
if rt.get("api_mode"):
out["api_mode"] = rt["api_mode"]
request_overrides = rt.get("request_overrides")
if isinstance(request_overrides, dict):
extra_body = request_overrides.get("extra_body")
if isinstance(extra_body, dict) and extra_body:
out["extra_body"] = dict(extra_body)
except Exception as exc: # pragma: no cover - defensive
logger.debug("MoA slot runtime resolution failed for %s: %s", _slot_label(slot), exc)
return out
def _merge_slot_extra_body(
slot_extra_body: Any,
caller_extra_body: Any,
) -> Any:
"""Merge slot defaults with a caller override for ``call_llm``."""
if isinstance(slot_extra_body, dict) and slot_extra_body:
if isinstance(caller_extra_body, dict):
return {**slot_extra_body, **caller_extra_body}
if caller_extra_body:
return caller_extra_body
return dict(slot_extra_body)
return caller_extra_body
def _maybe_apply_moa_cache_control(
messages: list[dict[str, Any]],
runtime: dict[str, Any],
@ -976,18 +995,26 @@ class MoAChatCompletions:
# actually governs the aggregator stream, not just call_llm's default.
if api_kwargs.get("timeout") is not None:
stream_kwargs["timeout"] = api_kwargs["timeout"]
agg_runtime = _slot_runtime(aggregator)
# _slot_runtime may carry the provider's request_overrides.extra_body;
# pop it and merge with the caller's extra_body (caller wins) so the
# explicit kwarg below never collides with **agg_runtime.
agg_extra_body = _merge_slot_extra_body(
agg_runtime.pop("extra_body", None),
extra_body,
)
_agg_response = call_llm(
task="moa_aggregator",
messages=agg_messages,
temperature=aggregator_temperature,
max_tokens=max_tokens,
tools=tools,
extra_body=extra_body,
extra_body=agg_extra_body,
# Prepared requests must retain the acting aggregator's reasoning
# policy exactly as the direct create() path does (#64187).
reasoning_config=_aggregator_reasoning_config(aggregator),
**stream_kwargs,
**_slot_runtime(aggregator),
**agg_runtime,
)
# Non-streaming path (quiet mode / eval / subagents): the aggregator
# output is available inline, so capture it into the pending trace now.

View file

@ -2201,11 +2201,18 @@ def get_model_context_length(
# acting context, so they're ignored here.
if (provider or "").strip().lower() == "moa":
try:
from hermes_cli.config import load_config
from hermes_cli.config import (
get_compatible_custom_providers,
load_config,
)
from hermes_cli.moa_config import resolve_moa_preset
from hermes_cli.runtime_provider import resolve_runtime_provider
preset = resolve_moa_preset(load_config().get("moa") or {}, model)
config = load_config()
effective_custom_providers = custom_providers
if effective_custom_providers is None:
effective_custom_providers = get_compatible_custom_providers(config)
preset = resolve_moa_preset(config.get("moa") or {}, model)
agg = preset.get("aggregator") or {}
agg_provider = str(agg.get("provider") or "").strip()
agg_model = str(agg.get("model") or "").strip()
@ -2215,7 +2222,8 @@ def get_model_context_length(
agg_model,
base_url=rt.get("base_url", "") or "",
api_key=rt.get("api_key", "") or "",
provider=agg_provider,
provider=rt.get("provider") or agg_provider,
custom_providers=effective_custom_providers,
)
except Exception:
logger.debug("MoA aggregator context-length resolution failed", exc_info=True)

View file

@ -884,7 +884,14 @@ PLATFORM_HINTS = {
"You're responding through an API server. The rendering layer is unknown — "
"assume plain text. No markdown formatting (no asterisks, bullets, headers, "
"code fences). Treat this like a conversation, not a document. Keep responses "
"brief and natural."
"brief and natural. "
"File/media delivery: images referenced as MEDIA:/absolute/path tags "
"(.png/.jpg/.jpeg/.gif/.webp/.bmp, up to 5MB) are inlined as base64 data "
"URLs in responses on the chat, completions, and responses endpoints. "
"Non-image files are NOT intercepted anywhere, and the runs endpoint "
"intercepts nothing — a MEDIA: tag there renders as literal text exposing "
"a raw host filesystem path. For those cases, state the plain file path "
"in your response text instead of a MEDIA: tag."
),
"webui": (
"You are in the Hermes WebUI, a browser-based chat interface. "

View file

@ -127,6 +127,10 @@ class CodexAppServerClient:
# Codex emits tracing to stderr; default WARN keeps it quiet for users.
spawn_env.setdefault("RUST_LOG", "warn")
# Hide the console the codex child would otherwise flash on Windows
# (#56747). Hide-only — stdio pipes stay intact for the app-server wire.
from hermes_cli._subprocess_compat import windows_hide_flags
self._proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
@ -134,6 +138,7 @@ class CodexAppServerClient:
stderr=subprocess.PIPE,
bufsize=0,
env=spawn_env,
creationflags=windows_hide_flags(),
)
self._next_id = 1
self._pending: dict[int, _Pending] = {}

View file

@ -36,6 +36,7 @@ from agent.conversation_compression import (
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
conversation_history_after_compression,
)
from agent.context_engine import automatic_compaction_status_message
from agent.iteration_budget import IterationBudget
from agent.memory_manager import build_memory_context_block
from agent.model_metadata import (
@ -668,11 +669,18 @@ def build_turn_context(
f"{_idle_floor:,}",
agent.session_id or "none",
)
agent._emit_status(
IDLE_COMPACTION_STATUS_TEMPLATE.format(
_idle_status = automatic_compaction_status_message(
_compressor,
phase="idle",
default_message=IDLE_COMPACTION_STATUS_TEMPLATE.format(
idle_seconds=int(_idle_gap), tokens=_idle_tokens
)
),
approx_tokens=_idle_tokens,
idle_seconds=int(_idle_gap),
model=agent.model,
)
if _idle_status:
agent._emit_status(_idle_status)
_idle_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message, approx_tokens=_idle_tokens,
@ -686,7 +694,7 @@ def build_turn_context(
# untouched.
if messages is not _idle_input:
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
# Compaction rebuilt the list, so the index of this turn's
# just-appended user message is stale — re-anchor it the
@ -747,6 +755,8 @@ def build_turn_context(
lambda: None,
)()
_should_compress_now = False
_compress_block_reason = None
if _preflight_deferred:
logger.info(
"Skipping preflight compression: rough estimate ~%s >= %s, "
@ -762,14 +772,42 @@ def build_turn_context(
int(_compression_cooldown.get("remaining_seconds", 0.0)),
agent.session_id or "none",
)
if _preflight_tokens >= _compressor.threshold_tokens:
# Context is over threshold but compression is blocked by the
# summary-LLM cooldown — surface a warning (see block below).
_cooldown_secs = _compression_cooldown.get("remaining_seconds", 0.0)
_compress_block_reason = f"cooldown:{_cooldown_secs:.0f}"
elif _codex_native_auto:
logger.info(
"Skipping Hermes preflight compression for codex app-server "
"(mode=%s); Hermes will not start thread compaction here.",
getattr(agent, "codex_app_server_auto_compaction", "native"),
)
elif _compressor.should_compress(_preflight_tokens):
else:
_should_compress_now = _compressor.should_compress(_preflight_tokens)
if not _should_compress_now:
# Context is over threshold but compression is blocked
# (summary-LLM cooldown or anti-thrashing). Ask should_compress_info
# for the human-readable reason so we can surface a warning below.
# getattr guard: minimal compressor doubles (SimpleNamespace in
# the engine-preflight tests) and older plugin engines lack the
# method — absence means no block reason, no warning.
_info = getattr(_compressor, "should_compress_info", None)
if callable(_info):
try:
_compress_block_reason = _info(_preflight_tokens)[1]
except Exception:
_compress_block_reason = None
if _should_compress_now:
_preflight_compressed = True
# Compression is actually running (block cleared / was never
# blocked) — reset the dedup so a future blocked-over-threshold
# turn can warn again. Real session boundary.
# getattr guard: test doubles built via object.__new__ lack the
# method (gateway test-double pitfall) — treat absence as no-op.
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
if callable(_clear_warn):
_clear_warn()
logger.info(
"Preflight compression: ~%s tokens >= %s threshold (model %s, ctx %s)",
f"{_preflight_tokens:,}",
@ -777,12 +815,20 @@ def build_turn_context(
agent.model,
f"{_compressor.context_length:,}",
)
agent._emit_status(
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(
_preflight_status = automatic_compaction_status_message(
_compressor,
phase="preflight",
default_message=PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(
tokens=_preflight_tokens,
threshold=_compressor.threshold_tokens,
)
),
approx_tokens=_preflight_tokens,
threshold_tokens=_compressor.threshold_tokens,
context_length=_compressor.context_length,
model=agent.model,
)
if _preflight_status:
agent._emit_status(_preflight_status)
# Preflight passes honor the same configured per-turn cap
# (compression.max_attempts) as the loop's compression sites;
# default 3 preserves the prior hardcoded behavior.
@ -811,7 +857,7 @@ def build_turn_context(
_preflight_compression_blocked = True
break # Cannot compress further: neither rows nor tokens moved
conversation_history = conversation_history_after_compression(
agent, messages
agent, messages, conversation_history
)
agent._empty_content_retries = 0
agent._thinking_prefill_retries = 0
@ -833,6 +879,100 @@ def build_turn_context(
f"{_preflight_tokens:,}",
)
break
elif _compress_block_reason:
# Context is already over the compression threshold, but compression
# is blocked (summary LLM cooldown or anti-thrashing). Without a
# signal the session keeps growing until the model silently stops
# answering — the conversation hits the hard provider token limit
# with no explanation. Surface a deduped warning so the user can
# take action (/new or /compress) instead of hitting a silent hang.
agent._warn_context_overflow_blocked(
_compress_block_reason,
_preflight_tokens,
_compressor.threshold_tokens,
)
else:
# Sub-threshold and unblocked — allow the overflow warning to fire
# again next time the context is over threshold but blocked.
# getattr guard: test doubles built via object.__new__ lack the
# method (gateway test-double pitfall) — treat absence as no-op.
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
if callable(_clear_warn):
_clear_warn()
# Engine maintenance only when NO skip-branch fired: a failure
# cooldown, deferred estimate, or codex-native route must keep
# the engine hook un-consulted (#20316 contract — the cooldown
# exists precisely because compression recently failed).
if _compression_cooldown or _preflight_deferred or _codex_native_auto:
_engine_preflight = None
else:
_engine_preflight = getattr(
_compressor, "should_compress_preflight", None
)
# ── Engine-driven sub-threshold preflight maintenance (#20316) ──
# None of the threshold-path branches fired (not deferred, no
# failure cooldown, not codex-native, and should_compress() said
# the request is under pressure). Context engines that override
# ``should_compress_preflight()`` (e.g. LCM-style incremental
# leaf-chunk compaction) can still request deferred maintenance
# below the token threshold. The default
# ``ContextEngine.should_compress_preflight()`` returns False, so
# the built-in ``ContextCompressor`` path is byte-identical.
#
# Attempt-cap integration: the engine gets exactly ONE
# ``compress()`` pass per turn. It is mutually exclusive with the
# threshold multi-pass loop above (if/elif), so turn-start
# preflight passes stay bounded by the resolved
# ``compression.max_attempts`` cap (floor 1) in every case.
#
# No-op-blocking integration: a sub-threshold engine pass that
# no-ops says nothing about over-threshold compressibility, so it
# must neither set nor clear ``_preflight_compression_blocked``
# (#64382) — and being in the ``else`` arm it can never run after
# the threshold loop has proven a retry ineffective.
# (resolved above, gated on no skip-branch having fired)
_wants_engine_preflight = False
if callable(_engine_preflight):
try:
_wants_engine_preflight = bool(_engine_preflight(messages))
except Exception as _preflight_exc:
# A buggy engine must never break an otherwise-healthy
# turn: swallow at debug level and skip maintenance.
logger.debug(
"should_compress_preflight raised %s; skipping "
"engine-driven preflight maintenance",
_preflight_exc,
)
_wants_engine_preflight = False
if _wants_engine_preflight:
logger.info(
"Engine-driven preflight maintenance: %s requested "
"compress() at ~%s tokens (below %s threshold)",
getattr(_compressor, "name", type(_compressor).__name__),
f"{_preflight_tokens:,}",
f"{getattr(_compressor, 'threshold_tokens', 0):,}",
)
_engine_input = messages
messages, active_system_prompt = agent._compress_context(
messages, system_message, approx_tokens=_preflight_tokens,
task_id=effective_task_id,
)
# ``_compress_context`` returns the INPUT list object on every
# skip path (per-session lock held elsewhere, cooldown,
# anti-thrash breaker, codex-native routing) and an engine may
# legitimately no-op. Only re-baseline the flush history and
# re-anchor the user row after a REAL compaction — a skip must
# leave the turn's bookkeeping untouched.
if messages is not _engine_input:
_preflight_compressed = True
conversation_history = conversation_history_after_compression(
agent, messages
)
agent._empty_content_retries = 0
agent._thinking_prefill_retries = 0
agent._last_content_with_tools = None
agent._last_content_tools_all_housekeeping = False
agent._mute_post_response = False
if _preflight_compressed:
# Compression rebuilt the list (tail messages are fresh compaction

View file

@ -0,0 +1,147 @@
/**
* E2E regression: desktop resume must hide agent-only transcript rows.
*
* Compaction handoffs are active user rows because the model needs them for
* context continuity. They are not authored chat content, so the desktop
* transcript must never display them after a real compressor-generated resume.
*/
import * as fs from 'node:fs'
import * as path from 'node:path'
import { expect, test } from './test'
import {
type MockBackendFixture,
buildAppEnv,
createSandbox,
launchDesktop,
waitForAppReady,
writeEnvFile,
writeMockProviderConfig,
} from './fixtures'
import {
MOCK_REPLY,
startMockServer,
VERIFICATION_STOP_TEXT,
VERIFICATION_STOP_TRIGGER,
} from './mock-server'
import { RealSessionBuilder } from './real-session-builder'
const SESSION_TITLE = 'E2E Hidden History Messages'
const VISIBLE_USER_TEXT = 'E2E_VISIBLE_USER_HISTORY'
const VISIBLE_POST_COMPACTION_TEXT = 'E2E_VISIBLE_POST_COMPACTION_HISTORY'
const COMPACTION_TRIGGER_PADDING = ' force real context compression'.repeat(600)
async function setupSeededMockBackend(): Promise<MockBackendFixture> {
const mock = await startMockServer()
const sandbox = createSandbox('hidden-history')
writeMockProviderConfig(sandbox.hermesHome, mock.url)
fs.appendFileSync(
path.join(sandbox.hermesHome, 'config.yaml'),
'\ncompression:\n threshold_tokens: 1\n',
'utf8',
)
writeEnvFile(sandbox.hermesHome)
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
try {
await builder.createSession({
title: SESSION_TITLE,
turns: [
`${VISIBLE_USER_TEXT}${COMPACTION_TRIGGER_PADDING}`,
VISIBLE_POST_COMPACTION_TEXT,
],
})
} finally {
await builder.close()
}
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
return {
app,
page,
mock,
mockUrl: mock.url,
sandbox,
cleanup: async () => {
await app.close().catch(() => undefined)
await mock.close()
sandbox.cleanup()
},
}
}
test('resume hides real context-compaction handoffs', async ({}, testInfo) => {
const fixture = await setupSeededMockBackend()
try {
const { page } = fixture
await waitForAppReady(fixture, 120_000)
const sessionRow = page
.locator('[data-slot="sidebar"] button')
.filter({ hasText: SESSION_TITLE })
.first()
await sessionRow.click()
const transcript = page.locator('[data-slot="aui_thread-viewport"]')
await expect(transcript).toContainText(VISIBLE_USER_TEXT)
await expect(transcript).toContainText(VISIBLE_POST_COMPACTION_TEXT)
await expect(transcript).toContainText(MOCK_REPLY)
await expect(transcript).not.toContainText('[CONTEXT COMPACTION — REFERENCE ONLY]')
await page.screenshot({ path: testInfo.outputPath('hidden-history-resume.png') })
} finally {
await fixture.cleanup()
}
})
test('live verify-on-stop continuations stay out of the transcript', async ({}, testInfo) => {
const sandbox = createSandbox('live-verification-nudge')
const projectRoot = path.join(sandbox.root, 'project')
const changedFile = path.join(projectRoot, 'e2e-verification-target.py')
fs.mkdirSync(projectRoot)
fs.writeFileSync(
path.join(projectRoot, 'pyproject.toml'),
'[project]\nname = "e2e-verification-project"\nversion = "0.0.0"\n',
'utf8',
)
const mock = await startMockServer({ verificationWritePath: changedFile })
writeMockProviderConfig(sandbox.hermesHome, mock.url)
fs.appendFileSync(path.join(sandbox.hermesHome, 'config.yaml'), '\nagent:\n verify_on_stop: true\n', 'utf8')
writeEnvFile(sandbox.hermesHome)
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
const fixture: MockBackendFixture = {
app,
page,
mock,
mockUrl: mock.url,
sandbox,
cleanup: async () => {
await app.close().catch(() => undefined)
await mock.close()
sandbox.cleanup()
},
}
try {
await waitForAppReady(fixture, 120_000)
const composer = page.locator('[contenteditable="true"]').first()
await composer.click()
await composer.type(VERIFICATION_STOP_TRIGGER)
await page.keyboard.press('Enter')
const transcript = page.locator('[data-slot="aui_thread-viewport"]')
await expect(transcript).toContainText(VERIFICATION_STOP_TEXT, { timeout: 60_000 })
await expect.poll(
() => mock.receivedPrompts.some(prompt => prompt.includes('[System: You edited code in this turn')),
{ timeout: 30_000 },
).toBe(true)
expect(fs.existsSync(changedFile), 'The scripted write_file call should edit only the sandbox project').toBe(true)
await expect(transcript).not.toContainText('[System: You edited code in this turn')
await page.screenshot({ path: testInfo.outputPath('live-verification-nudge.png') })
} finally {
await fixture.cleanup()
}
})

View file

@ -1,4 +1,3 @@
import { spawnSync } from 'node:child_process'
import * as path from 'node:path'
import { type TestInfo } from '@playwright/test'
@ -15,13 +14,16 @@ import {
writeMockProviderConfig,
} from './fixtures'
import { MOCK_REPLY, startMockServer, type MockServer, type MockServerOptions } from './mock-server'
import { RealSessionBuilder } from './real-session-builder'
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
const SEED_SCRIPT = path.resolve(import.meta.dirname, 'scripts', 'seed_large_session.py')
const SESSION_TITLE = 'E2E large persisted session'
const EXPECTED_TEXT = 'E2E persisted user message 52'
const BACKGROUND_PROMPT = 'E2E background inference must remain attached across resume'
const HISTORY_TURNS = Array.from(
{ length: 27 },
(_, index) => `E2E persisted user message ${index * 2}: audit the compatibility matrix`,
)
interface SeededFixture {
app: ElectronApplication
@ -43,13 +45,11 @@ async function setupSeededDesktop(mockServer?: MockServerOptions): Promise<Seede
writeMockProviderConfig(sandbox.hermesHome, mock.url)
writeEnvFile(sandbox.hermesHome)
const seeded = spawnSync('python3', [SEED_SCRIPT, path.join(sandbox.hermesHome, 'state.db')], {
cwd: REPO_ROOT,
encoding: 'utf8',
env: { ...process.env, PYTHONPATH: REPO_ROOT },
})
if (seeded.status !== 0) {
throw new Error(`large-session seed failed:\n${seeded.stdout}\n${seeded.stderr}`)
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
try {
await builder.createSession({ title: SESSION_TITLE, turns: HISTORY_TURNS })
} finally {
await builder.close()
}
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
@ -210,6 +210,7 @@ test.describe('large session resume', () => {
await waitForAppReady(fixture, 120_000)
await openSeededSession(fixture.page)
const initialMockReplyCount = await textNodeOccurrences(fixture.page, MOCK_REPLY)
await submitPrompt(fixture.page, BACKGROUND_PROMPT)
await fixture.mock.waitForHeldStream()
await openNewSession(fixture.page)
@ -229,7 +230,10 @@ test.describe('large session resume', () => {
await fixture.page.screenshot({ path: testInfo.outputPath(`${resumeKind}-background-inference-resume.png`), fullPage: false })
expect(await textNodeOccurrences(fixture.page, BACKGROUND_PROMPT), 'the running user prompt should appear once').toBe(1)
expect(await textNodeOccurrences(fixture.page, MOCK_REPLY), 'the completed assistant reply should appear once').toBe(1)
expect(
await textNodeOccurrences(fixture.page, MOCK_REPLY),
'the completed assistant reply should add exactly one transcript row',
).toBe(initialMockReplyCount + 1)
})
}
})

View file

@ -23,8 +23,10 @@ export const MOCK_REPLY = 'Hello from the mock inference server! The full boot c
export interface MockServerOptions {
/** Pause the matching stream after its first token for session-switch E2E coverage. */
holdFirstStreamForPrompt?: string
/** Pause the first completion whose request JSON contains this text. */
holdFirstCompletionContaining?: string
/** Pause the first completion whose request JSON contains this text. */
holdFirstCompletionContaining?: string
/** Absolute sandbox path written by the verify-on-stop scripted tool call. */
verificationWritePath?: string
}
export interface MockServer {
@ -104,6 +106,9 @@ let _queueStopIndex = 0
/** Per-server counter for the correction/session-switch script. */
let _correctionSwitchIndex = 0
/** Per-server counter for the verify-on-stop script. */
let _verificationStopIndex = 0
/** User messages received by the mock, for E2E assertions on real submits. */
const _receivedUserTexts: string[] = []
@ -114,6 +119,7 @@ function resetScriptIndex(): void {
_sidebarCrossIndex = 0
_queueStopIndex = 0
_correctionSwitchIndex = 0
_verificationStopIndex = 0
_receivedUserTexts.length = 0
}
@ -214,6 +220,32 @@ const CORRECTION_SWITCH_SCRIPT: ScriptedTurn[] = [
export const CORRECTION_SWITCH_TRIGGER = 'E2E_CORRECTION_SWITCH_TRIGGER'
/**
* Drives a real code edit followed by two finish attempts. Hermes should add
* its synthetic verify-on-stop continuation after each finish attempt until
* the bounded verifier gives up. The mock's request capture proves the nudge
* reached the model; desktop must never render it as chat content.
*/
function verificationStopScript(writePath: string): ScriptedTurn[] {
return [
{
text: 'I will make the requested code change.',
toolCalls: [{
name: 'write_file',
args: {
path: writePath,
content: 'def changed_by_e2e():\n return "changed"\n',
},
}],
},
{ text: 'The code edit is complete.' },
{ text: 'I cannot provide fresh verification evidence for that edit.' },
]
}
export const VERIFICATION_STOP_TRIGGER = 'E2E_VERIFY_ON_STOP_TRIGGER'
export const VERIFICATION_STOP_TEXT = 'I cannot provide fresh verification evidence for that edit.'
/**
* A marker that makes the mock emit a real blocking clarify tool call. Tests
* use it to hold a turn open while exercising busy-composer interactions.
@ -340,6 +372,9 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
const isSidebarTrigger = userText.includes('E2E_SIDEBAR_TRIGGER')
const isSidebarCrossTrigger = userText.includes('E2E_SIDEBAR_CROSS')
const isQueueStopTrigger = userText.includes('E2E_QUEUE_STOP_TRIGGER')
const isVerificationStopTrigger = messages.some(
message => typeof message?.content === 'string' && message.content.includes(VERIFICATION_STOP_TRIGGER),
)
const isCorrectionSwitchTrigger = messages.some(
message => typeof message?.content === 'string' && message.content.includes(CORRECTION_SWITCH_TRIGGER),
)
@ -364,6 +399,18 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
return
}
if (isVerificationStopTrigger) {
const script = verificationStopScript(options.verificationWritePath ?? 'e2e-verification-target.py')
const turn = script[_verificationStopIndex] ?? script[script.length - 1]
_verificationStopIndex++
if (stream) {
streamScriptedTurn(res, model, turn)
} else {
nonStreamingScriptedTurn(res, model, turn)
}
return
}
if (isCorrectionSwitchTrigger) {
const turn = CORRECTION_SWITCH_SCRIPT[_correctionSwitchIndex] ?? CORRECTION_SWITCH_SCRIPT[CORRECTION_SWITCH_SCRIPT.length - 1]
_correctionSwitchIndex++

View file

@ -0,0 +1,226 @@
import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process'
import * as path from 'node:path'
import { createInterface } from 'node:readline'
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
const DEFAULT_TIMEOUT_MS = 60_000
interface JsonRpcError {
code?: number
message?: string
}
interface JsonRpcFrame {
error?: JsonRpcError
id?: number
method?: string
params?: {
payload?: unknown
session_id?: string
type?: string
}
result?: unknown
}
interface CreatedSession {
session_id: string
stored_session_id: string
}
export interface RealSessionSpec {
/** Human-visible sidebar title, persisted by the first completed turn. */
title: string
/** Each item becomes one real user prompt followed by the mock provider's reply. */
turns: readonly string[]
}
export interface RealSession {
/** Runtime-only TUI session id, valid only while the builder process is alive. */
runtimeId: string
/** Durable SessionDB id that desktop resumes after the builder exits. */
sessionId: string
}
/**
* Creates durable desktop session history through the real TUI gateway and
* AIAgent loop, using the E2E mock provider configured in `hermesHome`.
*
* This intentionally uses the shipped stdio JSON-RPC transport instead of
* importing SessionDB or launching Electron. The desktop's WebSocket backend
* dispatches the same `tui_gateway.server` methods.
*/
export class RealSessionBuilder {
private readonly child: ChildProcessWithoutNullStreams
private nextRequestId = 0
private readonly pending = new Map<number, { reject: (reason: Error) => void; resolve: (value: unknown) => void }>()
private readonly events: JsonRpcFrame[] = []
private readonly eventWaiters: Array<{
predicate: (frame: JsonRpcFrame) => boolean
reject: (reason: Error) => void
resolve: (frame: JsonRpcFrame) => void
}> = []
private readonly stderr: string[] = []
private closed = false
private constructor(hermesHome: string) {
this.child = spawn('uv', ['run', '--active', '--no-sync', 'python', '-m', 'tui_gateway.entry'], {
cwd: REPO_ROOT,
env: {
...process.env,
HERMES_HOME: hermesHome,
PYTHONPATH: REPO_ROOT,
},
stdio: 'pipe',
})
createInterface({ input: this.child.stdout }).on('line', line => this.handleLine(line))
createInterface({ input: this.child.stderr }).on('line', line => {
this.stderr.push(line)
if (this.stderr.length > 80) this.stderr.shift()
})
this.child.once('error', error => this.failAll(new Error(`real-session gateway failed to start: ${error.message}`)))
this.child.once('exit', (code, signal) => {
if (!this.closed) {
this.failAll(new Error(`real-session gateway exited unexpectedly (${signal ?? code ?? 'unknown'}):\n${this.stderr.join('\n')}`))
}
})
}
static async start(hermesHome: string): Promise<RealSessionBuilder> {
const builder = new RealSessionBuilder(hermesHome)
await builder.waitForEvent(frame => frame.params?.type === 'gateway.ready')
return builder
}
async createSession(spec: RealSessionSpec): Promise<RealSession> {
if (spec.turns.length === 0) {
throw new Error('RealSessionBuilder requires at least one turn so the real agent creates a durable session row')
}
const created = await this.request<CreatedSession>('session.create', {
cols: 120,
cwd: REPO_ROOT,
source: 'desktop',
title: spec.title,
})
const runtimeId = requireString(created, 'session_id')
const sessionId = requireString(created, 'stored_session_id')
for (const text of spec.turns) {
const completion = this.waitForEvent(
frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId,
)
await this.request('prompt.submit', { session_id: runtimeId, text })
const frame = await completion
const status = readString(frame.params?.payload, 'status')
if (status !== 'complete') {
throw new Error(`real session turn failed with status ${status ?? 'unknown'}: ${JSON.stringify(frame.params?.payload)}`)
}
}
await this.request('session.close', { session_id: runtimeId })
return { runtimeId, sessionId }
}
async close(): Promise<void> {
if (this.closed) return
this.closed = true
this.child.stdin.end()
await new Promise<void>(resolve => {
const timeout = setTimeout(() => {
this.child.kill('SIGTERM')
resolve()
}, 5_000)
this.child.once('exit', () => {
clearTimeout(timeout)
resolve()
})
})
}
private request<T = unknown>(method: string, params: Record<string, unknown>): Promise<T> {
const id = ++this.nextRequestId
return this.withTimeout(new Promise<T>((resolve, reject) => {
this.pending.set(id, { resolve: value => resolve(value as T), reject })
this.child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`, error => {
if (error) {
this.pending.delete(id)
reject(error)
}
})
}), `request ${method}`)
}
private waitForEvent(predicate: (frame: JsonRpcFrame) => boolean): Promise<JsonRpcFrame> {
const index = this.events.findIndex(predicate)
if (index >= 0) {
return Promise.resolve(this.events.splice(index, 1)[0])
}
return this.withTimeout(new Promise<JsonRpcFrame>((resolve, reject) => {
this.eventWaiters.push({ predicate, resolve, reject })
}), 'gateway event')
}
private handleLine(line: string): void {
let frame: JsonRpcFrame
try {
frame = JSON.parse(line) as JsonRpcFrame
} catch {
return
}
if (typeof frame.id === 'number') {
const pending = this.pending.get(frame.id)
if (!pending) return
this.pending.delete(frame.id)
if (frame.error) {
pending.reject(new Error(`JSON-RPC error ${frame.error.code ?? 'unknown'}: ${frame.error.message ?? 'unknown error'}`))
} else {
pending.resolve(frame.result)
}
return
}
if (frame.method !== 'event') return
const waiter = this.eventWaiters.find(candidate => candidate.predicate(frame))
if (!waiter) {
this.events.push(frame)
return
}
this.eventWaiters.splice(this.eventWaiters.indexOf(waiter), 1)
waiter.resolve(frame)
}
private withTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`Timed out after ${DEFAULT_TIMEOUT_MS / 1000}s waiting for ${operation}:\n${this.stderr.join('\n')}`)), DEFAULT_TIMEOUT_MS)
promise.then(value => {
clearTimeout(timer)
resolve(value)
}, error => {
clearTimeout(timer)
reject(error)
})
})
}
private failAll(error: Error): void {
for (const pending of this.pending.values()) pending.reject(error)
this.pending.clear()
for (const waiter of this.eventWaiters) waiter.reject(error)
this.eventWaiters.length = 0
}
}
function readString(value: unknown, key: string): string | undefined {
if (!value || typeof value !== 'object') return undefined
const candidate = (value as Record<string, unknown>)[key]
return typeof candidate === 'string' ? candidate : undefined
}
function requireString(value: unknown, key: string): string {
const candidate = readString(value, key)
if (!candidate) throw new Error(`Gateway response omitted required ${key}: ${JSON.stringify(value)}`)
return candidate
}

View file

@ -1,52 +0,0 @@
#!/usr/bin/env python3
"""Seed a deterministic, tool-free large session into an isolated state.db."""
import sys
from pathlib import Path
repo_root = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(repo_root))
from hermes_state import SessionDB # noqa: E402
SESSION_ID = "e2e-large-session"
SESSION_TITLE = "E2E large persisted session"
def main() -> None:
if len(sys.argv) != 2:
raise SystemExit(f"usage: {sys.argv[0]} <state.db>")
messages = []
for index in range(53):
role = "user" if index % 2 == 0 else "assistant"
content = (
f"E2E persisted user message {index}: audit the compatibility matrix"
if role == "user"
else f"E2E persisted assistant reply {index}: recorded the audit result"
)
messages.append({"role": role, "content": content, "timestamp": 1_700_000_000 + index})
database = SessionDB(db_path=Path(sys.argv[1]))
result = database.import_sessions(
[
{
"id": SESSION_ID,
"source": "desktop",
"model": "mock-model",
"started_at": 1_700_000_000,
"title": SESSION_TITLE,
"cwd": str(repo_root),
"system_prompt": "",
"messages": messages,
}
]
)
database.close()
if not result.get("ok") or result.get("imported") != 1:
raise SystemExit(f"failed to seed large session: {result}")
if __name__ == "__main__":
main()

View file

@ -1,56 +0,0 @@
#!/usr/bin/env python3
"""Seed a Hermes state.db with a session exported from a real conversation.
Usage: seed_session_db.py <state_db_path> <fixture_json_path>
Creates the database with the full SessionDB schema (if it doesn't exist)
and imports the session from the JSON fixture. Uses the real
SessionDB.import_sessions() so the data shape matches what the desktop
backend expects.
"""
import json
import sys
from pathlib import Path
# Add the repo root to sys.path so we can import hermes_state.
# The script is invoked from apps/desktop/e2e/ — repo root is ../../..
repo_root = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(repo_root))
from hermes_state import SessionDB # noqa: E402
def main():
if len(sys.argv) != 3:
print(f"Usage: {sys.argv[0]} <state_db_path> <fixture_json_path>", file=sys.stderr)
sys.exit(1)
db_path = Path(sys.argv[1])
fixture_path = Path(sys.argv[2])
db_path.parent.mkdir(parents=True, exist_ok=True)
with open(fixture_path, "r", encoding="utf-8") as f:
session_data = json.load(f)
db = SessionDB(db_path=db_path)
result = db.import_sessions([session_data])
if not result.get("ok"):
print(f"Import failed: {result}", file=sys.stderr)
sys.exit(1)
imported = result.get("imported", 0)
skipped = result.get("skipped", 0)
errors = result.get("errors", [])
if errors:
print(f"Import had errors: {errors}", file=sys.stderr)
sys.exit(1)
print(f"Seeded {imported} session(s), skipped {skipped}{db_path}")
db.close()
if __name__ == "__main__":
main()

View file

@ -28,11 +28,6 @@
* Prerequisite: `npm run build` must have been run so dist/ exists.
*/
import { spawnSync } from 'node:child_process'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { expect, test } from './test'
import {
@ -45,12 +40,9 @@ import {
launchDesktop,
} from './fixtures'
import { startMockServer } from './mock-server'
import { RealSessionBuilder } from './real-session-builder'
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
const SEED_SCRIPT = path.join(DESKTOP_ROOT, 'e2e', 'scripts', 'seed_session_db.py')
const SESSION_TITLE = 'E2E Warm Resume Jitter Test'
const SESSION_ID = 'e2e-warm-resume-session'
/** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */
const MESSAGE_COUNT = 32
/** Seeded PRNG so the generated content is deterministic across runs. */
@ -82,54 +74,27 @@ function gibberish(rng: () => number): string {
const FIRST_USER_MSG = gibberish(mulberry32(RNG_SEED))
/**
* Generate a session fixture with MESSAGE_COUNT messages (user/assistant
* pairs) of seeded gibberish just role + content, enough for SessionDB
* to import and the transcript to render. Written to a temp file for the
* seed script.
* Generate the user turns for a real session. The mock provider produces the
* assistant side of each pair through the normal AIAgent persistence path.
*/
function generateSessionFixture(fixturePath: string): void {
function generateSessionTurns(): string[] {
const rng = mulberry32(RNG_SEED)
const messages: Array<{ role: string; content: string }> = []
const turns: string[] = []
for (let i = 0; i < MESSAGE_COUNT / 2; i++) {
messages.push({ role: 'user', content: gibberish(rng) })
messages.push({ role: 'assistant', content: gibberish(rng) })
turns.push(gibberish(rng))
gibberish(rng)
}
const session = {
id: SESSION_ID,
source: 'cli',
model: 'mock-model',
system_prompt: '',
started_at: 1721692800.0,
message_count: MESSAGE_COUNT,
title: SESSION_TITLE,
cwd: '/tmp',
archived: 0,
rewind_count: 0,
compression_fallback_streak: 0,
messages,
}
fs.writeFileSync(fixturePath, JSON.stringify(session), 'utf8')
}
/** Resolve the python binary from the nix devshell (falls back to python3). */
function findPython(): string {
const result = spawnSync('which', ['python'], { encoding: 'utf8' })
if (result.status === 0 && result.stdout.trim()) {
return result.stdout.trim()
}
return 'python3'
return turns
}
/**
* Set up a mock-backend sandbox with a pre-seeded session in state.db.
* Set up a mock-backend sandbox with a real persisted session in state.db.
*
* Unlike the shared `setupMockBackend()`, this variant seeds the DB
* BEFORE launching the app so the session appears in the sidebar on first
* load exercising the real `resumeSession()` cold path without needing
* to send a message first.
* Unlike the shared `setupMockBackend()`, this variant creates the session
* through the real stdio gateway before launching desktop so the session is
* visible in the sidebar on first load.
*/
async function setupSeededMockBackend(): Promise<MockBackendFixture> {
// 1. Start mock server
@ -140,28 +105,13 @@ async function setupSeededMockBackend(): Promise<MockBackendFixture> {
writeMockProviderConfig(sandbox.hermesHome, mock.url)
writeEnvFile(sandbox.hermesHome)
// 3. Pre-seed state.db: generate a fixture JSON to a temp file, then
// run the seed script to import it into state.db BEFORE launching.
const stateDbPath = path.join(sandbox.hermesHome, 'state.db')
const fixturePath = path.join(os.tmpdir(), `hermes-e2e-warm-resume-${Date.now()}.json`)
generateSessionFixture(fixturePath)
const python = findPython()
const seedResult = spawnSync(
python,
[SEED_SCRIPT, stateDbPath, fixturePath],
{
cwd: REPO_ROOT,
env: { ...process.env, PYTHONPATH: REPO_ROOT },
encoding: 'utf8',
timeout: 30_000,
},
)
fs.unlinkSync(fixturePath)
if (seedResult.status !== 0) {
throw new Error(
`Failed to seed state.db:\nstdout: ${seedResult.stdout}\nstderr: ${seedResult.stderr}`,
)
// 3. Produce all 16 user/assistant pairs through the real TUI gateway,
// AIAgent, mock provider, and SessionDB persistence path before desktop starts.
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
try {
await builder.createSession({ title: SESSION_TITLE, turns: generateSessionTurns() })
} finally {
await builder.close()
}
// 4. Build env + launch

View file

@ -134,6 +134,7 @@ const originalWebSocket = globalThis.WebSocket
beforeEach(() => {
// Drop any parked gateway left by a prior file/case (globalThis slot).
const leftover = takeGatewaySurvivor()
if (leftover) {
try {
leftover.gateway.close()
@ -141,6 +142,7 @@ beforeEach(() => {
// ignore
}
}
vi.useFakeTimers()
FakeWebSocket.mode = 'open'
FakeWebSocket.instances = []
@ -166,6 +168,7 @@ afterEach(() => {
// open gateway instead of tearing it down (the real HMR path). Drain + close
// that survivor so the next test boots a fresh socket instead of adoptBoot().
const survivor = takeGatewaySurvivor()
if (survivor) {
try {
survivor.gateway.close()
@ -173,6 +176,7 @@ afterEach(() => {
// ignore
}
}
vi.useRealTimers()
;(globalThis as { WebSocket: unknown }).WebSocket = originalWebSocket
delete (window as { hermesDesktop?: unknown }).hermesDesktop

View file

@ -501,27 +501,36 @@ export function useMessageStream({
const existing = prev[index]
const existingText = chatMessageText(existing).trim()
// The last assistant row is a sealed interim (a tool-call turn or a
// verify-on-stop candidate — `message.interim` fires for BOTH, see
// tui_gateway `_load_interim_assistant_messages`). When the final
// completion is the SAME turn's reply, settle it onto that interim
// instead of appending a second bubble. Continuity, not exact
// equality: streaming can drop characters and the final may add a
// trailing delta, so treat prefix-either-way as the same message.
// (mergeFinalAssistantText, via completeMessage, does the real
// text merge — replaces the interim's text with the full final.)
const finalContinuesInterim = Boolean(
existing.interim &&
finalText &&
existingText &&
(finalText === existingText || finalText.startsWith(existingText) || existingText.startsWith(finalText))
)
if (existing.pending || (!interimBoundaryPending && finalText && existingText === finalText)) {
nextMessages = prev.map((message, messageIndex) =>
messageIndex === index ? completeMessage(message) : message
)
} else if (
interimBoundaryPending &&
responsePreviewed &&
finalText &&
existingText &&
finalText.startsWith(existingText)
) {
// The verification candidate was published provisionally as an
// interim message and then reused as the terminal response
// (continuation-budget fallback). Settle the interim in place
// instead of creating a duplicate — the DB has one row, so the
// live UI must agree. (#65919 review: duplicate-message blocker)
//
// Prefix match (not exact equality): the final response may be
// the streamed text plus a trailing delta. mergeFinalAssistantText
// (called via completeMessage) handles the actual merge — it
// strips the old text parts and appends the full final text.
} else if (interimBoundaryPending && (responsePreviewed || finalContinuesInterim)) {
// Settle the interim in place instead of creating a duplicate —
// the DB has one row, so the live UI must agree. Previously this
// was gated on `responsePreviewed` alone, so a NON-previewed
// tool-call turn whose final matched its sealed interim appended a
// second bubble (the "renders twice: partial first copy + clean
// final" bug, #63679). `finalContinuesInterim` closes that gap
// for ordinary tool-call turns while `responsePreviewed` still
// covers the verify-on-stop continuation-budget case even when the
// final text was rewritten and no longer shares a prefix.
nextMessages = prev.map((message, messageIndex) =>
messageIndex === index ? completeMessage(message) : message
)

View file

@ -186,18 +186,51 @@ describe('useMessageStream interim text sealing', () => {
expect(getState().interimBoundaryPending).toBe(true)
})
it('keeps an identical final completion distinct from an interim reply without response_previewed', async () => {
it('settles an identical final onto a non-previewed interim (tool-call turn) instead of duplicating (#63679)', async () => {
await mountStream()
await start()
// A plain tool-call turn: the streamed text is sealed as an interim at the
// tool boundary (no response_previewed — that flag is only for verify-on-
// stop). The final completion is the SAME turn's reply. It must settle onto
// the interim, not append a second bubble — the DB has one row. This is the
// "renders twice" bug: partial streamed copy + clean final copy side by side.
await interim('same reply')
await complete('same reply')
// Without response_previewed, the interim and terminal replies are
// distinct messages — the gateway didn't signal that the final reuses
// the provisional candidate.
const texts = assistantMessages()
expect(texts.filter(t => t === 'same reply')).toHaveLength(2)
expect(texts.filter(t => t === 'same reply')).toHaveLength(1)
})
it('settles a prefix-extended final onto a non-previewed interim (streamed + trailing delta)', async () => {
await mountStream()
await start()
// The stream dropped/settled early at the tool boundary; the final adds a
// trailing delta. Same turn — one bubble with the full final text.
await delta('partial')
await interim('partial')
await complete('partial answer continued')
const texts = assistantMessages()
expect(texts.filter(t => t.includes('partial'))).toHaveLength(1)
expect(texts[0]).toBe('partial answer continued')
})
it('appends a genuinely different final as its own bubble (two real assistant segments)', async () => {
await mountStream()
await start()
// The interim is one segment (pre-tool commentary); the final is different
// content, not a continuation of it. These are two real messages and must
// both render — the fix must not over-collapse distinct replies.
await interim('let me check the files')
await complete('the answer is 42')
const texts = assistantMessages()
expect(texts).toContain('let me check the files')
expect(texts).toContain('the answer is 42')
expect(texts).toHaveLength(2)
})
it('settles an identical final completion onto the interim when response_previewed', async () => {

View file

@ -132,6 +132,7 @@ export const en: Translations = {
errors: {
elevenLabsNeedsKey: 'ElevenLabs STT needs ELEVENLABS_API_KEY.',
elevenLabsRejectedKey: 'ElevenLabs rejected the API key (401).',
gatewayAuthFailed: 'Gateway authentication failed — check your API_SERVER_KEY.',
methodNotAllowed:
'The desktop backend rejected that request (405 Method Not Allowed). Try restarting Hermes Desktop.',
microphonePermission: 'Microphone permission was denied.',

View file

@ -133,6 +133,7 @@ export const ja = defineLocale({
errors: {
elevenLabsNeedsKey: 'ElevenLabs STT には ELEVENLABS_API_KEY が必要です。',
elevenLabsRejectedKey: 'ElevenLabs が API キーを拒否しました (401)。',
gatewayAuthFailed: 'ゲートウェイ認証に失敗しました — API_SERVER_KEY を確認してください。',
methodNotAllowed:
'デスクトップバックエンドがそのリクエストを拒否しました (405 Method Not Allowed)。Hermes Desktop を再起動してください。',
microphonePermission: 'マイクのアクセス許可が拒否されました。',

View file

@ -173,6 +173,7 @@ export interface Translations {
errors: {
elevenLabsNeedsKey: string
elevenLabsRejectedKey: string
gatewayAuthFailed: string
methodNotAllowed: string
microphonePermission: string
openaiRejectedApiKey: string

View file

@ -129,6 +129,7 @@ export const zhHant = defineLocale({
errors: {
elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。',
elevenLabsRejectedKey: 'ElevenLabs 拒絕了該 API 金鑰 (401)。',
gatewayAuthFailed: '閘道認證失敗 — 請檢查你的 API_SERVER_KEY。',
methodNotAllowed: '桌面後端拒絕了該請求 (405 Method Not Allowed)。請嘗試重新啟動 Hermes Desktop。',
microphonePermission: '麥克風權限已被拒絕。',
openaiRejectedApiKey: 'OpenAI 拒絕了該 API 金鑰。',

View file

@ -129,6 +129,7 @@ export const zh: Translations = {
errors: {
elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。',
elevenLabsRejectedKey: 'ElevenLabs 拒绝了该 API key (401)。',
gatewayAuthFailed: '网关认证失败 — 请检查你的 API_SERVER_KEY。',
methodNotAllowed: '桌面后端拒绝了该请求 (405 Method Not Allowed)。请尝试重启 Hermes Desktop。',
microphonePermission: '麦克风权限已被拒绝。',
openaiRejectedApiKey: 'OpenAI 拒绝了该 API key。',

View file

@ -158,6 +158,39 @@ describe('toChatMessages', () => {
expect(chatMessageText(message)).toBe('@file:foo.ts\n\nlook')
})
it('projects durable timeline kinds without inspecting their text', () => {
const messages = toChatMessages([
{ role: 'user', content: 'real user turn', timestamp: 1 },
{ role: 'assistant', content: 'real assistant reply', timestamp: 2 },
{
role: 'user',
content: 'opaque compaction payload',
display_kind: 'hidden',
timestamp: 3
},
{
role: 'user',
content: 'opaque model context payload',
display_kind: 'model_switch',
timestamp: 4
},
{
role: 'user',
content: 'opaque delegation context payload',
display_kind: 'async_delegation_complete',
timestamp: 5
}
])
expect(messages.map(message => message.role)).toEqual(['user', 'assistant', 'system', 'system'])
expect(messages.map(chatMessageText)).toEqual([
'real user turn',
'real assistant reply',
'model changed',
'background agent work finished'
])
})
})
describe('renderMediaTags', () => {

View file

@ -303,6 +303,29 @@ function displayContentForMessage(role: SessionMessage['role'], content: unknown
return [refs.join('\n'), visibleText].filter(Boolean).join('\n\n') || visibleText
}
function transcriptContent(displayKind: SessionMessage['display_kind'], content: string): string | null {
return displayKind === 'hidden' ? null : content
}
function timelineDisplayContent(message: SessionMessage, content: string): string {
if (message.display_kind === 'model_switch') {
return 'model changed'
}
if (message.display_kind === 'async_delegation_complete') {
const count =
message.display_metadata && 'task_count' in message.display_metadata
? message.display_metadata.task_count
: undefined
return count === undefined
? 'background agent work finished'
: `${count} background agent${count === 1 ? '' : 's'} finished`
}
return content
}
const STREAM_PART: Record<'reasoning' | 'text', (text: string) => ChatMessagePart> = {
reasoning: reasoningPart,
text: textPart
@ -884,7 +907,17 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
}
const content = message.content || message.text || message.context || message.name
const displayContent = displayContentForMessage(message.role, content)
const displayContent = transcriptContent(
message.display_kind,
timelineDisplayContent(message, displayContentForMessage(message.role, content))
)
const displayRole =
message.display_kind === 'model_switch' || message.display_kind === 'async_delegation_complete'
? 'system'
: message.role
const parts: ChatMessagePart[] = []
const reasoning =
@ -897,7 +930,7 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
}
if (displayContent) {
parts.push(message.role === 'assistant' ? assistantTextPart(displayContent) : textPart(displayContent))
parts.push(displayRole === 'assistant' ? assistantTextPart(displayContent) : textPart(displayContent))
}
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
@ -951,8 +984,8 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
}
result.push({
id: `${message.timestamp || Date.now()}-${index}-${message.role}`,
role: message.role,
id: `${message.timestamp || Date.now()}-${index}-${displayRole}`,
role: displayRole,
parts,
timestamp: message.timestamp
})

View file

@ -0,0 +1,34 @@
import { beforeEach, expect, test } from 'vitest'
import { $notifications, clearNotifications, notifyError } from './notifications'
beforeEach(() => {
clearNotifications()
})
function lastMessage(): string {
return $notifications.get()[0]?.message ?? ''
}
// Regression for #39365: a gateway auth 401 (bad API_SERVER_KEY) must not be
// summarized as a provider (OpenAI/OpenRouter) API key problem.
test('gateway_auth_failed error is summarized as gateway auth, not provider key', () => {
notifyError(
new Error(
'401 {"error": {"message": "Invalid gateway API key (API_SERVER_KEY)", "type": "gateway_auth_error", "code": "gateway_auth_failed"}}'
),
'Request failed'
)
expect(lastMessage()).toContain('API_SERVER_KEY')
expect(lastMessage()).not.toMatch(/OpenAI/i)
})
test('provider invalid_api_key error still maps to the OpenAI summary', () => {
notifyError(
new Error('401 {"error": {"message": "Incorrect API key provided", "code": "invalid_api_key"}}'),
'Request failed'
)
expect(lastMessage()).toMatch(/OpenAI rejected the API key/i)
})

View file

@ -77,6 +77,10 @@ function cleanErrorText(value: string) {
}
const ERROR_SUMMARIES: { test: (msg: string) => boolean; summarize: (msg: string) => string }[] = [
{
test: msg => /['"]code['"]\s*:\s*['"]gateway_auth_failed['"]/i.test(msg),
summarize: () => translateNow('notifications.errors.gatewayAuthFailed')
},
{
test: msg => /incorrect api key provided/i.test(msg) || /['"]code['"]\s*:\s*['"]invalid_api_key['"]/i.test(msg),
summarize: msg => {

View file

@ -423,6 +423,16 @@ export interface SessionInfo {
is_default_profile?: boolean
}
export type TimelineDisplayMetadata =
| { model: string; provider?: string }
| {
delegation_id: string
task_count: number
completed_count?: number
failed_count?: number
duration_seconds?: number
}
export interface SessionMessage {
codex_reasoning_items?: unknown
content: unknown
@ -431,6 +441,8 @@ export interface SessionMessage {
reasoning?: null | string
reasoning_content?: null | string
reasoning_details?: unknown
display_kind?: 'async_delegation_complete' | 'hidden' | 'model_switch' | string
display_metadata?: TimelineDisplayMetadata
role: 'assistant' | 'system' | 'tool' | 'user'
text?: unknown
timestamp?: number

195
cli.py
View file

@ -1743,6 +1743,66 @@ def _worktree_is_dirty(worktree_path: str, timeout: int = 10) -> bool:
return True
def _worktree_commits_all_merged_upstream(
worktree_path: str, timeout: int = 30, max_ahead: int = 20
) -> bool:
"""Return whether every local-only commit is patch-equivalent to a commit
already on the default upstream branch.
The dominant ``.worktrees/`` leak: a branch is pushed, its PR is
squash-merged (or cherry-picked), and the remote branch is deleted. The
local commits are then unreachable from ``refs/remotes/*`` forever, so the
unpushed-commits guard preserves the worktree indefinitely even though its
content is fully merged. ``git cherry`` detects patch-equivalence, letting
the pruner reap these.
Bounded: skips (returns False) when the branch is more than ``max_ahead``
commits ahead a stale-base tree, too expensive to diff-hash and unlikely
to be a merged scratch branch. Fails SAFE toward False (preserve).
"""
import subprocess
base = None
for candidate in ("origin/HEAD", "origin/main", "origin/master"):
try:
probe = subprocess.run(
["git", "rev-parse", "--verify", "--quiet", candidate],
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
)
if probe.returncode == 0 and probe.stdout.strip():
base = candidate
break
except Exception:
return False
if base is None:
return False
try:
ahead = subprocess.run(
["git", "rev-list", "--count", f"{base}..HEAD"],
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
)
if ahead.returncode != 0:
return False
count = int(ahead.stdout.strip() or "0")
if count == 0:
return True
if count > max_ahead:
return False
cherry = subprocess.run(
["git", "cherry", base, "HEAD"],
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
)
if cherry.returncode != 0:
return False
lines = [ln for ln in cherry.stdout.splitlines() if ln.strip()]
# "-" = patch-equivalent commit exists upstream; "+" = unique local work
return bool(lines) and all(ln.startswith("-") for ln in lines)
except Exception:
return False
def _worktree_lock_is_live(repo_root: str, worktree_path: str, timeout: int = 10):
"""Classify a worktree's git lock as live, dead, or absent.
@ -1949,11 +2009,21 @@ def _run_checkpoint_auto_maintenance() -> None:
def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
"""Remove stale worktrees and orphaned branches on startup.
Age-based tiers (aggressive cleanup keeps ``.worktrees/`` from growing
unbounded):
- Under max_age_hours (24h): skip session may still be active.
- 24h72h: remove if no unpushed commits.
- Over 72h: force remove regardless (nothing should sit this long).
Covers EVERY directory under ``.worktrees/`` except kanban task trees
(``t_<hex>`` owned by the kanban dispatcher's own gc). Scratch trees
created by ``hermes -w`` (``hermes-*``) age out fast; named trees created
manually for salvage/review lanes age out on a slower schedule:
- ``hermes-*``: skip under 24h; reap 24h+ when clean and merged/pushed;
72h+ is the aggressive tier (still never deletes real work).
- named trees: same logic at 3x the timeline (72h soft / 9d hard).
Work-preservation guards (all tiers, any age):
- uncommitted changes (dirty) never removed;
- unpushed commits never removed, UNLESS every local-only commit is
patch-equivalent to a commit already on upstream (``git cherry``): the
squash-merged-PR case, which is the dominant ``.worktrees/`` leak since
those commits stay unreachable from ``refs/remotes/*`` forever.
Lock handling (orthogonal to age): ``hermes -w`` locks each worktree with
reason ``hermes pid=<pid>`` so a concurrent hermes process leaves an in-use
@ -1966,9 +2036,14 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
removal never orphans the branch (which would drop easy reachability of any
commits still in the worktree).
Preserved-work visibility: trees skipped for unpushed/dirty reasons that
are older than 7 days are listed in a single WARNING so real in-flight
work can't rot silently.
Also prunes orphaned ``hermes/*`` and ``pr-*`` local branches that
have no corresponding worktree.
"""
import re
import subprocess
import time
@ -1978,13 +2053,23 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
return
now = time.time()
soft_cutoff = now - (max_age_hours * 3600) # 24h default
hard_cutoff = now - (max_age_hours * 3 * 3600) # 72h default
stale_work_cutoff = now - (7 * 24 * 3600)
preserved_stale: list = []
# Kanban task worktrees (<repo>/.worktrees/t_<hex>) have their own
# dispatcher-driven lifecycle (hermes kanban gc) — never touch them here.
kanban_re = re.compile(r"^t_[0-9a-f]+$")
for entry in worktrees_dir.iterdir():
if not entry.is_dir() or not entry.name.startswith("hermes-"):
if not entry.is_dir() or kanban_re.match(entry.name):
continue
# Scratch trees (hermes-*) age out on the default schedule; named
# trees (salvage/review lanes someone created deliberately) get 3x.
scratch = entry.name.startswith("hermes-")
tier_hours = max_age_hours if scratch else max_age_hours * 3
soft_cutoff = now - (tier_hours * 3600)
hard_cutoff = now - (tier_hours * 3 * 3600)
# Check age
try:
mtime = entry.stat().st_mtime
@ -1993,19 +2078,24 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
except Exception:
continue
force = mtime <= hard_cutoff # Over 72h — reap aggressively
force = mtime <= hard_cutoff # Aggressive tier — reap clean trees
# Never delete real work, regardless of age. Unpushed commits and
# uncommitted changes may be a crashed session's in-flight work; the
# >72h tier reaps only abandoned *clean, fully-pushed* worktrees (the
# scratch trees that actually cause .worktrees/ bloat).
# Never delete real work, regardless of age or tier. Uncommitted
# changes and unpushed commits may be a crashed session's in-flight
# work; only clean, fully-merged/pushed trees (the scratch trees that
# actually cause .worktrees/ bloat) are ever reaped.
if _worktree_is_dirty(str(entry), timeout=5):
if mtime <= stale_work_cutoff:
preserved_stale.append(f"{entry.name} (uncommitted changes)")
continue
if _worktree_has_unpushed_commits(str(entry), timeout=5):
continue # Has unpushed commits or can't check — skip
if not force:
# 24h72h tier is conservative: unpushed check above is enough.
pass
elif _worktree_is_dirty(str(entry), timeout=5):
continue # >72h but dirty — preserve uncommitted work
# Squash-merge escape hatch: commits unreachable from any remote
# ref but patch-equivalent to upstream commits are merged work,
# not unpushed work.
if not _worktree_commits_all_merged_upstream(str(entry), timeout=30):
if mtime <= stale_work_cutoff:
preserved_stale.append(f"{entry.name} (unpushed commits)")
continue
# Respect git-native session locks. A lock owned by a still-running
# hermes process means the worktree is actively in use — never touch
@ -2054,6 +2144,13 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
except Exception as e:
logger.debug("Failed to prune worktree %s: %s", entry.name, e)
if preserved_stale:
logger.warning(
"Preserving %d worktree(s) older than 7 days with unmerged work "
"(push or remove them to reclaim disk): %s",
len(preserved_stale), ", ".join(sorted(preserved_stale)),
)
_prune_orphaned_branches(repo_root)
@ -4168,6 +4265,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
self._pending_tool_info: dict = {} # function_name -> list of (preview, args) for stacked scrollback
self._last_scrollback_tool: str = "" # last tool name printed to scrollback (for "new" dedup)
self._command_running = False
self._command_blocks_input = False
self._command_status = ""
# Petdex mascot (opt-in via display.pet). The base CLI mirrors the TUI's
# PetPane: a half-block sprite above the prompt that reacts to agent
@ -6181,9 +6279,17 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
return _COMMAND_SPINNER_FRAMES[frame_idx]
@contextmanager
def _busy_command(self, status: str):
"""Expose a temporary busy state in the TUI while a slash command runs."""
def _busy_command(self, status: str, *, blocks_input: bool = True):
"""Expose a temporary busy state in the TUI while a slash command runs.
Most synchronous slash commands must reserve the composer because their
completion changes the active session state. Manual compression is safe
to draft through: the queued input is processed against the compacted
history after the command completes.
"""
previous_blocks_input = getattr(self, "_command_blocks_input", False)
self._command_running = True
self._command_blocks_input = blocks_input
self._command_status = status
self._invalidate(min_interval=0.0)
try:
@ -6191,6 +6297,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
yield
finally:
self._command_running = False
self._command_blocks_input = previous_blocks_input
self._command_status = ""
self._invalidate(min_interval=0.0)
@ -9381,9 +9488,12 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
# has all API keys in os.environ.
from tools.environments.local import _sanitize_subprocess_env
sanitized_env = _sanitize_subprocess_env(os.environ.copy())
from hermes_cli._subprocess_compat import windows_hide_flags
result = subprocess.run(
exec_cmd, shell=True, capture_output=True,
text=True, timeout=30, env=sanitized_env
text=True, timeout=30, env=sanitized_env,
# No console flash on Windows (#56747).
creationflags=windows_hide_flags(),
)
output = result.stdout.strip() or result.stderr.strip()
if output:
@ -10017,7 +10127,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
return
original_count = len(self.conversation_history)
with self._busy_command("Compressing context..."):
with self._busy_command("Compressing context...", blocks_input=False):
try:
from agent.model_metadata import estimate_request_tokens_rough
from agent.manual_compression_feedback import summarize_manual_compression
@ -10073,6 +10183,39 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
force=True,
defer_context_engine_notification=True,
)
# If _compress_context returned unchanged because a
# concurrent compression lock is held, tell the user
# clearly instead of showing the misleading
# "No changes from compression" no-op text. The wording
# distinguishes a confirmed holder from an unconfirmed
# acquisition failure (describe_compression_lock_skip).
# Type-pinned check (is True / str): the flag's only real
# values are None/True/holder-string, and a bare getattr
# truthiness test is fooled by MagicMock auto-attributes on
# test-double agents (skill pitfall: MagicMock vs hasattr).
_lock_skip_signal = getattr(
self.agent, "_compression_skipped_due_to_lock", None
)
if _lock_skip_signal is True or isinstance(_lock_skip_signal, str):
from agent.manual_compression_feedback import (
describe_compression_lock_skip,
)
print(
" "
+ describe_compression_lock_skip(
self.agent._compression_skipped_due_to_lock
)
)
self.agent._compression_skipped_due_to_lock = None
# No boundary was committed on a lock-skip; discard the
# deferred context-engine notification (exactly-once).
finalize_context_engine_compression_notification(
self.agent,
committed=False,
)
return
if partial and tail:
compressed = rejoin_compressed_head_and_tail(compressed, tail)
self.conversation_history = compressed
@ -13443,6 +13586,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
# Slash command loading state
self._command_running = False
self._command_blocks_input = False
self._command_status = ""
# Secure secret capture state for skill setup
@ -14404,7 +14548,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
style='class:input-area',
multiline=True,
wrap_lines=True,
read_only=Condition(lambda: bool(cli_ref._command_running)),
read_only=Condition(lambda: bool(cli_ref._command_blocks_input)),
history=FileHistory(str(self._history_file)),
# complete_while_typing fires the completer on every keystroke. The
# completer does blocking work — fuzzy @-file indexing shells out to
@ -14615,8 +14759,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
if cli_ref._command_running:
frame = cli_ref._command_spinner_frame()
detail = "input temporarily disabled" if cli_ref._command_blocks_input else "input stays active; Enter queues"
return [
('class:hint', f' {frame} command in progress · input temporarily disabled'),
('class:hint', f' {frame} command in progress · {detail}'),
]
return []

View file

@ -0,0 +1 @@
2001Y

View file

@ -0,0 +1,2 @@
metamon-p
# PR #36220 salvage (channel continuity hint)

View file

@ -0,0 +1 @@
2001Y

View file

@ -0,0 +1 @@
benjamin2026-dot

View file

@ -0,0 +1 @@
patp

View file

@ -0,0 +1 @@
dnth

View file

@ -0,0 +1 @@
gonzalofrancoceballos

View file

@ -0,0 +1 @@
airclear

View file

@ -0,0 +1,2 @@
harrisonmedmedmetrics
# C17 Slack reaction events salvage (#33111/#44508/#45265)

View file

@ -0,0 +1 @@
ianks

View file

@ -0,0 +1 @@
Trantor-develops

View file

@ -0,0 +1,2 @@
johnkattenhorn
# C17 Slack reaction events salvage (#33111/#44508/#45265)

View file

@ -0,0 +1 @@
jordanhubbard

View file

@ -0,0 +1,2 @@
Kev-fs
# C17 Slack reaction events salvage (#33111/#44508/#45265)

View file

@ -0,0 +1 @@
lanyusea

View file

@ -0,0 +1 @@
cifangyiquan

View file

@ -0,0 +1 @@
enzo2

View file

@ -0,0 +1 @@
mattshapsss

View file

@ -0,0 +1 @@
mwbrooks

View file

@ -0,0 +1,2 @@
mzkarami
# PR #66204 contributor identity

View file

@ -0,0 +1 @@
peterw

View file

@ -0,0 +1 @@
arimu1

View file

@ -0,0 +1 @@
panDing19

View file

@ -0,0 +1 @@
replygirl

View file

@ -0,0 +1 @@
MustafaK99

View file

@ -0,0 +1 @@
trac3r00

View file

@ -0,0 +1 @@
schattenan

View file

@ -0,0 +1 @@
shivasymbl

View file

@ -0,0 +1 @@
navahc09

View file

@ -0,0 +1 @@
drleadflow

View file

@ -0,0 +1 @@
Skywind5487

View file

@ -0,0 +1 @@
sl4m3

View file

@ -0,0 +1 @@
peacockesq

View file

@ -0,0 +1 @@
ddifa86

View file

@ -0,0 +1 @@
elphamale

View file

@ -0,0 +1 @@
yemi-lagosinternationalmarket

View file

@ -0,0 +1 @@
z23

View file

@ -0,0 +1 @@
kylezh

View file

@ -9,6 +9,7 @@ action="list" and for resolving human-friendly channel names to numeric IDs.
import asyncio
import json
import logging
import time
from datetime import datetime
from typing import Any, Dict, List, Optional
@ -18,6 +19,14 @@ from utils import atomic_json_write
logger = logging.getLogger(__name__)
DIRECTORY_PATH = get_hermes_home() / "channel_directory.json"
# Throttle window for repeated Slack channel-directory refresh failures.
# The directory rebuilds on a timer, so a persistent workspace error (e.g.
# missing scope, revoked token) would otherwise re-log the same warning on
# every refresh. Warn once per (team, error detail) per interval; repeats
# drop to DEBUG.
_SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS = 3600
_slack_directory_warning_last: Dict[tuple[str, str], float] = {}
# User-maintained friendly-name overlay. The directory is fully regenerated
# from live adapters + session data on a timer, so hand-edits to
# channel_directory.json don't survive. Aliases declared here are re-applied
@ -105,6 +114,27 @@ def _session_entry_name(origin: Dict[str, Any]) -> str:
return f"{base_name} / {topic_label}"
def _warn_slack_directory(team_id: str, detail: str) -> None:
"""Warn once per team/error per interval for recurring Slack refresh failures."""
key = (str(team_id), str(detail))
now = time.monotonic()
last = _slack_directory_warning_last.get(key)
if last is None or now - last >= _SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS:
_slack_directory_warning_last[key] = now
logger.warning(
"Channel directory: failed to list Slack channels for team %s: %s",
team_id,
detail,
)
else:
logger.debug(
"Channel directory: suppressed repeated Slack channel list failure "
"for team %s: %s",
team_id,
detail,
)
# ---------------------------------------------------------------------------
# Build / refresh
# ---------------------------------------------------------------------------
@ -214,13 +244,29 @@ def _build_discord(adapter) -> List[Dict[str, str]]:
return channels
def _slack_api_error_code(error: Exception) -> Optional[str]:
"""Return Slack Web API error code from SlackApiError-like exceptions."""
response = getattr(error, "response", None)
if isinstance(response, dict):
value = response.get("error")
return str(value) if value else None
if response is not None:
try:
value = response.get("error")
return str(value) if value else None
except Exception:
pass
return None
async def _build_slack(adapter) -> List[Dict[str, Any]]:
"""List Slack channels the bot has joined across all workspaces.
Uses ``users.conversations`` against each workspace's web client. Pulls
public + private channels the bot is a member of, then merges in DMs
discovered from session history (IMs aren't useful to enumerate
proactively).
proactively). If the Slack app lacks channels:read, fall back to session
history quietly instead of logging a recurring warning every refresh.
"""
team_clients = getattr(adapter, "_team_clients", None) or {}
if not team_clients:
@ -240,11 +286,15 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]:
cursor=cursor,
)
if not response.get("ok"):
logger.warning(
"Channel directory: users.conversations not ok for team %s: %s",
team_id,
response.get("error", "unknown"),
)
error_code = response.get("error", "unknown")
if error_code == "missing_scope":
logger.debug(
"Channel directory: Slack team %s lacks channels:read; using session history only",
team_id,
)
else:
detail = f"users.conversations not ok: {error_code}"
_warn_slack_directory(team_id, detail)
break
for ch in response.get("channels", []):
cid = ch.get("id")
@ -261,17 +311,59 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]:
if not cursor:
break
except Exception as e:
logger.warning(
"Channel directory: failed to list Slack channels for team %s: %s",
team_id, e,
)
if _slack_api_error_code(e) == "missing_scope":
logger.debug(
"Channel directory: Slack team %s lacks channels:read; using session history only",
team_id,
)
else:
_warn_slack_directory(team_id, str(e))
continue
# Merge in DM/group entries discovered from session history.
# Build a lookup from API-discovered channels so we can enrich session entries.
api_name_lookup = {ch["id"]: ch["name"] for ch in channels}
for entry in await asyncio.to_thread(_build_from_sessions, "slack"):
if entry.get("id") not in seen_ids:
eid = entry.get("id")
if eid not in seen_ids:
# If the entry name is still a raw Slack ID (e.g. C0xxx / D0xxx),
# try to resolve it from the API lookup first.
if entry.get("name", "").startswith(("C0", "D0", "G0")):
if eid in api_name_lookup:
entry["name"] = api_name_lookup[eid]
channels.append(entry)
seen_ids.add(entry.get("id"))
seen_ids.add(eid)
# Resolve remaining raw-ID entries (DMs, private channels not in bot scope)
# by calling conversations.info + users.info for each.
unresolved = [ch for ch in channels if ch.get("name", "").startswith(("C0", "D0", "G0"))]
if unresolved and team_clients:
client = next(iter(team_clients.values()))
for entry in unresolved:
try:
resp = await client.conversations_info(channel=entry["id"])
if not resp.get("ok"):
continue
ch_info = resp.get("channel", {})
if ch_info.get("is_im"):
peer_user = ch_info.get("user", "")
if peer_user:
user_resp = await client.users_info(user=peer_user)
if user_resp.get("ok"):
u = user_resp["user"]
entry["name"] = (
u.get("profile", {}).get("display_name")
or u.get("real_name")
or u.get("name")
or entry["id"]
)
entry["type"] = "dm"
else:
entry["name"] = ch_info.get("name") or ch_info.get("name_normalized") or entry["id"]
except Exception as e:
logger.debug("Channel directory: failed to resolve %s: %s", entry["id"], e)
continue
return channels

View file

@ -789,6 +789,22 @@ class StreamingConfig:
# platform is sufficiently configured to be considered "connected". Platforms
# that rely on the generic ``token or api_key`` check (Telegram, Discord,
# Slack, Matrix, Mattermost, HomeAssistant) do not need an entry here.
def _has_usable_api_server_key(key: object) -> bool:
"""True when API_SERVER_KEY is present and strong enough to be usable.
Mirrors the startup guard in ``gateway/platforms/api_server.py``
(``has_usable_secret`` with ``min_length=16``) so the platform is only
enrolled at load time when the adapter would actually agree to start.
"""
if not key:
return False
try:
from hermes_cli.auth import has_usable_secret
except ImportError:
return len(str(key).strip()) >= 16
return has_usable_secret(key, min_length=16)
_PLATFORM_CONNECTED_CHECKERS: dict[Platform, Callable[[PlatformConfig], bool]] = {
Platform.WEIXIN: lambda cfg: bool(
cfg.extra.get("account_id") and (cfg.token or cfg.extra.get("token"))
@ -797,7 +813,9 @@ _PLATFORM_CONNECTED_CHECKERS: dict[Platform, Callable[[PlatformConfig], bool]] =
cfg.extra.get("phone_number_id") and cfg.extra.get("access_token")
),
Platform.SIGNAL: lambda cfg: bool(cfg.extra.get("http_url")),
Platform.API_SERVER: lambda cfg: True,
Platform.API_SERVER: lambda cfg: _has_usable_api_server_key(
cfg.extra.get("key") if cfg else None
),
Platform.WEBHOOK: lambda cfg: True,
Platform.MSGRAPH_WEBHOOK: lambda cfg: bool(
str(cfg.extra.get("client_state") or "").strip()
@ -1386,6 +1404,41 @@ def load_gateway_config() -> GatewayConfig:
_merge_platform_map(gateway_platforms)
_merge_platform_map(yaml_cfg.get("platforms"))
# Also merge platform configs placed directly under ``gateway.*``
# (e.g. ``gateway.api_server``) so subsections are discovered the
# same way ``gateway.streaming`` is handled elsewhere. Iterate
# all ``gateway:*`` keys and merge only those that match a known
# platform value, skipping reserved keys like ``platforms``.
if isinstance(gateway_cfg, dict):
_nested_platforms: dict = {}
for _k, _v in gateway_cfg.items():
if _k == "platforms":
continue
try:
Platform(_k)
except (ValueError, AttributeError):
continue
if isinstance(_v, dict):
_nested_platforms[_k] = _v
if _nested_platforms:
_merge_platform_map(_nested_platforms)
# Bridge api_server-specific keys (port, key, host, cors_origins,
# model_name) into extra so PlatformConfig.from_dict preserves
# them — adapting what _apply_env_overrides does for env vars to
# the YAML path. Users writing ``gateway.api_server.port: 8642``
# expect these to end up in the platform's extra dict.
_api_plat = platforms_data.get("api_server")
if isinstance(_api_plat, dict):
_api_extra = _api_plat.get("extra")
if not isinstance(_api_extra, dict):
_api_extra = {}
_api_plat["extra"] = _api_extra
for _bridge_key in ("port", "key", "host", "cors_origins", "model_name"):
if _bridge_key in _api_plat and _bridge_key not in _api_extra:
_api_extra[_bridge_key] = _api_plat.pop(_bridge_key)
if platforms_data:
gw_data["platforms"] = platforms_data
# Iterate built-in platforms plus any registered plugin platforms
@ -1497,6 +1550,24 @@ def load_gateway_config() -> GatewayConfig:
bridged["typing_indicator"] = platform_cfg["typing_indicator"]
if "typing_status_text" in platform_cfg:
bridged["typing_status_text"] = platform_cfg["typing_status_text"]
# Bridge top-level port/host/secret into extra for platforms
# whose adapters read these from config.extra (webhook,
# msgraph_webhook, api_server). Without this, YAML like:
# platforms:
# webhook:
# enabled: true
# port: 8649
# silently falls back to the hardcoded DEFAULT_PORT because
# PlatformConfig.from_dict only extracts ``extra`` from the
# ``extra:`` sub-key, not from the top level.
if plat in {Platform.WEBHOOK, Platform.MSGRAPH_WEBHOOK}:
for _bridge_key in ("port", "host", "secret"):
if _bridge_key in platform_cfg and _bridge_key not in platform_cfg.get("extra", {}):
bridged[_bridge_key] = platform_cfg[_bridge_key]
if plat == Platform.API_SERVER:
for _bridge_key in ("port", "host"):
if _bridge_key in platform_cfg and _bridge_key not in platform_cfg.get("extra", {}):
bridged[_bridge_key] = platform_cfg[_bridge_key]
has_channel_overrides = "channel_overrides" in platform_cfg
if has_channel_overrides:
raw_overrides = platform_cfg.get("channel_overrides")
@ -2008,10 +2079,27 @@ def _apply_env_overrides(config: GatewayConfig) -> None:
api_server_cors_origins = getenv("API_SERVER_CORS_ORIGINS", "")
api_server_port = getenv("API_SERVER_PORT")
api_server_host = getenv("API_SERVER_HOST")
if api_server_enabled or api_server_key:
# Require a usable key: API_SERVER_ENABLED alone would load an
# unauthenticated platform whose adapter refuses to start at connect()
# anyway (startup guard in gateway/platforms/api_server.py), leaving the
# reconnect watcher spinning and logging errors forever. Same strength
# bar as the startup guard (has_usable_secret, min_length=16).
if _has_usable_api_server_key(api_server_key):
if Platform.API_SERVER not in config.platforms:
config.platforms[Platform.API_SERVER] = PlatformConfig()
config.platforms[Platform.API_SERVER].enabled = True
# Respect an explicit ``enabled: false`` in config.yaml (flagged by
# ``_enabled_explicit``). In multiplex mode a secondary profile's
# config.yaml pins ``platforms.api_server.enabled: false`` so it shares
# the default profile's listener instead of binding its own port. That
# profile still inherits the process-level env (including
# ``API_SERVER_KEY``); without this guard the env-var presence would
# force-enable the listener and trip the MultiplexConfigError check.
# Pop (don't read) the marker — the api_server branch is terminal (no
# later registry pass re-enables it), so this both consumes the flag and
# avoids reading it twice, matching the pop convention used elsewhere.
api_server_explicit = config.platforms[Platform.API_SERVER].extra.pop("_enabled_explicit", False)
if not api_server_explicit or config.platforms[Platform.API_SERVER].enabled:
config.platforms[Platform.API_SERVER].enabled = True
if api_server_key:
config.platforms[Platform.API_SERVER].extra["key"] = api_server_key
if api_server_cors_origins:

View file

@ -140,7 +140,12 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
# Tier 2 — edit support, often customer/workspace channels
# Slack: tool_progress off by default — Bolt posts cannot be edited like CLI;
# "new"/"all" spam permanent lines in channels (hermes-agent#14663).
"slack": {**_TIER_MEDIUM, "tool_progress": "off"},
"slack": {
**_TIER_MEDIUM,
"tool_progress": "off",
"long_running_notifications": False,
"busy_ack_detail": False,
},
"mattermost": _TIER_MEDIUM,
"matrix": _TIER_MEDIUM,
"feishu": _TIER_MEDIUM,
@ -154,6 +159,13 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
# status update as a separate message. Promote to TIER_MEDIUM once
# Cloud's edit_message lands.
"whatsapp_cloud": _TIER_LOW,
# Photon (managed iMessage over the gRPC sidecar) and BlueBubbles are both
# permanent-message iMessage inboxes with no message-edit support, so both
# stay TIER_LOW. This keeps tool progress, interim scratch commentary,
# "still working" heartbeats, and busy-ack iteration detail out of the
# user's iMessage thread. Without this entry Photon inherited the noisy
# global ("all") defaults and compacted/narrated on nearly every turn.
"photon": _TIER_LOW,
"bluebubbles": _TIER_LOW,
"weixin": _TIER_LOW,
"wecom": _TIER_LOW,

View file

@ -336,6 +336,14 @@ class GatewayKanbanWatchersMixin:
continue
title = (task.title if task else sub["task_id"])[:120]
board_tag = f"[{board_slug}] " if board_slug else ""
# Per-subscription failure-counter key. Hoisted out of the
# event loop: the wake self-post path (in the loop's
# ``else`` clause) needs it even when every event in the
# claim was skipped before reaching the send site.
sub_key = (
sub["task_id"], sub["platform"],
sub["chat_id"], sub.get("thread_id") or "",
)
for ev in d["events"]:
kind = ev.kind
# Identity prefix: attribute terminal pings to the
@ -408,14 +416,49 @@ class GatewayKanbanWatchersMixin:
metadata: dict[str, Any] = {}
if sub.get("thread_id"):
metadata["thread_id"] = sub["thread_id"]
sub_key = (
sub["task_id"], sub["platform"],
sub["chat_id"], sub.get("thread_id") or "",
)
# Adapters with no push channel (the API server —
# ``supports_async_delivery = False``) can NEVER
# satisfy a text-send: ``send()`` always reports
# SendResult(success=False) by design (see
# ApiServerAdapter.send()). Treating that as a
# delivery failure would rewind/drop the subscription
# forever and — because the wake dispatch below lives
# in this loop's ``else`` clause — would also make the
# wake-on-completion path (the actual fix for the
# api_server wrong-session bug) unreachable. So for
# non-push adapters, skip the doomed send attempt
# entirely: there is nothing to text-notify, the
# creator is woken via the self-post below instead.
from gateway.wake import adapter_supports_push
if not adapter_supports_push(adapter):
logger.debug(
"kanban notifier: adapter %s has no push "
"channel; skipping text ping for %s, relying "
"on wake self-post instead",
platform_str, sub["task_id"],
)
# Do NOT reset the failure counter here: on this
# path the wake self-post below IS the delivery,
# so the counter is resolved (reset or bumped) by
# the self-post outcome, not by skipping the send.
continue
try:
await adapter.send(
_send_res = await adapter.send(
sub["chat_id"], msg, metadata=metadata,
)
# A SendResult(success=False) without an exception
# (returned by push-capable adapters on a genuine
# transient failure) must count as a FAILED
# delivery — otherwise the cursor advances and the
# event is permanently lost. Adapters returning
# None (or anything non-SendResult shaped) keep
# the legacy "no exception == delivered" contract.
if getattr(_send_res, "success", True) is False:
raise RuntimeError(
"adapter send() reported failure: "
f"{getattr(_send_res, 'error', None) or 'unknown error'}"
)
logger.debug(
"kanban notifier: delivered %s event for %s to %s/%s on board %s",
kind, sub["task_id"], platform_str, sub["chat_id"], board_slug,
@ -475,12 +518,108 @@ class GatewayKanbanWatchersMixin:
# dropping the subscription is the terminal action.
break
else:
# All events delivered; advance cursor. The cursor
# All text pings delivered (or intentionally skipped
# for non-push adapters, whose delivery is the wake
# self-post below). Whether the cursor may advance now
# depends on the adapter class:
#
# * push-capable: the text send WAS the delivery, so
# advance immediately (pre-existing behavior); the
# wake injection below stays best-effort.
# * non-push (api_server): the wake self-post IS the
# delivery. Advancing first would let a failed /
# retry-exhausted self-post (swallowed by the
# best-effort except) permanently lose the event.
# So the self-post runs FIRST and the cursor only
# advances after it succeeds — a failure rewinds the
# claim exactly like a failed send() above, so the
# next tick retries.
task_terminal = task and task.status in {"done", "archived"}
_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked")
_wake_kinds = {ev.kind for ev in d["events"] if ev.kind in _WAKE_KINDS}
from gateway.wake import adapter_supports_push as _adapter_push_ok
_is_push_adapter = _adapter_push_ok(adapter)
_session_key = ""
_synth = ""
if _wake_kinds:
_session_key = getattr(task, "session_id", None) or ""
if _wake_kinds and _session_key:
_title = (task.title if task else sub["task_id"])[:120]
_assignee = task.assignee if task else ""
_parts = []
if "completed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.completed"))
if "gave_up" in _wake_kinds: _parts.append(t("gateway.kanban.wake.gave_up"))
if "crashed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.crashed"))
if "timed_out" in _wake_kinds: _parts.append(t("gateway.kanban.wake.timed_out"))
if "blocked" in _wake_kinds: _parts.append(t("gateway.kanban.wake.blocked"))
_status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
_synth = t(
"gateway.kanban.wake.message",
task_id=sub["task_id"],
status=_status,
title=_title,
assignee=_assignee,
board=board_slug,
)
if not _is_push_adapter and _wake_kinds and _session_key:
# Wake self-post IS the delivery on this path —
# it must succeed BEFORE the cursor advances.
from gateway.wake import deliver_wake
try:
await deliver_wake(
adapter,
text=_synth,
session_id=_session_key,
)
logger.info(
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
)
sub_fail_counts.pop(sub_key, None)
except Exception as _wk_err:
fails = sub_fail_counts.get(sub_key, 0) + 1
sub_fail_counts[sub_key] = fails
logger.warning(
"kanban notifier: wake self-post failed "
"for %s (attempt %d/%d): %s",
sub["task_id"], fails,
MAX_SEND_FAILURES, _wk_err, exc_info=True,
)
if fails >= MAX_SEND_FAILURES:
logger.warning(
"kanban notifier: dropping subscription "
"%s on %s after %d consecutive wake failures",
sub["task_id"], platform_str, fails,
)
await asyncio.to_thread(self._kanban_unsub, sub, board_slug)
sub_fail_counts.pop(sub_key, None)
else:
# Rewind the pre-send claim so the next
# tick retries the self-post — the event
# is NOT lost.
await asyncio.to_thread(
self._kanban_rewind,
sub,
d["cursor"],
d.get("old_cursor", 0),
board_slug,
)
continue
# Delivery complete (text ping for push adapters, wake
# self-post for non-push): advance cursor. The cursor
# is the dedup mechanism — it prevents re-delivery
# of the same event on subsequent ticks.
await asyncio.to_thread(
self._kanban_advance, sub, d["cursor"], board_slug,
)
if not _is_push_adapter:
# Nothing left to deliver on this path (the wake,
# if any, already succeeded above).
sub_fail_counts.pop(sub_key, None)
# Unsubscribe only when the task has reached a truly
# final status (done / archived). For blocked /
# gave_up / crashed / timed_out the subscription is
@ -488,68 +627,50 @@ class GatewayKanbanWatchersMixin:
# dispatcher respawns the task and it cycles into the
# same state. See the longer comment on TERMINAL_KINDS
# above for the failure mode this prevents.
task_terminal = task and task.status in {"done", "archived"}
_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked")
_wake_kinds = {ev.kind for ev in d["events"] if ev.kind in _WAKE_KINDS}
if _wake_kinds:
if _is_push_adapter and _wake_kinds and _session_key:
try:
_session_key = getattr(task, "session_id", None) or ""
if _session_key:
_title = (task.title if task else sub["task_id"])[:120]
_assignee = task.assignee if task else ""
_parts = []
if "completed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.completed"))
if "gave_up" in _wake_kinds: _parts.append(t("gateway.kanban.wake.gave_up"))
if "crashed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.crashed"))
if "timed_out" in _wake_kinds: _parts.append(t("gateway.kanban.wake.timed_out"))
if "blocked" in _wake_kinds: _parts.append(t("gateway.kanban.wake.blocked"))
_status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
_synth = t(
"gateway.kanban.wake.message",
task_id=sub["task_id"],
status=_status,
title=_title,
assignee=_assignee,
board=board_slug,
)
from gateway.session import SessionSource
from gateway.platforms.base import MessageEvent, MessageType
# KNOWN LIMITATION (tracked follow-up): the
# subscription row does not persist the
# creator's chat_type, and it is not carried
# on the session-context bridge, so we cannot
# faithfully reconstruct the creator's real
# session key here. build_session_key() keys
# DMs (":dm:<chat_id>") on a wholly different
# shape from group/thread, so any hardcoded
# value mis-routes some creators. "group" is
# the least-surprising default for the
# dashboard/group flows this wake primarily
# serves; DM-originated creators are handled
# by the follow-up that stamps + persists
# chat_type end-to-end. handle_message()
# get_or_create_session's the target, so a
# mismatch degrades to "wake lands in a fresh
# group session" — never an exception.
_source = SessionSource(
platform=plat,
chat_id=sub["chat_id"],
chat_type="group",
thread_id=sub.get("thread_id") or None,
user_id=sub.get("user_id"),
profile=sub_profile or None,
)
_synth_event = MessageEvent(
text=_synth,
message_type=MessageType.TEXT,
source=_source,
internal=True,
)
await adapter.handle_message(_synth_event)
logger.info(
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
)
from gateway.session import SessionSource
from gateway.wake import deliver_wake
# KNOWN LIMITATION (tracked follow-up): the
# subscription row does not persist the
# creator's chat_type, and it is not carried
# on the session-context bridge, so we cannot
# faithfully reconstruct the creator's real
# session key here. build_session_key() keys
# DMs (":dm:<chat_id>") on a wholly different
# shape from group/thread, so any hardcoded
# value mis-routes some creators. "group" is
# the least-surprising default for the
# dashboard/group flows this wake primarily
# serves; DM-originated creators are handled
# by the follow-up that stamps + persists
# chat_type end-to-end. handle_message()
# get_or_create_session's the target, so a
# mismatch degrades to "wake lands in a fresh
# group session" — never an exception.
_source = SessionSource(
platform=plat,
chat_id=sub["chat_id"],
chat_type="group",
thread_id=sub.get("thread_id") or None,
user_id=sub.get("user_id"),
profile=sub_profile or None,
)
# deliver_wake preserves the synthetic
# MessageEvent/handle_message path for
# push-capable adapters (the non-push /
# self-post branch is handled BEFORE the
# cursor advance above).
await deliver_wake(
adapter,
text=_synth,
session_id=_session_key,
source=_source,
)
logger.info(
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
)
except Exception as _wk_err:
# Best-effort: the notification itself already
# delivered and the cursor has advanced, so a

View file

@ -1335,7 +1335,7 @@ class APIServerAdapter(BasePlatformAdapter):
self._request_audit_log_suffix(request),
)
return web.json_response(
{"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": "invalid_api_key"}},
{"error": {"message": "Invalid gateway API key (API_SERVER_KEY)", "type": "gateway_auth_error", "code": "gateway_auth_failed"}},
status=401,
)
@ -5101,7 +5101,17 @@ class APIServerAdapter(BasePlatformAdapter):
# environment state.
approval_token = set_current_session_key(approval_session_key)
session_tokens = self._bind_api_server_session(
# chat_id carries the raw session id (the
# X-Hermes-Session-Id equivalent) exactly like
# the other agent-entry routes bind it via
# _run_agent(). Without it,
# tools.async_delegation reads an empty
# HERMES_SESSION_CHAT_ID on /v1/runs and
# background delegations stay forced-sync
# (no wake target).
chat_id=session_id or "",
session_key=approval_session_key,
session_id=session_id or "",
)
register_gateway_notify(approval_session_key, _approval_notify)
r = agent.run_conversation(
@ -5491,19 +5501,32 @@ class APIServerAdapter(BasePlatformAdapter):
try:
from hermes_cli.auth import has_usable_secret
if not has_usable_secret(self._api_key, min_length=16):
logger.error(
"[%s] Refusing to start: API_SERVER_KEY is a "
"placeholder or too short (<16 chars). This endpoint "
"dispatches terminal-capable agent work — a guessable "
"key is remote code execution. Generate a strong secret "
"(e.g. `openssl rand -hex 32`) and set API_SERVER_KEY "
"before starting the API server on %s.",
self.name, self._host,
)
return False
except ImportError:
pass
except Exception as exc:
# Fail CLOSED. This guard is the only thing between a guessable
# key and a terminal-capable endpoint, so "the check could not be
# run" must not resolve to "start anyway" — the same posture
# tools/credential_files.py takes when its deny-list cannot be
# consulted.
logger.error(
"[%s] Refusing to start: API_SERVER_KEY strength could not be "
"verified (%s: %s), and this endpoint dispatches "
"terminal-capable agent work. Repair the installation before "
"starting the API server on %s.",
self.name, type(exc).__name__, exc, self._host,
)
return False
if not has_usable_secret(self._api_key, min_length=16):
logger.error(
"[%s] Refusing to start: API_SERVER_KEY is a "
"placeholder or too short (<16 chars). This endpoint "
"dispatches terminal-capable agent work — a guessable "
"key is remote code execution. Generate a strong secret "
"(e.g. `openssl rand -hex 32`) and set API_SERVER_KEY "
"before starting the API server on %s.",
self.name, self._host,
)
return False
return True
async def connect(self, *, is_reconnect: bool = False) -> bool:
@ -5513,6 +5536,26 @@ class APIServerAdapter(BasePlatformAdapter):
return False
if not self._api_key_passes_startup_guard():
# A rejected API_SERVER_KEY is a configuration error, not a
# transient blip — the key will not become valid on its own. A
# bare ``return False`` makes the reconnect watcher in
# gateway.run treat it as retryable and loop forever at the
# backoff cap, re-instantiating the adapter (and its
# ResponseStore sqlite connection) every retry (#38803: ~501
# leaked connections / 1002 fds over 2.5 days until EMFILE took
# the whole gateway down). Non-retryable drops it from the
# reconnect queue — same treatment as the port-conflict guard
# (api_server_port_in_use). The guard already logged the
# specific rejection reason just above.
self._set_fatal_error(
"api_server_key_invalid",
"API_SERVER_KEY was rejected by the startup guard (missing, "
"placeholder/too short, or strength unverifiable — see the "
"error logged above). Generate a strong secret (e.g. "
"`openssl rand -hex 32`), set API_SERVER_KEY, then "
"`/platform resume api_server`.",
retryable=False,
)
return False
try:

View file

@ -105,6 +105,18 @@ def _reply_anchor_for_event(event) -> str | None:
source = getattr(event, "source", None)
platform = _platform_name(getattr(source, "platform", None))
thread_id = getattr(source, "thread_id", None)
raw_message = getattr(event, "raw_message", None)
if (
platform == "slack"
and isinstance(raw_message, dict)
and raw_message.get("_hermes_no_thread_response")
):
# Slack reaction handoffs into a configured target channel are meant
# to create a new top-level message there. Returning the synthetic
# event's message_id as reply_to would make
# SlackAdapter._resolve_thread_ts() treat it as a thread anchor and
# reply in a (nonexistent) thread anyway.
return None
if platform == "telegram" and thread_id and getattr(source, "chat_type", None) == "dm":
# Reply to the triggering user message. Replying to Telegram's earlier
# topic seed/anchor can render the bot response outside the active lane.
@ -749,14 +761,14 @@ async def cache_image_from_url(url: str, ext: str = ".jpg", retries: int = 2) ->
Raises:
ValueError: If the URL targets a private/internal network (SSRF protection).
"""
from tools.url_safety import is_safe_url
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
if not is_safe_url(url):
raise ValueError(f"Blocked unsafe URL (SSRF protection): {safe_url_for_log(url)}")
import httpx
_log = logging.getLogger(__name__)
async with httpx.AsyncClient(
async with create_ssrf_safe_async_client(
timeout=30.0,
follow_redirects=True,
event_hooks={"response": [_ssrf_redirect_guard]},
@ -869,14 +881,14 @@ async def cache_audio_from_url(url: str, ext: str = ".ogg", retries: int = 2) ->
Raises:
ValueError: If the URL targets a private/internal network (SSRF protection).
"""
from tools.url_safety import is_safe_url
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
if not is_safe_url(url):
raise ValueError(f"Blocked unsafe URL (SSRF protection): {safe_url_for_log(url)}")
import httpx
_log = logging.getLogger(__name__)
async with httpx.AsyncClient(
async with create_ssrf_safe_async_client(
timeout=30.0,
follow_redirects=True,
event_hooks={"response": [_ssrf_redirect_guard]},
@ -2443,6 +2455,11 @@ class BasePlatformAdapter(ABC):
self.config = config
self.platform = platform
self._message_handler: Optional[MessageHandler] = None
# Optional gateway-supplied fan-out for platform-native emoji
# reaction events (see ``set_reaction_handler``).
self._reaction_handler: Optional[
Callable[[Dict[str, Any]], Awaitable[None]]
] = None
# Optional hook (e.g. Telegram DM topic recovery) that rewrites
# ``event.source.thread_id`` before session keying. Returns the
# corrected thread_id or None to leave the source untouched.
@ -2988,6 +3005,25 @@ class BasePlatformAdapter(ABC):
"""Set an optional handler for messages arriving during active sessions."""
self._busy_session_handler = handler
def set_reaction_handler(
self, handler: Optional[Callable[[Dict[str, Any]], Awaitable[None]]]
) -> None:
"""Set the handler for emoji-reaction events on platform messages.
Called by adapters that subscribe to platform-native reaction events
(currently the Slack adapter's ``reaction_added``/``reaction_removed``).
The handler receives a normalised event dict ``platform``,
``event_name`` ("reaction:added"/"reaction:removed"), ``reaction``,
``user_id``, ``item_user_id``, ``channel_id``, ``message_ts``,
``event_ts``, ``raw_event`` and fans out via
``HookRegistry.emit(event_name, ...)``.
Adapters without reaction support simply never call the handler.
"""
# Assign defensively: subclasses initialized via ``object.__new__``
# in tests never run ``BasePlatformAdapter.__init__``.
self._reaction_handler = handler # type: ignore[attr-defined]
def set_authorization_check(
self,
callback: Optional[Callable[[str, Optional[str], Optional[str]], bool]],
@ -5781,6 +5817,7 @@ class BasePlatformAdapter(ABC):
user_id_alt=user_id_alt,
chat_id_alt=chat_id_alt,
is_bot=is_bot,
scope_id=str(scope_id) if scope_id else None,
guild_id=str(guild_id) if guild_id else None,
parent_chat_id=str(parent_chat_id) if parent_chat_id else None,
message_id=str(message_id) if message_id else None,
@ -5895,7 +5932,26 @@ class BasePlatformAdapter(ABC):
# Everything remaining fits in one final chunk
if _len(prefix) + _len(remaining) <= max_length - INDICATOR_RESERVE:
chunks.append(prefix + remaining)
final_chunk = prefix + remaining
# Check fence balance: if carry_lang was set, the chunk
# starts with an opening fence. Walk the remaining text
# to see if the code block was closed; if not, close it.
_final_in_code = carry_lang is not None
_final_lang = carry_lang or ""
if _final_in_code:
for _line in remaining.split("\n"):
_stripped = _line.strip()
if _stripped.startswith("```"):
if _final_in_code:
_final_in_code = False
_final_lang = ""
else:
_final_in_code = True
_tag = _stripped[3:].strip()
_final_lang = _tag.split()[0] if _tag else ""
if _final_in_code:
final_chunk += FENCE_CLOSE
chunks.append(final_chunk)
break
# Find a natural split point (prefer newlines, then spaces).

View file

@ -312,7 +312,8 @@ class QQAdapter(BasePlatformAdapter):
# Tighter keepalive pool so idle CLOSE_WAIT sockets drain
# faster behind proxies like Cloudflare Warp (#18451).
from gateway.platforms._http_client_limits import platform_httpx_limits
self._http_client = httpx.AsyncClient(
from tools.url_safety import create_ssrf_safe_async_client
self._http_client = create_ssrf_safe_async_client(
timeout=30.0,
follow_redirects=True,
event_hooks={"response": [_ssrf_redirect_guard]},

View file

@ -100,7 +100,7 @@ def _create_bind_task(timeout: float = ONBOARD_API_TIMEOUT) -> Tuple[str, str]:
if data.get("retcode") != 0:
raise RuntimeError(data.get("msg", "create_bind_task failed"))
task_id = data.get("data", {}).get("task_id")
task_id = (data.get("data") or {}).get("task_id")
if not task_id:
raise RuntimeError("create_bind_task: missing task_id in response")

View file

@ -2153,7 +2153,7 @@ class GroupAtGuardMiddleware(InboundMiddleware):
for elem in msg_body:
if elem.get("msg_type") != "TIMCustomElem":
continue
data_str = elem.get("msg_content", {}).get("data", "")
data_str = (elem.get("msg_content") or {}).get("data", "")
if not data_str:
continue
try:
@ -2172,7 +2172,7 @@ class GroupAtGuardMiddleware(InboundMiddleware):
for elem in msg_body:
if elem.get("msg_type") != "TIMCustomElem":
continue
data_str = elem.get("msg_content", {}).get("data", "")
data_str = (elem.get("msg_content") or {}).get("data", "")
if not data_str:
continue
try:

View file

@ -220,7 +220,7 @@ async def download_url(
# SSRF protection: yuanbao downloads model-supplied and inbound URLs
# server-side. Reject private/internal targets up front, and re-validate
# every redirect hop so a public URL can't 302 to http://169.254.169.254/.
from tools.url_safety import is_safe_url
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
if not is_safe_url(url):
raise ValueError(f"Blocked unsafe URL (SSRF protection): {url}")
@ -234,7 +234,7 @@ async def download_url(
)
max_bytes = max_size_mb * 1024 * 1024
async with httpx.AsyncClient(
async with create_ssrf_safe_async_client(
timeout=30.0,
follow_redirects=True,
event_hooks={"response": [_redirect_guard]},

View file

@ -529,10 +529,31 @@ async def _send_or_update_status_coro(adapter, chat_id, status_key, content, met
return await adapter.send(chat_id, content, metadata=metadata)
def _resolve_progress_thread_id(platform: Any, source_thread_id: Any, event_message_id: Any) -> Optional[str]:
"""Return thread/root ID that progress/status bubbles should target."""
def _resolve_progress_thread_id(
platform: Any,
source_thread_id: Any,
event_message_id: Any,
*,
reply_in_thread: bool = True,
) -> Optional[str]:
"""Return thread/root ID that progress/status bubbles should target.
``reply_in_thread=False`` (Slack ``platforms.slack.extra.reply_in_thread``)
disables the synthetic-thread fallback: progress messages must not create
a thread the final flat reply would then inherit. A source.thread_id equal
to the event's own message id is the adapter's synthetic session-keying
thread, not a real thread treat it as "no thread" too (#18859).
"""
platform_value = getattr(platform, "value", platform)
platform_key = str(platform_value or "").lower()
if not reply_in_thread:
if (
source_thread_id
and event_message_id
and str(source_thread_id) == str(event_message_id)
):
return None
return str(source_thread_id) if source_thread_id else None
if source_thread_id:
return str(source_thread_id)
if platform_key in {"slack", "mattermost"} and event_message_id:
@ -920,6 +941,53 @@ def _uses_telegram_observed_group_context(channel_prompt: Optional[str]) -> bool
return bool(channel_prompt and _TELEGRAM_OBSERVED_CONTEXT_PROMPT_MARKER in channel_prompt)
def _csv_or_list_to_set(raw: Any) -> set[str]:
"""Normalize a config list or comma-separated scalar into a string set."""
if raw is None:
return set()
if isinstance(raw, list):
return {str(part).strip() for part in raw if str(part).strip()}
s = str(raw).strip()
if not s:
return set()
return {part.strip() for part in s.split(",") if part.strip()}
def _slack_ignored_channels_from_gateway_config(config: Any) -> set[str]:
"""Return Slack channels that the generic gateway must never dispatch.
The Slack adapter has the first-line drop, but this runner-level guard is
intentionally duplicated as a fail-safe. If a future Slack code path, test
hook, malformed event, or stale adapter instance bypasses the Slack plugin
adapter, ignored channels still cannot reach auth, pairing, sessions, or
the agent/home-channel prompt pipeline.
"""
platform_cfg = getattr(config, "platforms", {}).get(Platform.SLACK)
raw = None
if platform_cfg is not None:
raw = getattr(platform_cfg, "extra", {}).get("ignored_channels")
if raw is None:
# Top-level ``slack.ignored_channels`` config flows through the
# plugin's YAML→env bridge (SLACK_IGNORED_CHANNELS) rather than
# PlatformConfig.extra — honor it here too (#46925).
raw = os.getenv("SLACK_IGNORED_CHANNELS") or None
return _csv_or_list_to_set(raw)
def _slack_parent_channel_id(chat_id: Any) -> str:
"""Return the parent Slack channel from a possibly thread-scoped chat ID."""
if not chat_id:
return ""
return str(chat_id).split(":", 1)[0]
def _is_slack_ignored_channel(config: Any, chat_id: Any) -> bool:
"""Check the generic Slack gateway blacklist for channel or thread IDs."""
channel_id = _slack_parent_channel_id(chat_id)
ignored = _slack_ignored_channels_from_gateway_config(config)
return bool(channel_id and ("*" in ignored or channel_id in ignored))
def _message_timestamps_enabled(user_config: Optional[dict]) -> bool:
"""True when gateway.message_timestamps.enabled is opted in.
@ -1951,6 +2019,7 @@ from gateway.session import (
SessionContext,
build_session_context,
build_session_context_prompt,
build_channel_continuity_note,
build_session_key,
is_shared_multi_user_session,
neutralize_untrusted_inline_text,
@ -4416,6 +4485,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
except Exception:
logger.debug("Failed to sync gateway session model metadata", exc_info=True)
async def _handle_reaction_event(self, ctx: Dict[str, Any]) -> None:
"""Fan a normalised platform reaction event out to the HookRegistry.
Adapters call this via ``set_reaction_handler`` for every
platform-native reaction event they surface. The adapter-supplied
``event_name`` ("reaction:added" / "reaction:removed") becomes the
hook event so user hooks subscribe with the same name scheme as the
existing ``agent:*`` family. Errors never block the adapter's event
loop the hook contract is non-blocking.
"""
event_name = str(ctx.get("event_name") or "reaction:added")
try:
await self.hooks.emit(event_name, ctx)
except Exception:
logger.debug("[Gateway] reaction hook emit failed", exc_info=True)
async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None:
"""React to an adapter failure after startup.
@ -6643,6 +6728,44 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# loop is never blocked; mirrors the /new reset path's fix (#35994).
_CLEANUP_TIMEOUT_S = 30.0
def _defer_agent_cleanup_until_future_done(
self,
future: asyncio.Future,
agent: Any,
*,
context: str,
) -> None:
"""Clean up ``agent`` only after its executor future has finished.
A timed-out executor call keeps running in its worker thread. Closing
the agent before that thread exits can tear down clients or providers
it is still using. Keep a strong task reference and wait for the real
future before invoking the normal bounded, off-loop cleanup path.
"""
async def _cleanup_when_done() -> None:
try:
await asyncio.shield(future)
except asyncio.CancelledError:
# Loop shutdown can cancel this waiter while the executor still
# runs. Never turn that cancellation into premature cleanup.
return
except Exception as exc:
logger.debug(
"Deferred agent worker%s finished with an error: %s",
f" ({context})" if context else "",
exc,
)
await self._cleanup_agent_resources_off_loop(agent, context=context)
task = asyncio.create_task(_cleanup_when_done())
tasks = getattr(self, "_deferred_agent_cleanup_tasks", None)
if tasks is None:
tasks = set()
self._deferred_agent_cleanup_tasks = tasks
tasks.add(task)
task.add_done_callback(tasks.discard)
async def _cleanup_agent_resources_off_loop(
self, agent: Any, *, context: str = ""
) -> None:
@ -6903,19 +7026,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# run as a self-restart loop guard and the gateway stays stopped.
watcher_env.pop("_HERMES_GATEWAY", None)
project_root = Path(__file__).resolve().parent.parent
# The watcher runs sys.executable (console python) under the
# CREATE_NO_WINDOW detach kwargs below: it owns one hidden
# console, inherited by the `hermes gateway restart` child, so
# nothing flashes. Do NOT swap in GUI-subsystem pythonw.exe —
# a console-less watcher forces every console-subsystem
# descendant to allocate a visible conhost (#54220/#56747).
watcher_python = sys.executable
try:
# Prefer a real GUI-subsystem interpreter for the watcher
# itself. With uv venvs, ``python.exe`` can re-exec the base
# console interpreter and flash even when the Popen carries
# CREATE_NO_WINDOW; pythonw.exe avoids console allocation.
from hermes_cli.gateway_windows import _resolve_detached_python
watcher_python, _watcher_venv_dir, _watcher_site_packages = (
_resolve_detached_python(sys.executable)
)
except Exception:
watcher_python = sys.executable
venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv")
site_packages = venv_dir / "Lib" / "site-packages"
if site_packages.exists():
@ -7837,6 +7954,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
adapter.set_fatal_error_handler(self._handle_adapter_fatal_error)
adapter.set_session_store(self.session_store)
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
_set_reaction = getattr(adapter, "set_reaction_handler", None)
if callable(_set_reaction):
_set_reaction(self._handle_reaction_event)
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform))
adapter._busy_text_mode = self._busy_text_mode
@ -8815,6 +8935,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
adapter.set_fatal_error_handler(self._handle_adapter_fatal_error)
adapter.set_session_store(self.session_store)
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
_set_reaction = getattr(adapter, "set_reaction_handler", None)
if callable(_set_reaction):
_set_reaction(self._handle_reaction_event)
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform))
adapter._busy_text_mode = self._busy_text_mode
@ -9689,6 +9812,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
)
adapter.set_session_store(self.session_store)
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
_set_reaction = getattr(adapter, "set_reaction_handler", None)
if callable(_set_reaction):
_set_reaction(self._handle_reaction_event)
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
adapter.set_authorization_check(
self._make_adapter_auth_check(platform, profile_name=profile_name)
@ -10110,6 +10236,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
return
config = getattr(self, "config", None)
if (
config
and getattr(source, "platform", None) == Platform.SLACK
and _is_slack_ignored_channel(config, getattr(source, "chat_id", None))
):
logger.info(
"Skipping Slack platform notice for configured ignored channel %s",
getattr(source, "chat_id", None),
)
return
notice_delivery = "public"
if config and hasattr(config, "get_notice_delivery"):
notice_delivery = config.get_notice_delivery(source.platform)
@ -10316,18 +10453,37 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
except Exception:
logger.debug("reset_session_vars failed at handler entry", exc_info=True)
# Internal events (e.g. background-process completion notifications)
# are system-generated and must skip user authorization.
is_internal = bool(getattr(event, "internal", False))
# Ignored-channel guard runs FIRST — before startup-restore queueing,
# plugin hooks, auth, and session setup — so a configured ignored
# channel can never reach pairing/auth/session state (#51899).
# getattr: bare test runners construct GatewayRunner via
# object.__new__ without config (see AGENTS.md pitfall on
# object.__new__ test pattern).
if (
not is_internal
and getattr(source, "platform", None) == Platform.SLACK
and _is_slack_ignored_channel(
getattr(self, "config", None), getattr(source, "chat_id", None)
)
):
logger.info(
"Dropping Slack message from configured ignored channel %s",
getattr(source, "chat_id", None),
)
return None
if (
getattr(self, "_startup_restore_in_progress", False)
and not getattr(event, "internal", False)
and not is_internal
and not getattr(event, "_hermes_startup_restore_replay", False)
):
self._queue_startup_restore_event(event)
return None
# Internal events (e.g. background-process completion notifications)
# are system-generated and must skip user authorization.
is_internal = bool(getattr(event, "internal", False))
# scale-to-zero (Phase 0, 0.B/F13): stamp the gateway-scoped last-inbound
# clock for real (user-originated) inbound only. Internal/system events
# (background-process completions, startup-restore replays) are NOT
@ -12532,6 +12688,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
context_note = "[System note: The previous gateway session could not be recovered after a restart (API recovery timed out). This is a fresh conversation — use /resume to restore history if needed.]"
else:
context_note = "[System note: The user's previous session expired due to inactivity. This is a fresh conversation with no prior context.]"
# Slack/Discord channels/threads are long-lived: point the agent at
# the specific prior same-channel session so it recalls that context
# via session_search instead of an unrelated recent session. Returns
# None (appends nothing) for other platforms or when there's no prior
# activity to recall. Deterministic — no extra API/DB calls (#36220).
try:
continuity_note = build_channel_continuity_note(session_entry, source)
except Exception:
continuity_note = None
if continuity_note:
context_note = context_note + "\n\n" + continuity_note
turn_sidecar_notes.append(context_note)
# Send a user-facing notification explaining the reset, unless:
@ -12692,6 +12859,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_hyg_threshold_pct = 0.85
_hyg_compression_enabled = True
_hyg_hard_msg_limit = 5000
_hyg_timeout_seconds = 30.0
_hyg_failure_cooldown_seconds = 300.0
_hyg_config_context_length = None
_hyg_provider = None
_hyg_base_url = None
@ -12737,6 +12906,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_hyg_hard_msg_limit = _parsed
except (TypeError, ValueError):
pass
_raw_timeout = _comp_cfg.get("hygiene_timeout_seconds")
if _raw_timeout is not None:
try:
_parsed = float(_raw_timeout)
if _parsed > 0:
_hyg_timeout_seconds = _parsed
except (TypeError, ValueError):
pass
_raw_cooldown = _comp_cfg.get("hygiene_failure_cooldown_seconds")
if _raw_cooldown is not None:
try:
_parsed = float(_raw_cooldown)
if _parsed >= 0:
_hyg_failure_cooldown_seconds = _parsed
except (TypeError, ValueError):
pass
_hyg_configured_model = _hyg_model
_hyg_configured_provider = _hyg_provider
@ -12847,6 +13032,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
or _msg_count >= _HARD_MSG_LIMIT
)
if _needs_compress:
_cooldowns = getattr(self, "_hygiene_compression_failure_cooldowns", None)
if _cooldowns is None:
_cooldowns = {}
self._hygiene_compression_failure_cooldowns = _cooldowns
_cooldown_key = session_entry.session_id
_cooldown_until = float(_cooldowns.get(_cooldown_key) or 0.0)
if _cooldown_until > time.time():
logger.info(
"Session hygiene: skipping compression for %s; "
"previous failure cooldown active for %.1fs",
_cooldown_key,
max(0.0, _cooldown_until - time.time()),
)
_needs_compress = False
if _needs_compress:
logger.info(
"Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing "
@ -12860,6 +13061,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_hyg_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
try:
from agent.conversation_compression import CompressionCommitFence
from run_agent import AIAgent
_hyg_model, _hyg_runtime = self._resolve_session_agent_runtime(
@ -12894,6 +13096,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
session_id=session_entry.session_id,
session_db=_hyg_session_db,
)
_hyg_cleanup_deferred = False
try:
# Gateway hygiene runs before the user turn
# starts and already owns the session binding.
@ -12921,13 +13124,76 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_hyg_agent._print_fn = lambda *a, **kw: None
loop = asyncio.get_running_loop()
_compressed, _ = await loop.run_in_executor(
_hyg_commit_fence = CompressionCommitFence()
_hyg_future = loop.run_in_executor(
None,
lambda: _hyg_agent._compress_context(
_hyg_msgs, "",
approx_tokens=_approx_tokens,
commit_fence=_hyg_commit_fence,
),
)
try:
_compressed, _ = await asyncio.wait_for(
asyncio.shield(_hyg_future),
timeout=_hyg_timeout_seconds,
)
except asyncio.TimeoutError:
_cancelled = None
while _cancelled is None:
_cancelled = (
_hyg_commit_fence.try_cancel_before_commit()
)
if _cancelled is None:
await asyncio.sleep(0.001)
if not _cancelled:
# The worker crossed the commit boundary just
# before the timeout. The fence poll waited for
# that boundary to finish, so consume the
# completed result instead of treating a
# successful compaction as a timeout.
_compressed, _ = await _hyg_future
else:
self._defer_agent_cleanup_until_future_done(
_hyg_future,
_hyg_agent,
context="session hygiene timeout",
)
_hyg_cleanup_deferred = True
if _hyg_failure_cooldown_seconds >= 0:
self._hygiene_compression_failure_cooldowns[
session_entry.session_id
] = time.time() + _hyg_failure_cooldown_seconds
logger.warning(
"Session hygiene compression for session %s "
"timed out after %.1fs; continuing without "
"compression",
session_entry.session_id,
_hyg_timeout_seconds,
)
_timeout_msg = (
"⚠️ Context compression timed out "
f"after {_hyg_timeout_seconds:.1f}s. "
"No messages were dropped — continuing without "
"compression. Run /compress to retry, /reset for "
"a clean session, or check your "
"auxiliary.compression model configuration."
)
try:
_adapter = self._adapter_for_source(source)
if _adapter and source.chat_id:
await _adapter.send(
source.chat_id,
_timeout_msg,
metadata=_hyg_meta,
)
except Exception as _werr:
logger.warning(
"Failed to deliver compression-timeout "
"warning to user: %s",
_werr,
)
raise
# _compress_context ends the old session and creates
# a new session_id. Write compressed messages into
@ -13035,6 +13301,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# fresh.
_comp = getattr(_hyg_agent, "context_compressor", None)
if _comp is not None and getattr(_comp, "_last_compress_aborted", False):
if _hyg_failure_cooldown_seconds >= 0:
self._hygiene_compression_failure_cooldowns[
session_entry.session_id
] = time.time() + _hyg_failure_cooldown_seconds
_err = getattr(_comp, "_last_summary_error", None) or "unknown error"
# Force-redact: provider exception text
# may contain credentials; this message
@ -13087,9 +13357,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# rebuilds its system prompt from current
# SOUL.md, memory, and skills.
self._evict_cached_agent(session_key)
await self._cleanup_agent_resources_off_loop(
_hyg_agent, context="session hygiene"
)
if not _hyg_cleanup_deferred:
await self._cleanup_agent_resources_off_loop(
_hyg_agent, context="session hygiene"
)
except Exception as e:
logger.warning(
@ -13475,6 +13746,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
if _show_reasoning_effective and response and not _intentional_silence:
last_reasoning = agent_result.get("last_reasoning")
if last_reasoning:
from gateway.stream_consumer import escape_code_fences_for_display
# Collapse long reasoning to keep messages readable
lines = last_reasoning.strip().splitlines()
if len(lines) > 15:
@ -13506,6 +13778,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
)
response = f"> 💭 **Reasoning:**\n{_quoted}\n\n{response}"
else:
# Escape ``` inside reasoning so inner fences don't
# break the outer code block used to render it.
display_reasoning = escape_code_fences_for_display(display_reasoning)
response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}"
# Runtime-metadata footer — only on the FINAL message of the turn.
@ -14843,7 +15118,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
has_agent_tts = any(
msg.get("role") == "assistant"
and any(
tc.get("function", {}).get("name") == "text_to_speech"
(tc.get("function") or {}).get("name") == "text_to_speech"
for tc in (msg.get("tool_calls") or [])
)
for msg in agent_messages
@ -16141,6 +16416,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
metadata["direct_messages_topic_id"] = tid
if reply_to_message_id is not None:
metadata["telegram_reply_to_message_id"] = str(reply_to_message_id)
if platform == Platform.SLACK and reply_to_message_id is not None:
# Slack's reply_in_thread=false path uses message_id to distinguish
# real existing threads from synthetic top-level session keys.
metadata["message_id"] = str(reply_to_message_id)
return metadata
@staticmethod
@ -17243,6 +17522,43 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
"""
source = self._build_process_event_source(evt)
if not source:
# API-server-originated sessions bind a RAW session key (the
# X-Hermes-Session-Id value — see _bind_api_server_session), not a
# structured ``agent:main:...`` key, so _build_process_event_source
# cannot derive routing metadata from it and returns None above.
# Recover the raw session id and wake the real session via the API
# server's own /v1/chat/completions entry point instead of
# dropping the event.
raw_sid = str(evt.get("origin_session_id") or "").strip()
if not raw_sid:
_sk = str(evt.get("session_key") or "").strip()
if _sk and _parse_session_key(_sk) is None:
raw_sid = _sk
if raw_sid:
adapter = self.adapters.get(Platform.API_SERVER)
from gateway.wake import adapter_supports_push, deliver_wake
if adapter is not None and not adapter_supports_push(adapter):
try:
logger.info(
"Watch pattern notification — waking api_server "
"session %s via self-post",
raw_sid,
)
await deliver_wake(adapter, text=synth_text, session_id=raw_sid)
return True
except Exception as e:
logger.warning(
"Watch notification self-post wake failed for "
"session %s: %s",
raw_sid, e,
)
return False
logger.warning(
"Dropping watch notification for raw session %s: no "
"api_server adapter to self-post through",
raw_sid,
)
return None
logger.warning(
"Dropping watch notification with no routing metadata for process %s",
evt.get("session_id", "unknown"),
@ -17256,6 +17572,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
break
if not adapter:
return None
from gateway.wake import adapter_supports_push as _wake_push_ok
if not _wake_push_ok(adapter):
# Non-push adapter (api_server) resolved WITH routing metadata:
# its chat_id is the raw session id (see _bind_api_server_session,
# which binds chat_id = session_id). handle_message would run the
# wake under a build_session_key()-derived key that never matches
# the raw X-Hermes-Session-Id session — self-post instead.
from gateway.wake import deliver_wake
raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "")
try:
logger.info(
"Watch pattern notification — waking api_server session "
"%s via self-post",
raw_sid,
)
await deliver_wake(adapter, text=synth_text, session_id=raw_sid)
return True
except Exception as e:
logger.warning(
"Watch notification self-post wake failed for session "
"%s: %s",
raw_sid, e,
)
return False
try:
metadata = {}
parent_session_id = str(evt.get("parent_session_id") or "").strip()
@ -17665,6 +18005,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
break
# --- Normal text-only notification ---
# Skip when the agent already consumed this completion via
# wait/log (#65379): process(wait) returned the exit code and
# output inline, so the raw "[Background process ... finished
# with exit code ...]" message would be a duplicate delivery
# of the same completion. The agent_notify branch above
# already honors _completion_consumed; without this check its
# skip FALLS THROUGH to this block and re-delivers the output
# the agent is actively summarizing. poll() is read-only and
# intentionally does not mark consumed (#10156), so a status
# check never suppresses this message.
if _pr_check.is_completion_consumed(session_id):
logger.debug(
"Process watcher: completion for %s already consumed "
"via wait/log — skipping raw notification (#65379)",
session_id,
)
break
# Decide whether to notify based on mode
should_notify = (
notify_mode in {"all", "result"}
@ -18503,6 +18860,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
"1" if src.message_id else "0",
)
# Slack renders a capability-aware platform note gated on
# _slack_tools_loaded() — the gate state must appear in the key
# (same parity contract as the Discord gate above) so a config /
# MCP-registration flip re-renders once instead of serving a
# stale pinned note for the rest of the session.
slack_tools = ""
if src.platform == Platform.SLACK:
from gateway.session import _slack_tools_loaded
slack_tools = "1" if _slack_tools_loaded() else "0"
try:
from hermes_constants import display_hermes_home
@ -18523,6 +18891,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
bool(context.shared_multi_user_session),
discord_ids,
discord_tools,
slack_tools,
tuple(p.value for p in context.connected_platforms),
tuple(
(
@ -19898,13 +20267,38 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# - Feishu only honors reply_in_thread when sending a reply, so topic
# progress uses the triggering event message as the reply target
# - Other platforms should use explicit source.thread_id only
#
# Slack honours platforms.slack.extra.reply_in_thread=false: if the
# user has opted out of threaded replies, don't synthesise a thread
# for progress messages either — the very first progress message
# would otherwise create a thread that all subsequent replies
# (including the final answer) would inherit (#18859).
_progress_reply_in_thread = True
if source.platform == Platform.SLACK:
_slack_adapter_for_progress = self._adapter_for_source(source)
if _slack_adapter_for_progress is not None:
try:
_progress_reply_in_thread = bool(
_slack_adapter_for_progress.config.extra.get(
"reply_in_thread", True
)
)
except Exception:
_progress_reply_in_thread = True
_progress_thread_id = _resolve_progress_thread_id(
source.platform, source.thread_id, event_message_id,
reply_in_thread=_progress_reply_in_thread,
)
_progress_metadata = (
self._thread_metadata_for_source(source, event_message_id)
if _progress_thread_id == source.thread_id
else {"thread_id": _progress_thread_id}
else self._thread_metadata_for_target(
source.platform,
source.chat_id,
_progress_thread_id,
chat_type=getattr(source, "chat_type", None),
reply_to_message_id=event_message_id,
)
) if _progress_thread_id else None
_progress_metadata = _non_conversational_metadata(_progress_metadata, platform=source.platform)
_progress_reply_to = (
@ -20076,8 +20470,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
async def _roll_progress_overflow_if_needed() -> bool:
"""Start fresh editable progress bubbles before a bubble exceeds limit.
Returns True when it delivered/split the current buffer and the
caller should skip the normal send/edit path for this tick.
Returns True when it delivered/split the current buffer, or when
a transient edit failure left the buffer and message identity
intact for a later retry. In either case the caller should skip
the normal send/edit path for this tick.
"""
nonlocal progress_msg_id, progress_lines, can_edit
if not progress_lines or not can_edit:
@ -20090,6 +20486,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
if progress_msg_id is not None:
result = await _edit_progress_message(progress_msg_id, first_text)
if not result.success:
if getattr(result, "retryable", False):
logger.debug(
"[%s] Transient overflow edit failure — keeping can_edit=True",
adapter.name,
)
return True
can_edit = False
# Fall back to the existing non-edit behavior below.
return False
@ -20359,7 +20761,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
"reply_to_message_id": event_message_id,
}
else:
_status_thread_metadata = self._thread_metadata_for_source(source, event_message_id) if _progress_thread_id else None
_status_thread_metadata = (
self._thread_metadata_for_source(source, event_message_id)
if _progress_thread_id == source.thread_id
else self._thread_metadata_for_target(
source.platform,
source.chat_id,
_progress_thread_id,
chat_type=getattr(source, "chat_type", None),
reply_to_message_id=event_message_id,
)
) if _progress_thread_id else None
def _status_callback_sync(event_type: str, message: str) -> None:
if not _status_adapter or not _run_still_current():

View file

@ -17,7 +17,7 @@ import threading
import uuid
from pathlib import Path
from datetime import datetime, timedelta
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from typing import Dict, List, Optional, Any
logger = logging.getLogger(__name__)
@ -342,6 +342,52 @@ that requires raw IDs). Discord is excluded because mentions use ``<@user_id>``
and the LLM needs the real ID to tag users."""
def _slack_tools_loaded() -> bool:
"""True iff the agent will actually have Slack tools this session.
Two independent paths grant Slack capability:
1. Native `slack` toolset enabled via `hermes tools` (opt-in, default
OFF) AND `SLACK_BOT_TOKEN` set the tool's `check_fn` gates on it
at registry time, so config alone isn't enough.
2. An MCP server that has ACTUALLY registered tools into the live
registry (tools/mcp_tool.get_registered_mcp_server_names()), whose
name suggests Slack. This is the real, availability-filtered
signal (post-connection, post include/exclude filtering) rather
than just what's listed in config.yaml -- a configured-but-
unconnected or zero-tool MCP server must not claim capability.
Named MCP servers are process-wide (one gateway connects each MCP
server once, not per-session), so this check is intentionally NOT
scoped further per-session -- unlike the earlier get_all_tool_names()
approach this replaces, which conflated ALL built-in tool names
process-wide, this only inspects the small, purpose-built MCP
server-name map.
Returns False (safe default keeps the stale-API disclaimer) on any
error so a bad config can never silently promise tools the agent lacks.
"""
try:
from tools.mcp_tool import get_registered_mcp_server_names
if any("slack" in name.lower() for name in get_registered_mcp_server_names()):
return True
except Exception:
pass
if not (os.environ.get("SLACK_BOT_TOKEN") or "").strip():
return False
try:
from hermes_cli.config import load_config
from hermes_cli.tools_config import _get_platform_tools
cfg = load_config()
# include_default_mcp_servers=True (the default) so a Slack MCP
# server that's enabled by default for this platform (not
# explicitly listed) is also counted, in addition to the native
# 'slack' toolset.
enabled = _get_platform_tools(cfg, "slack")
return "slack" in enabled
except Exception:
return False
def _discord_tools_loaded() -> bool:
"""True iff the agent will actually have Discord tools this session.
@ -517,15 +563,31 @@ def build_session_context_prompt(
# Platform-specific behavioral notes
if context.source.platform == Platform.SLACK:
lines.append("")
lines.append(
"**Platform notes:** You are running inside Slack. "
"You do NOT have access to Slack-specific APIs — you cannot search "
"channel history, pin/unpin messages, manage channels, or list users. "
"Do not promise to perform these actions. The gateway may inline the "
"current message's Slack block/attachment payload when available, but "
"you still cannot call Slack APIs yourself."
)
# Inject the Slack capability note only when the agent actually has
# Slack tools loaded this session — native `slack` toolset opt-in,
# or a connected MCP server that has registered Slack tools.
# Otherwise keep the stale-API disclaimer honest so we never
# promise tools the agent lacks. Mirrors the Discord pattern below.
if _slack_tools_loaded():
lines.append("")
lines.append(
"**Platform notes:** You are running inside Slack and have access "
"to Slack-specific tools this session. Consult the available Slack "
"tool schemas for the exact operations supported (e.g. channel "
"history and thread lookups, posting, reactions) — use those tools "
"for Slack-specific requests, and do not promise Slack actions "
"beyond what the loaded tools actually expose."
)
else:
lines.append("")
lines.append(
"**Platform notes:** You are running inside Slack. "
"You do NOT have access to Slack-specific APIs — you cannot search "
"channel history, pin/unpin messages, manage channels, or list users. "
"Do not promise to perform these actions. The gateway may inline the "
"current message's Slack block/attachment payload when available, but "
"you still cannot call Slack APIs yourself."
)
if context.shared_multi_user_session:
lines.append(
"In shared Slack threads, use the current turn's sender prefix "
@ -722,6 +784,13 @@ class SessionEntry:
auto_reset_reason: Optional[str] = None # "idle" or "daily"
reset_had_activity: bool = False # whether the expired session had any messages
# When this session was created by an auto-reset, the session_id of the
# session it replaced. Used to give Slack/Discord channels/threads a
# lightweight continuity hint (see build_channel_continuity_note) so the
# agent recalls the prior same-channel session via session_search instead
# of binding the request to an unrelated recent session.
prev_session_id: Optional[str] = None
# Set by reset_session() when the user explicitly sends /new or /reset.
# Consumed once by _handle_message_with_agent to trigger topic/channel
# skill re-injection on the first message of the new session. We can't
@ -794,6 +863,7 @@ class SessionEntry:
"was_auto_reset": self.was_auto_reset,
"auto_reset_reason": self.auto_reset_reason,
"reset_had_activity": self.reset_had_activity,
"prev_session_id": self.prev_session_id,
}
if self.model_override:
# Defence-in-depth: strip credentials even if a caller stored an
@ -870,10 +940,51 @@ class SessionEntry:
was_auto_reset=data.get("was_auto_reset", False),
auto_reset_reason=data.get("auto_reset_reason"),
reset_had_activity=data.get("reset_had_activity", False),
prev_session_id=data.get("prev_session_id"),
model_override=sanitize_model_override(data.get("model_override")),
)
def build_channel_continuity_note(
entry: "SessionEntry",
source: SessionSource,
) -> Optional[str]:
"""Build a lightweight session-continuity hint for Slack/Discord channels.
Slack and Discord channels/threads are long-lived: when the daily/idle
reset policy starts a fresh session, the agent loses the thread's prior
context and can mistakenly bind a new request to an unrelated recent
session. This deterministic one-line hint points the agent at the
specific prior session in *this* channel/thread so it recalls that
context via ``session_search`` before acting.
Returns ``None`` (and the caller adds nothing) unless **all** hold:
- the source platform is Slack or Discord,
- this session was created by an auto-reset that had real activity,
- the previous session_id was recorded on the entry.
No LLM calls, no extra API/DB lookups the previous session id is
already known from :meth:`SessionStore.get_or_create_session`.
"""
if source.platform not in (Platform.SLACK, Platform.DISCORD):
return None
if not getattr(entry, "reset_had_activity", False):
return None
prev = getattr(entry, "prev_session_id", None)
if not prev:
return None
where = "thread" if source.thread_id else "channel"
return (
f"[System note: This {where} had an earlier Hermes session "
f"(session_id: {prev}) that was auto-reset. If the user refers to "
f"earlier work here, or the request depends on this {where}'s history, "
f"use the session_search tool to recall that prior session before "
f"acting — do not assume an unrelated recent session is the right "
f"context.]"
)
def is_shared_multi_user_session(
source: SessionSource,
*,
@ -931,12 +1042,16 @@ def build_session_key(
multiplexing gateway passes a non-default profile.
DM rules:
- Slack ``scope_id`` identifies the workspace before chat/user ids. Other
platforms retain their existing key format; in particular, Discord
guild scope is intentionally not added here as a compatibility change.
- DMs include chat_id when present, so each private conversation is isolated.
- thread_id further differentiates threaded DMs within the same DM chat.
- Without chat_id, thread_id is used as a best-effort fallback.
- Without thread_id or chat_id, DMs share a single session.
Group/channel rules:
- Slack ``scope_id`` identifies the workspace before chat/thread ids.
- chat_id identifies the parent group/channel.
- user_id/user_id_alt isolates participants within that parent chat when available when
``group_sessions_per_user`` is enabled.
@ -951,15 +1066,24 @@ def build_session_key(
"""
ns = _session_key_namespace(profile)
platform = source.platform.value
slack_scope_id = (
str(source.scope_id)
if source.platform == Platform.SLACK and source.scope_id
else None
)
if source.chat_type == "dm":
dm_chat_id = source.chat_id
if source.platform == Platform.WHATSAPP:
dm_chat_id = canonical_whatsapp_identifier(source.chat_id)
dm_parts = [ns, platform, "dm"]
if slack_scope_id:
dm_parts.append(slack_scope_id)
if dm_chat_id:
dm_parts.append(dm_chat_id)
if source.thread_id:
return f"{ns}:{platform}:dm:{dm_chat_id}:{source.thread_id}"
return f"{ns}:{platform}:dm:{dm_chat_id}"
dm_parts.append(source.thread_id)
return ":".join(str(part) for part in dm_parts)
# No chat_id — fall back to the sender's own identifier before the
# bare per-platform sink. Without this, every DM from every user that
# arrives without a chat_id (non-standard adapters / synthetic sources)
@ -973,12 +1097,13 @@ def build_session_key(
or dm_participant_id
)
if dm_participant_id:
dm_parts.append(str(dm_participant_id))
if source.thread_id:
return f"{ns}:{platform}:dm:{dm_participant_id}:{source.thread_id}"
return f"{ns}:{platform}:dm:{dm_participant_id}"
dm_parts.append(source.thread_id)
return ":".join(str(part) for part in dm_parts)
if source.thread_id:
return f"{ns}:{platform}:dm:{source.thread_id}"
return f"{ns}:{platform}:dm"
dm_parts.append(source.thread_id)
return ":".join(str(part) for part in dm_parts)
participant_id = source.user_id_alt or source.user_id
if participant_id and source.platform == Platform.WHATSAPP:
@ -988,6 +1113,8 @@ def build_session_key(
participant_id = canonical_whatsapp_identifier(str(participant_id)) or participant_id
key_parts = [ns, platform, source.chat_type]
if slack_scope_id:
key_parts.append(slack_scope_id)
if source.chat_id:
key_parts.append(source.chat_id)
if source.thread_id:
@ -1003,7 +1130,7 @@ def build_session_key(
if isolate_user and participant_id:
key_parts.append(str(participant_id))
return ":".join(key_parts)
return ":".join(str(part) for part in key_parts)
class _SessionFlight:
@ -1053,6 +1180,11 @@ class SessionStore:
self._persisted_routing_generation = 0
self._inflight_lock = threading.Lock()
self._inflight_sessions: Dict[str, _SessionFlight] = {}
# An unscoped pre-migration Slack key can represent at most one
# workspace. Claim it once per process so simultaneous first messages
# from two workspaces cannot both revive the same legacy session.
self._legacy_slack_claim_lock = threading.Lock()
self._claimed_legacy_slack_keys: set[str] = set()
self._transcript_retry_lock = threading.Lock()
self._dirty_transcripts: Dict[str, List[Dict[str, Any]]] = {}
self._transcript_append_failures: Dict[str, int] = {}
@ -1436,6 +1568,74 @@ class SessionStore:
profile=self._resolve_profile_for_key(source),
)
def _legacy_slack_session_key(self, source: SessionSource) -> Optional[str]:
"""Return the pre-workspace Slack key for an explicitly scoped source.
The compatibility path is deliberately Slack-only. Discord and every
other platform keep byte-identical keys, and an unscoped Slack session
may be claimed by only one workspace because its old key contains no
information that could safely distinguish multiple teams.
"""
if source.platform != Platform.SLACK or not source.scope_id:
return None
legacy_source = replace(source, scope_id=None, guild_id=None)
return build_session_key(
legacy_source,
group_sessions_per_user=getattr(
self.config, "group_sessions_per_user", True
),
thread_sessions_per_user=getattr(
self.config, "thread_sessions_per_user", False
),
profile=self._resolve_profile_for_key(source),
)
def _claim_legacy_slack_key(self, legacy_key: Optional[str]) -> bool:
"""Atomically reserve one ambiguous legacy Slack key for migration."""
if not legacy_key:
return False
claim_lock = getattr(self, "_legacy_slack_claim_lock", None)
if claim_lock is None:
claim_lock = threading.Lock()
self._legacy_slack_claim_lock = claim_lock
with claim_lock:
claimed = getattr(self, "_claimed_legacy_slack_keys", None)
if claimed is None:
claimed = set()
self._claimed_legacy_slack_keys = claimed
if legacy_key in claimed:
return False
claimed.add(legacy_key)
return True
@staticmethod
def _recovered_row_matches_source_scope(
recovered: Dict[str, Any], source: SessionSource
) -> bool:
"""Reject recovered rows whose recorded origin belongs to another workspace.
Slack group/channel rows recorded with an origin_json carry the
workspace (scope_id) they were created under. A workspace-scoped
lookup must not adopt a row another team recorded even via the
legacy-key fallback unless the recorded origin names the same
workspace. Rows without a parseable origin are rejected for scoped
sources: an unattributable transcript is precisely the ambiguity
this guard exists to avoid.
"""
if (
source.platform != Platform.SLACK
or source.chat_type == "dm"
or not source.scope_id
):
return True
try:
origin = json.loads(recovered.get("origin_json") or "")
except (TypeError, ValueError):
return False
if not isinstance(origin, dict):
return False
return origin.get("scope_id", origin.get("guild_id")) == source.scope_id
def _create_entry_from_recovered_row(
self,
*,
@ -1460,6 +1660,45 @@ class SessionStore:
chat_type=source.chat_type,
)
def _find_gateway_session_row(
self,
*,
session_key: str,
source: SessionSource,
allow_peer_fallback: bool,
raise_on_lookup_error: bool = False,
) -> Optional[Dict[str, Any]]:
"""Query one durable gateway session row.
Scoped Slack lookups disable SessionDB's platform/chat/user fallback:
that tuple does not contain a workspace id and could therefore revive
another team's session. The caller performs one explicit exact lookup
of the old unscoped key instead.
"""
if not self._db:
return None
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
if not callable(finder):
return None
try:
return finder(
source=source.platform.value,
user_id=source.user_id,
session_key=session_key,
chat_id=source.chat_id if allow_peer_fallback else None,
chat_type=source.chat_type if allow_peer_fallback else None,
thread_id=source.thread_id,
)
except Exception as exc:
logger.debug(
"Gateway session DB recovery failed for %s: %s",
session_key,
exc,
)
if raise_on_lookup_error:
raise
return None
def _recover_session_from_db(
self,
*,
@ -1469,27 +1708,30 @@ class SessionStore:
raise_on_lookup_error: bool = False,
) -> Optional[SessionEntry]:
"""Rebuild a missing session-key mapping from durable state.db data."""
if not self._db:
return None
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
if not callable(finder):
return None
try:
recovered = finder(
source=source.platform.value,
user_id=source.user_id,
session_key=session_key,
chat_id=source.chat_id,
chat_type=source.chat_type,
thread_id=source.thread_id,
legacy_key = self._legacy_slack_session_key(source)
recovered = self._find_gateway_session_row(
session_key=session_key,
source=source,
allow_peer_fallback=legacy_key is None,
raise_on_lookup_error=raise_on_lookup_error,
)
migrated_legacy = False
if (
not recovered
and legacy_key
and self._claim_legacy_slack_key(legacy_key)
):
recovered = self._find_gateway_session_row(
session_key=legacy_key,
source=source,
allow_peer_fallback=False,
raise_on_lookup_error=raise_on_lookup_error,
)
except Exception as exc:
logger.debug("Gateway session DB recovery failed for %s: %s", session_key, exc)
if raise_on_lookup_error:
raise
return None
migrated_legacy = bool(recovered)
if not recovered:
return None
if not self._recovered_row_matches_source_scope(recovered, source):
return None
if not self._recovered_row_allowed_for_active_profile(
requested_session_key=session_key,
recovered=recovered,
@ -1506,38 +1748,50 @@ class SessionStore:
self._db.reopen_session(str(recovered["id"]))
except Exception as exc:
logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc)
return self._create_entry_from_recovered_row(
entry = self._create_entry_from_recovered_row(
row=recovered,
session_key=session_key,
source=source,
now=now,
)
if migrated_legacy:
self._record_gateway_session_peer(
entry.session_id,
session_key,
source,
display_name=entry.display_name,
)
return entry
def _query_recoverable_session(self, *, session_key, source, now):
def _query_recoverable_session(
self, *, session_key, source, now, lookup_session_key=None
):
"""DB-only half of _recover_session_from_db (no lock needed).
Returns a SessionEntry or None. Caller assigns _entries[key] under lock.
"""
if not self._db:
return None
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
if not callable(finder):
return None
try:
recovered = finder(
source=source.platform.value,
user_id=source.user_id,
session_key=session_key,
chat_id=source.chat_id,
chat_type=source.chat_type,
thread_id=source.thread_id,
legacy_key = self._legacy_slack_session_key(source)
recovered = self._find_gateway_session_row(
session_key=session_key,
source=source,
allow_peer_fallback=legacy_key is None,
)
migrated_legacy = False
if (
not recovered
and legacy_key
and self._claim_legacy_slack_key(legacy_key)
):
recovered = self._find_gateway_session_row(
session_key=legacy_key,
source=source,
allow_peer_fallback=False,
)
except Exception as exc:
logger.debug("Gateway session DB recovery failed for %s: %s",
session_key, exc)
return None
migrated_legacy = bool(recovered)
if not isinstance(recovered, dict):
return None
if not self._recovered_row_matches_source_scope(recovered, source):
return None
if not self._recovered_row_allowed_for_active_profile(
requested_session_key=session_key,
recovered=recovered,
@ -1555,9 +1809,17 @@ class SessionStore:
except Exception as exc:
logger.debug("Gateway session DB reopen failed for %s: %s",
session_key, exc)
return self._create_entry_from_recovered_row(
entry = self._create_entry_from_recovered_row(
row=recovered, session_key=session_key, source=source, now=now,
)
if migrated_legacy:
self._record_gateway_session_peer(
entry.session_id,
session_key,
source,
display_name=entry.display_name,
)
return entry
def _record_gateway_session_peer(
self,
session_id: str,
@ -1921,6 +2183,52 @@ class SessionStore:
session_key = self._generate_session_key(source)
now = _now()
# One-time routing-index migration for Slack sessions created before
# workspace scope was part of the key. Move (rather than copy) the
# legacy entry so a second workspace with identical Slack ids cannot
# attach to the same transcript.
#
# Adoption policy (composed from #20583/#66398 and #68925):
# - The legacy entry's recorded origin names a workspace → migrate
# only when it matches the incoming workspace (precise).
# - Scope-less origin, DM → first workspace claims it once
# (claim-once): a 1:1 DM has a single human peer, so continuity
# across the key-format change outweighs the ambiguity risk.
# - Scope-less origin, channel/group → refuse: channel ids collide
# across workspaces and a shared transcript leaking to a second
# tenant is exactly the bug this fix removes.
migrated_legacy_entry: Optional[SessionEntry] = None
legacy_key = self._legacy_slack_session_key(source)
if legacy_key and not force_new:
with self._lock:
self._ensure_loaded_locked()
legacy_entry = self._entries.get(legacy_key)
if session_key not in self._entries and legacy_entry is not None:
origin_scope = (
getattr(legacy_entry.origin, "scope_id", None)
if legacy_entry.origin is not None
else None
)
if origin_scope is not None:
adopt = origin_scope == source.scope_id
else:
adopt = source.chat_type == "dm"
if adopt and self._claim_legacy_slack_key(legacy_key):
migrated_legacy_entry = self._entries.pop(legacy_key)
migrated_legacy_entry.session_key = session_key
migrated_legacy_entry.origin = source
migrated_legacy_entry.platform = source.platform
migrated_legacy_entry.chat_type = source.chat_type
self._entries[session_key] = migrated_legacy_entry
if migrated_legacy_entry is not None:
self._save_entries()
self._record_gateway_session_peer(
migrated_legacy_entry.session_id,
session_key,
source,
display_name=migrated_legacy_entry.display_name,
)
db_end_session_id = None
db_create_kwargs = None
existing_session_id = None
@ -1990,6 +2298,7 @@ class SessionStore:
was_auto_reset = False
auto_reset_reason = None
reset_had_activity = False
prev_session_id: Optional[str] = None
with self._lock:
self._ensure_loaded_locked()
@ -2024,6 +2333,7 @@ class SessionStore:
auto_reset_reason = _reset_reason
reset_had_activity = entry.last_prompt_tokens > 0
db_end_session_id = entry.session_id
prev_session_id = entry.session_id
entry = None
_needs_recover = True
elif entry.session_id != _stale_session_id:
@ -2038,6 +2348,7 @@ class SessionStore:
auto_reset_reason = _reset_reason
reset_had_activity = entry.last_prompt_tokens > 0
db_end_session_id = entry.session_id
prev_session_id = entry.session_id
self._entries.pop(session_key, None)
entry = None
_needs_recover = True
@ -2050,6 +2361,10 @@ class SessionStore:
# ---- Phase 3: no-lock I/O -- recovery + create + save + DB ops ----
if _needs_recover and db_end_session_id is None:
# The legacy (pre-workspace) Slack key fallback happens INSIDE
# _query_recoverable_session (#20583/#66398 design): it performs
# the exact-key legacy lookup, claims the key once per process,
# and rewrites the peer row to the scoped key on success.
recovered = self._query_recoverable_session(
session_key=session_key, source=source, now=now,
)
@ -2078,6 +2393,7 @@ class SessionStore:
was_auto_reset=was_auto_reset,
auto_reset_reason=auto_reset_reason,
reset_had_activity=reset_had_activity,
prev_session_id=prev_session_id,
)
with self._lock:
current = self._entries.get(session_key)

View file

@ -96,12 +96,11 @@ _SESSION_PROFILE: ContextVar = ContextVar("HERMES_SESSION_PROFILE", default=_UNS
# Whether the current session's delivery channel can route an ASYNC completion
# back to the agent AFTER the current turn ends (i.e. wake a fresh turn).
#
# True — CLI (in-process completion_queue drain) and the real gateway
# platforms (Telegram/Discord/Slack/...), which hold a persistent
# outbound channel and run the watcher/drain loops.
# False — stateless request/response adapters (the API server: every route,
# spec and proprietary, tears down its channel when the turn ends, so
# a background completion that finishes later has nowhere to go).
# True — long-lived CLI sessions (in-process completion_queue drain) and the
# real gateway platforms (Telegram/Discord/Slack/...), which hold a
# persistent outbound channel and run the watcher/drain loops.
# False — finite runtimes that can end before a detached completion returns:
# stateless API-server requests and dispatcher-spawned Kanban workers.
#
# Tools that promise async delivery (terminal notify_on_complete /
# watch_patterns, delegate_task background=True) read this via
@ -355,18 +354,29 @@ def declare_stateless_channel() -> None:
def async_delivery_supported() -> bool:
"""Whether the current session can deliver a background completion later.
Returns ``False`` when the active session was bound by a stateless channel:
an adapter that cannot route a notification back after the turn ends (the
API server), or a one-shot runner that exits after its final response
(``hermes -z``, cron see :func:`declare_stateless_channel`). The real
gateway platforms, the interactive CLI, and any path that never bound the
contextvar return ``True``.
Returns ``False`` for finite runtimes that can end before a detached result
is delivered: sessions explicitly bound by a stateless channel an adapter
that cannot route a notification back after the turn ends (the API server),
or a one-shot runner that exits after its final response (``hermes -z``,
cron see :func:`declare_stateless_channel`) and dispatcher-spawned
Kanban workers (identified by ``HERMES_KANBAN_TASK``), which are one-shot
``chat -q`` subprocesses. The real gateway platforms, the interactive CLI,
and any other path that never bound the contextvar return ``True``.
Tools that promise async delivery (``terminal`` notify_on_complete /
watch_patterns, ``delegate_task`` background=True) consult this before
registering a watcher / dispatching a detached child, so they can refuse a
promise the channel can't keep instead of silently no-op'ing.
"""
import os
# A Kanban worker is a one-shot subprocess. Its parent session and process
# disappear after the quiet turn returns, so a completion queued later has
# no durable consumer even though an ordinary CLI session can drain that
# queue. Force tools onto their existing synchronous/polling fallbacks.
if os.environ.get("HERMES_KANBAN_TASK"):
return False
value = _SESSION_ASYNC_DELIVERY.get()
if value is _UNSET:
return True

View file

@ -3562,6 +3562,22 @@ class GatewaySlashCommandsMixin:
defer_context_engine_notification=True,
)
)
# If _compress_context returned unchanged because a
# concurrent compression lock is held, tell the user
# clearly instead of showing the misleading
# "No changes from compression" no-op text. The wording
# distinguishes a confirmed holder from an unconfirmed
# acquisition failure (describe_compression_lock_skip).
# The deferred context-engine notification is discarded by
# the finally block below (finalize committed=False).
_lock_skipped = getattr(tmp_agent, "_compression_skipped_due_to_lock", None)
if _lock_skipped is True or isinstance(_lock_skipped, str):
from agent.manual_compression_feedback import (
describe_compression_lock_skip,
)
return describe_compression_lock_skip(_lock_skipped)
if partial and tail:
compressed = rejoin_compressed_head_and_tail(compressed, tail)

View file

@ -42,13 +42,7 @@ logger = logging.getLogger("gateway.stream_consumer")
# Sentinel to signal the stream is complete
_DONE = object()
# Sentinel to signal a tool boundary — finalize current message and start a
# new one so that subsequent text appears below tool progress messages.
_NEW_SEGMENT = object()
# Queue marker for a completed assistant commentary message emitted between
# API/tool iterations (for example: "I'll inspect the repo first.").
_COMMENTARY = object()
# Queue marker for a synchronous flush barrier. Enqueued as
@ -61,6 +55,76 @@ _COMMENTARY = object()
_FLUSH = object()
def escape_code_fences_for_display(text: str) -> str:
"""Escape triple-backtick markers so text can be safely wrapped
inside an outer ``` code block without breaking the fence.
When reasoning content contains ``` (e.g. the model quotes code
in its thinking), wrapping it in an outer ``` for display causes
the inner fence to break the outer block. Solution: replace each
`` ``` `` with `` \\`\\`\\` `` before wrapping.
Returns:
The input text with each `` ``` `` replaced by `` \\`\\`\\` ``,
or the input unchanged if no triple-backticks are present.
"""
if not isinstance(text, str) or "```" not in text:
return text
return text.replace("```", "\\`\\`\\`")
def ensure_closed_code_fences(text: str) -> str:
"""Append a closing `` ``` `` fence and/or `` ` `` if the text has
orphaned code-block or inline-code markers.
When model output is truncated mid-code-block (e.g. by token limits
or a finish_reason="length"), the resulting message has an unclosed
code fence. On Discord, Slack, and other platforms this causes
everything after the orphaned fence to render as a single code block.
The same problem applies to inline-code spans closed by a single
backtick: an orphaned `` ` `` makes the remainder of the message
render as inline code.
Triple-backtick: count `` ``` `` occurrences. If odd, append a
closing fence on its own line. This is safe because nested
triple-backtick fences (e.g. a literal `` ``` `` inside a code block)
are exceedingly rare in model output and, when they do appear, the
extra closing fence just creates a brief empty code block at the end
of the message far less harmful than the entire message being one
giant code block.
Single backtick: after balancing triple-backtick fences, strip all
complete `` `````` `` regions and count remaining standalone `` ` ``.
If odd, append a closing inline-code backtick. Same trade-off: a
stray closing backtick may produce a brief empty inline-code span,
which is far less harmful than the rest of the message being rendered
as inline code.
Returns:
The input text with closing markers appended if needed, or the
input text unchanged.
"""
if not isinstance(text, str) or not text:
return text
# Step 1: fix triple-backtick code-block fences (existing logic)
if text.count("```") % 2 == 1:
text = text.rstrip("\n") + "\n```"
# Step 2: fix single-backtick inline-code spans
# Remove complete ```…``` regions so their internal backticks don't
# pollute the standalone count. Also remove any trailing unclosed
# ``` that leaks through (defence in depth).
import re
without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
without_fences = re.sub(r"```[^`]*$", "", without_fences)
if without_fences.count("`") % 2 == 1:
text = text + "`"
return text
@dataclass
class StreamConsumerConfig:
"""Runtime config for a single stream consumer instance."""
@ -744,39 +808,76 @@ class GatewayStreamConsumer:
and self._message_id is None
):
# No existing message to edit (first message or after a
# segment break). Use truncate_message — the same
# helper the non-streaming path uses — to split with
# proper word/code-fence boundaries and chunk
# indicators like "(1/2)".
chunks = self.adapter.truncate_message(
self._accumulated, _safe_limit, len_fn=_len_fn,
# segment break). Seal only the overflowing head chunks
# as fixed messages, then keep the trailing chunk in
# _accumulated so the normal send/edit path below makes
# it the active preview. That lets chunk 2, 3, ... keep
# updating in-place as later streamed deltas arrive
# instead of posting every split as an immutable message.
chunks = self._truncate_for_stream(
self._accumulated, _safe_limit, _len_fn,
)
if len(chunks) <= 1:
# A malformed/legacy adapter result must not leave
# this overflow branch with an unsplittable payload.
chunks = self._split_text_chunks(
self._accumulated, _safe_limit, _len_fn,
)
chunks_delivered = False
reply_to = self._message_id or self._initial_reply_to_id
for chunk in chunks:
reply_to = self._initial_reply_to_id
all_heads_delivered = len(chunks) > 1
for chunk in chunks[:-1]:
new_id = await self._send_new_chunk(
chunk,
reply_to,
final=got_done,
)
if new_id is not None and new_id != reply_to:
chunks_delivered = True
self._accumulated = ""
self._last_sent_text = ""
if new_id is None or new_id == reply_to:
# Failed to deliver a sealed head; keep the
# full accumulated text intact so the gateway's
# fallback path can still deliver it completely.
all_heads_delivered = False
chunks_delivered = False
break
chunks_delivered = True
reply_to = new_id
if all_heads_delivered:
self._accumulated = chunks[-1]
# The head chunks are sealed. Clear the edit target
# so the remaining tail is sent as a fresh active
# chunk, then edited by subsequent deltas.
self._message_id = None
self._message_created_ts = None
self._last_sent_text = ""
else:
# A prior head may have landed before a later head
# failed. Do not edit that sealed message with the
# unsplit full payload; let the fallback path retry.
self._message_id = None
self._message_created_ts = None
self._last_sent_text = ""
self._last_edit_time = time.monotonic()
if got_done:
# Only claim final delivery if THESE chunks actually
# landed. ``_already_sent`` may be True from prior
# tool-progress edits or fallback-mode promotion (#10748)
# — that doesn't mean the final answer reached the user.
self._final_response_sent = chunks_delivered
if chunks_delivered:
tail_delivered = True
if self._accumulated:
tail_delivered = await self._send_or_edit(
self._accumulated, finalize=True,
)
# Only claim final delivery if the sealed chunks and
# final tail actually landed. ``_already_sent`` may
# be True from prior progress/fallback state (#10748).
self._final_response_sent = chunks_delivered and tail_delivered
if self._final_response_sent:
self._final_content_delivered = True
return
if got_segment_break:
self._message_id = None
self._fallback_final_send = False
self._fallback_prefix = ""
if not self._accumulated:
continue
# This iteration consumed a _FLUSH barrier and delivered
# the buffered prose via the chunk loop above, then takes
@ -797,8 +898,8 @@ class GatewayStreamConsumer:
self._accumulated, _safe_limit, _len_fn,
)
split_at = self._accumulated.rfind("\n", 0, _cp_budget)
if split_at < _safe_limit // 2:
split_at = _safe_limit
if split_at < _cp_budget // 2:
split_at = _cp_budget
chunk = self._accumulated[:split_at]
# finalize=True so the adapter applies platform-specific
# rich-text markup (e.g. Telegram MarkdownV2). This
@ -1074,26 +1175,103 @@ class GatewayStreamConsumer:
return final_text[len(prefix):].lstrip()
return final_text
@staticmethod
def _balance_fences_across_chunks(chunks: "list[str]") -> "list[str]":
"""Close orphaned ``` fences at each chunk boundary and reopen on the next.
When a split lands inside a triple-backtick code block, the head chunk
would render everything after the orphaned fence as code, and the tail
chunk's content would lose its code formatting. Mirror
``BasePlatformAdapter.truncate_message``'s contract: close the fence at
the end of the chunk and reopen it (with the original language tag) at
the start of the next one, so EVERY delivered chunk is fence-balanced
on its own.
"""
if len(chunks) <= 1:
return chunks
out: "list[str]" = []
carry_lang: "Optional[str]" = None
for chunk in chunks:
prefix = f"```{carry_lang}\n" if carry_lang is not None else ""
in_code = carry_lang is not None
lang = carry_lang or ""
for line in chunk.split("\n"):
stripped = line.strip()
if stripped.startswith("```"):
if in_code:
in_code = False
lang = ""
else:
in_code = True
tag = stripped[3:].strip()
lang = tag.split()[0] if tag else ""
body = prefix + chunk
if in_code:
body += "\n```"
carry_lang = lang
else:
carry_lang = None
out.append(body)
return out
@staticmethod
def _split_text_chunks(
text: str, limit: int,
text: str,
limit: int,
len_fn: "Callable[[str], int]" = len,
) -> list[str]:
"""Split text into reasonably sized chunks for fallback sends."""
"""Split text into reasonably sized chunks for fallback sends.
Chunks are fence-balanced: a split inside a ``` code block closes the
fence on the head chunk and reopens it on the tail, so no chunk leaves
the rest of a message rendering as one giant code block.
"""
if len_fn(text) <= limit:
return [text]
# Reserve headroom for the close/reopen fence markers the balancing
# pass may add, so balanced chunks stay within the platform limit.
split_limit = limit
if "```" in text:
split_limit = max(limit - 16, limit // 2, 1)
chunks: list[str] = []
remaining = text
while len_fn(remaining) > limit:
_cp_budget = _custom_unit_to_cp(remaining, limit, len_fn)
while len_fn(remaining) > split_limit:
_cp_budget = _custom_unit_to_cp(remaining, split_limit, len_fn)
split_at = remaining.rfind("\n", 0, _cp_budget)
if split_at < limit // 2:
split_at = limit
if split_at < _cp_budget // 2:
split_at = _cp_budget
chunks.append(remaining[:split_at])
remaining = remaining[split_at:].lstrip("\n")
if remaining:
chunks.append(remaining)
return chunks
return GatewayStreamConsumer._balance_fences_across_chunks(chunks)
def _truncate_for_stream(
self,
text: str,
limit: int,
len_fn: "Callable[[str], int]",
) -> list[str]:
"""Use the adapter's canonical splitter for streaming overflow.
Platform adapters may add word-boundary, code-fence, table, or
platform-specific formatting rules. The consumer must not replace
those rules with newline-only slicing. Non-base test doubles and
legacy adapters retain the historical two-argument call shape.
"""
truncate = getattr(self.adapter, "truncate_message", None)
if not callable(truncate):
return self._split_text_chunks(text, limit, len_fn)
if isinstance(self.adapter, _BasePlatformAdapter):
chunks = truncate(text, limit, len_fn=len_fn)
else:
chunks = truncate(text, limit)
if not isinstance(chunks, (list, tuple)) or not all(
isinstance(chunk, str) for chunk in chunks
):
return self._split_text_chunks(text, limit, len_fn)
return list(chunks)
async def _send_fallback_final(self, text: str) -> None:
"""Send the final continuation after streaming edits stop working.
@ -1101,6 +1279,10 @@ class GatewayStreamConsumer:
Retries each chunk once on flood-control failures with a short delay.
"""
final_text = self._clean_for_display(text)
# Ensure balanced code fences before computing continuation,
# so the closing fence reaches the user even when the fallback
# only delivers the tail after mid-stream edits failed.
final_text = ensure_closed_code_fences(final_text)
continuation = self._continuation_text(final_text)
self._fallback_final_send = False
if not continuation.strip():
@ -1758,6 +1940,12 @@ class GatewayStreamConsumer:
# Media files are delivered as native attachments after the stream
# finishes (via _deliver_media_from_response in gateway/run.py).
text = self._clean_for_display(text)
# Ensure code fences are balanced before send/edit. Model output
# truncated mid-code-block (e.g. finish_reason="length") leaves an
# orphaned ``` which, on Discord/Slack/Matrix, causes the entire
# remaining output to render as a single code block. This covers
# the streaming edit path (G2) and first-send path alike.
text = ensure_closed_code_fences(text)
# A bare streaming cursor is not meaningful user-visible content and
# can render as a stray tofu/white-box message on some clients.
visible_without_cursor = text

184
gateway/wake.py Normal file
View file

@ -0,0 +1,184 @@
"""Wake an existing agent session from a background completion event.
Two delivery strategies, selected by the target adapter's
``supports_async_delivery`` capability flag:
* Push-capable adapters (telegram, discord, plugin platforms, ...): inject a
synthetic ``MessageEvent(internal=True)`` through ``adapter.handle_message``
the pre-existing wake path, preserved exactly.
* Stateless request/response adapters (the API server,
``supports_async_delivery = False``): ``handle_message`` would run the wake
turn under a ``build_session_key()``-derived key
(``agent:main:api_server:group:<sid>``) that NEVER matches the raw
``X-Hermes-Session-Id`` key real gateway/HQ turns run under
(``_bind_api_server_session``), so the wake lands in a parallel, invisible
session. Instead we self-POST ``/v1/chat/completions`` on the in-pod API
server with the raw session id in the ``X-Hermes-Session-Id`` header the
exact entry point real turns use so the wake turn resumes the REAL
session, with full history, and its result is visible the next time the
client polls/reopens the conversation.
Failures RAISE (after bounded retries on transient errors) so callers can
rewind cursors / retry instead of silently losing the event.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any, Optional
logger = logging.getLogger(__name__)
# A wake self-post runs the entire agent turn synchronously (stream=false);
# generous ceiling so long tool-using turns aren't killed mid-flight.
WAKE_TURN_TIMEOUT_SECONDS = 600.0
# Backoff delays between retries on transient failures (429 concurrency cap,
# connection errors). The API server has no per-session lock — concurrent
# turns on one session are last-writer-wins — but it DOES enforce a global
# max_concurrent_runs cap via HTTP 429, which is worth waiting out.
_RETRY_DELAYS_SECONDS = (2.0, 5.0, 10.0)
def adapter_supports_push(adapter: Any) -> bool:
"""Whether this adapter can push a message to the user after a turn ends.
Mirrors ``gateway.session_context.async_delivery_supported`` but reads the
capability off the adapter class (``supports_async_delivery``) instead of
the request-scoped contextvar background watchers run outside any bound
session context. Adapters that don't declare the flag are push-capable.
"""
return bool(getattr(adapter, "supports_async_delivery", True))
async def deliver_wake(
adapter: Any,
*,
text: str,
session_id: str = "",
source: Any = None,
) -> None:
"""Deliver a wake turn to the session behind ``adapter``.
``session_id`` is the RAW session id (the ``X-Hermes-Session-Id`` value /
``state.db`` key) required for non-push adapters. ``source`` is the
``SessionSource`` used to build the synthetic event required for
push-capable adapters.
Raises on failure (bad arguments, exhausted retries, HTTP error) so the
caller can rewind/retry instead of treating the wake as delivered.
"""
if adapter_supports_push(adapter):
if source is None:
raise ValueError(
"deliver_wake: push-capable adapter requires a SessionSource"
)
from gateway.platforms.base import MessageEvent, MessageType
synth_event = MessageEvent(
text=text,
message_type=MessageType.TEXT,
source=source,
internal=True,
)
await adapter.handle_message(synth_event)
return
if not session_id:
raise ValueError(
"deliver_wake: non-push adapter (supports_async_delivery=False) "
"requires the raw session id to self-post the wake turn"
)
await _self_post_chat_completion(adapter, text=text, session_id=session_id)
async def _self_post_chat_completion(
adapter: Any, *, text: str, session_id: str
) -> None:
"""POST the wake text to the in-pod API server as a normal session turn.
Uses the adapter's own bind host/port/key (``ApiServerAdapter.__init__``).
Session continuation via ``X-Hermes-Session-Id`` is 403-gated on
``API_SERVER_KEY`` being configured, so a missing key is a hard error
raise loudly rather than run the wake in a fresh fingerprint-derived
session nobody is looking at.
"""
import aiohttp
host = str(getattr(adapter, "_host", "") or "127.0.0.1")
if host in ("0.0.0.0", "::", "*"):
# Wildcard bind address — connect over loopback.
host = "127.0.0.1"
port = int(getattr(adapter, "_port", 0) or 8642)
api_key = str(getattr(adapter, "_api_key", "") or "")
if not api_key:
raise RuntimeError(
"wake self-post requires API_SERVER_KEY: session continuation via "
"X-Hermes-Session-Id is rejected (403) on an unauthenticated API "
"server, so the wake cannot reach the target session"
)
if ":" in host and not host.startswith("["):
host = f"[{host}]" # bare IPv6 literal
url = f"http://{host}:{port}/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"X-Hermes-Session-Id": session_id,
}
payload = {
"model": str(getattr(adapter, "_model_name", "") or "hermes-agent"),
"messages": [{"role": "user", "content": text}],
"stream": False,
}
last_err: Optional[BaseException] = None
attempts = 1 + len(_RETRY_DELAYS_SECONDS)
for attempt in range(attempts):
if attempt:
await asyncio.sleep(_RETRY_DELAYS_SECONDS[attempt - 1])
try:
timeout = aiohttp.ClientTimeout(total=WAKE_TURN_TIMEOUT_SECONDS)
async with aiohttp.ClientSession(timeout=timeout) as http:
async with http.post(url, json=payload, headers=headers) as resp:
if resp.status == 429:
# Global concurrency cap (max_concurrent_runs) —
# transient; back off and retry.
last_err = RuntimeError(
f"wake self-post got HTTP 429 (concurrency cap) "
f"for session {session_id}"
)
logger.warning(
"%s; attempt %d/%d", last_err, attempt + 1, attempts
)
continue
if resp.status >= 400:
body = (await resp.text())[:300]
# Non-transient (auth/validation) — fail immediately.
raise RuntimeError(
f"wake self-post failed for session {session_id}: "
f"HTTP {resp.status}: {body}"
)
await resp.read()
logger.info(
"wake self-post delivered for session %s (attempt %d)",
session_id,
attempt + 1,
)
return
except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as exc:
last_err = exc
logger.warning(
"wake self-post transient failure for session %s "
"(attempt %d/%d): %s",
session_id,
attempt + 1,
attempts,
exc,
)
continue
raise RuntimeError(
f"wake self-post gave up for session {session_id} after "
f"{attempts} attempts: {last_err}"
) from last_err

View file

@ -0,0 +1,226 @@
"""Dependency-light venv recovery that runs BEFORE hermes_cli.main's imports.
The ``hermes`` console entry point is ``hermes_cli.main:main``. Importing
``hermes_cli.main`` pulls in third-party packages at module level (``dotenv``
via ``hermes_cli.env_loader``, ``yaml`` via ``hermes_cli.config``, ...). In
the exact failure state the update-recovery markers exist for a failed lazy
backend refresh or interrupted core install that wiped a core package's
import files (#57828) — a normal launch crashes *while importing main.py*,
before ``_recover_from_interrupted_install()`` can run. The marker system is
unreachable precisely when it is needed most.
This module is deliberately **stdlib-only** so importing it can never fail on
a corrupted venv. ``hermes_cli.main`` imports and calls
:func:`recover_if_needed` at the very top of its module body, before any
third-party import.
Scope: this early pass only repairs enough for ``hermes_cli.main`` to become
importable again (force-reinstall of the known-fragile core packages, using
the pins from pyproject.toml). It NEVER clears the recovery markers the
full, confirmed marker lifecycle stays with ``_recover_from_interrupted_install()``
in main.py, which runs right after import succeeds.
"""
from __future__ import annotations
import importlib
import os
import subprocess
import sys
import time
from pathlib import Path
# Core packages a failed lazy ``uv pip install`` is known to leave with intact
# distribution metadata but wiped import files (#57828). ``module`` is what we
# probe via a real import; ``attr`` guards against an empty/stub module.
# main.py's marker-recovery path reuses these tables — keep them here (the
# dependency-light module) so both layers probe and repair the same set.
LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = (
("yaml", "SafeDumper"),
("dotenv", "load_dotenv"),
("click", "Command"),
("certifi", "contents"),
("rich", "print"),
("cryptography", "__version__"),
("jwt", "encode"),
)
LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = {
"yaml": "PyYAML",
"dotenv": "python-dotenv",
"click": "click",
"certifi": "certifi",
"rich": "rich",
"cryptography": "cryptography",
"jwt": "PyJWT",
}
def _project_root() -> Path:
return Path(__file__).resolve().parent.parent
def _pinned_specs(packages: list[str], project_root: Path) -> list[str]:
"""Map bare package names to their pinned specs from pyproject.toml.
Stdlib-only (tomllib + naive requirement-head parsing ``packaging`` may
itself be broken in the failure state this module exists for). Unknown
packages fall back to their bare name.
"""
pyproject = project_root / "pyproject.toml"
if not pyproject.is_file():
return packages
try:
import tomllib
with open(pyproject, "rb") as f:
raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or []
except Exception:
return packages
name_to_spec: dict[str, str] = {}
for spec in raw_deps:
head = spec.split(";", 1)[0].strip()
bare = head
for op in ("==", ">=", "<=", "~=", ">", "<", "!="):
if op in bare:
bare = bare.split(op, 1)[0]
break
key = bare.strip().split("[", 1)[0].strip().lower()
if key:
name_to_spec[key] = head
return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages]
def _probe_broken_packages() -> list[str]:
"""Import-probe the fragile core packages in THIS process.
Returns repair package names (deduped, probe order) for modules that fail
to import or lack their sentinel attribute. Failed imports leave nothing
in ``sys.modules``, so a post-repair retry in the same process works.
"""
broken: list[str] = []
for mod_name, attr in LAZY_REFRESH_IMPORT_PROBES:
try:
mod = importlib.import_module(mod_name)
if not hasattr(mod, attr):
raise ImportError(f"{mod_name} missing {attr}")
except Exception:
pkg = LAZY_REFRESH_REPAIR_PACKAGES.get(mod_name)
if pkg and pkg not in broken:
broken.append(pkg)
return broken
def _run_repair_install(specs: list[str], project_root: Path) -> bool:
"""ensurepip + ``pip install --force-reinstall`` the given specs.
Streams nothing to stdout (``hermes acp`` speaks JSON-RPC on stdout);
output is captured and replayed to stderr only on failure. Never raises.
"""
try:
subprocess.run(
[sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
cwd=project_root,
capture_output=True,
)
except Exception:
pass
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "--force-reinstall", *specs],
cwd=project_root,
capture_output=True,
text=True,
)
except Exception as exc:
print(f" ✗ Early venv repair could not run pip: {exc}", file=sys.stderr)
return False
if result.returncode != 0:
tail = (result.stderr or result.stdout or "")[-2000:]
if tail:
print(tail, file=sys.stderr)
return False
return True
def recover_if_needed(
project_root: Path | None = None,
argv: list[str] | None = None,
) -> None:
"""Repair wiped core packages so ``hermes_cli.main`` can import at all.
Fast path (no marker present) is two ``lstat`` calls. Only acts when a
recovery marker from a prior ``hermes update`` exists AND an import probe
confirms a core package is actually broken. Markers are intentionally
NOT cleared here ``_recover_from_interrupted_install()`` in main.py owns
the confirmed marker lifecycle and runs immediately after import succeeds.
Never raises: on any failure the import of main.py proceeds and surfaces
the real error.
"""
try:
args = sys.argv[1:] if argv is None else argv
# Same deliberately-loose match as main(): the real update flow writes
# and clears its own markers — a recovery install must not race it.
if "update" in args:
return
root = _project_root() if project_root is None else project_root
core_marker = root / ".update-incomplete"
lazy_marker = root / ".lazy-refresh-incomplete"
if not core_marker.exists() and not lazy_marker.exists():
return
# Managed/Docker/PyPI installs have no source tree here — the marker
# is not ours to act on; main.py's recovery clears it.
if not (root / "pyproject.toml").is_file():
return
broken = _probe_broken_packages()
if not broken:
# Imports are fine — main.py will load and run full recovery.
return
# Single-flight: share main.py's recovery lock so an early repair
# never races a concurrent full recovery into the same shared venv.
lock_path = root / ".update-incomplete.lock"
try:
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.write(fd, f"{os.getpid()}\n".encode())
os.close(fd)
except FileExistsError:
try:
if time.time() - lock_path.stat().st_mtime > 3600:
lock_path.unlink()
except OSError:
pass
return
except OSError:
pass # read-only fs / perms — proceed unlocked, install surfaces it
try:
specs = _pinned_specs(broken, root)
print(
"⚠ Core package(s) broken by an interrupted update — "
f"repairing before launch: {', '.join(broken)}",
file=sys.stderr,
)
if _run_repair_install(specs, root) and not _probe_broken_packages():
print(" ✓ Core packages repaired.", file=sys.stderr)
else:
print(
" ✗ Automatic repair incomplete. Recover manually with:",
file=sys.stderr,
)
print(
f" {sys.executable} -m pip install --force-reinstall "
+ " ".join(specs),
file=sys.stderr,
)
finally:
try:
lock_path.unlink()
except OSError:
pass
except Exception:
# Never block launch — the import of main.py will surface the truth.
pass

View file

@ -10,8 +10,9 @@ Several common subprocess patterns break silently-or-loudly on Windows:
* ``start_new_session=True`` on POSIX, this maps to ``os.setsid()`` and
actually detaches the child. On Windows it's silently ignored; the
Windows equivalent is ``CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS``
creationflags, which Python only applies when you pass them explicitly.
Windows equivalent is the ``CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW``
creationflags bundle, which Python only applies when you pass it
explicitly.
* Console-window flashes every ``subprocess.Popen`` of a ``.exe`` on
Windows spawns a cmd window briefly unless ``CREATE_NO_WINDOW`` is
@ -98,7 +99,23 @@ def resolve_node_command(name: str, argv: Sequence[str]) -> list[str]:
# because CREATE_NO_WINDOW and DETACHED_PROCESS aren't guaranteed to be
# present on stdlib subprocess on older Pythons or non-Windows builds.
_CREATE_NEW_PROCESS_GROUP = 0x00000200
_DETACHED_PROCESS = 0x00000008
# DETACHED_PROCESS is intentionally NOT part of any flag bundle here — do not
# re-add it. Two reasons (the recurring console-flash bug #54220 / #56747):
#
# 1. MSDN (Process Creation Flags): CREATE_NO_WINDOW "is ignored if used with
# either CREATE_NEW_CONSOLE or DETACHED_PROCESS". Combining them means
# DETACHED_PROCESS governs and the no-window bit is dead.
# 2. A DETACHED_PROCESS child has NO console at all, so every console-subsystem
# descendant it ever spawns (git, gh, cmd, node, wmic, powershell, …) must
# allocate its OWN console — a visible flash per spawn, including spawns
# inside third-party libraries that no per-call-site CREATE_NO_WINDOW sweep
# can reach. A CREATE_NO_WINDOW child instead OWNS a hidden console that
# all descendants inherit, making "no flashing windows" a property of the
# one daemon launch. Root cause isolated + A/B verified on Windows 11 by
# the desktop backend fix (commit aa2ae36c3f): with per-site hide flags
# neutered, naive git/gh/cmd spawns don't flash under a hidden-console
# parent and do flash under a console-less one.
_DETACHED_PROCESS = 0x00000008 # kept for reference; must stay out of bundles
_CREATE_NO_WINDOW = 0x08000000
# Escape any Win32 job object the parent process belongs to. Without this,
# a detached child still inherits its parent's job object membership, and
@ -114,7 +131,8 @@ _CREATE_BREAKAWAY_FROM_JOB = 0x01000000
def windows_detach_flags() -> int:
"""Return Win32 creationflags that detach a child from the parent
console and process group. 0 on non-Windows.
console and process group without leaving it console-less. 0 on
non-Windows.
Pair with ``start_new_session=False`` (default) when calling
subprocess.Popen on POSIX use ``start_new_session=True`` instead,
@ -123,19 +141,23 @@ def windows_detach_flags() -> int:
Rationale:
- ``CREATE_NEW_PROCESS_GROUP`` child has its own process group so
Ctrl+C in the parent console doesn't propagate.
- ``DETACHED_PROCESS`` child has no console at all. Necessary for
background daemons (gateway watchers, update respawners) because
without it, closing the console kills the child.
- ``CREATE_NO_WINDOW`` suppress the brief cmd flash that would
otherwise appear when launching a console app. Redundant with
DETACHED_PROCESS but explicit for clarity.
- ``CREATE_NO_WINDOW`` the child gets its own fresh console that is
never shown. This both detaches it from the parent's console
lifetime (closing the launching terminal doesn't CTRL_CLOSE it) AND
gives every console-subsystem descendant (git, gh, cmd, node, ) a
console to inherit, so they don't allocate visible flashing ones.
This deliberately replaces the old ``DETACHED_PROCESS`` approach:
MSDN specifies CREATE_NO_WINDOW is *ignored* when combined with
DETACHED_PROCESS, and a truly console-less daemon re-creates the
per-descendant console-flash bug (#54220/#56747) at every spawn —
see the note on ``_DETACHED_PROCESS`` above.
- ``CREATE_BREAKAWAY_FROM_JOB`` escape any job object the parent is
in. Electron (Desktop app) and Tauri (bootstrap installer) wrap
their children in job objects; without breakaway, those children
die when the parent process exits even if they were spawned with
DETACHED_PROCESS. This was the missing flag that made the
post-update gateway respawn watcher silently die alongside the
Tauri updater after the Electron Desktop's update flow finished.
die when the parent process exits even though they have their own
console. This was the missing flag that made the post-update
gateway respawn watcher silently die alongside the Tauri updater
after the Electron Desktop's update flow finished.
If a process is in a job that disallows breakaway (rare
JOB_OBJECT_LIMIT_BREAKAWAY_OK isn't set), CreateProcess returns
@ -149,7 +171,6 @@ def windows_detach_flags() -> int:
return 0
return (
_CREATE_NEW_PROCESS_GROUP
| _DETACHED_PROCESS
| _CREATE_NO_WINDOW
| _CREATE_BREAKAWAY_FROM_JOB
)
@ -182,7 +203,7 @@ def windows_detach_flags_without_breakaway() -> int:
"""
if not IS_WINDOWS:
return 0
return _CREATE_NEW_PROCESS_GROUP | _DETACHED_PROCESS | _CREATE_NO_WINDOW
return _CREATE_NEW_PROCESS_GROUP | _CREATE_NO_WINDOW
def windows_hide_flags() -> int:
@ -193,10 +214,12 @@ def windows_hide_flags() -> int:
operation (``taskkill``, ``where``, version probes) where we want no
flash but also want to collect stdout/exit code synchronously.
The key difference from :func:`windows_detach_flags`: NO
``DETACHED_PROCESS`` the child still inherits stdio handles so
``capture_output=True`` works. ``DETACHED_PROCESS`` would sever
stdio and break stdout capture.
The difference from :func:`windows_detach_flags`: no
``CREATE_NEW_PROCESS_GROUP`` / ``CREATE_BREAKAWAY_FROM_JOB`` the
child stays in the parent's process group and job so Ctrl+C and job
teardown propagate normally, as a short-lived helper wants. Stdio
handles are inherited either way, so ``capture_output=True`` works
with both bundles.
"""
if not IS_WINDOWS:
return 0

View file

@ -1447,6 +1447,73 @@ def read_credential_pool(provider_id: Optional[str] = None) -> Dict[str, Any]:
return list(global_entries) if isinstance(global_entries, list) else []
_POOL_STATUS_FIELDS = (
"last_status",
"last_status_at",
"last_error_code",
"last_error_reason",
"last_error_message",
"last_error_reset_at",
)
def _merge_disk_cooldown_state(
entry: Dict[str, Any],
disk_entry: Optional[Dict[str, Any]],
provider_id: str,
) -> Dict[str, Any]:
"""Keep a newer on-disk cooldown/quarantine over a stale in-memory one.
``write_credential_pool`` callers persist an in-memory snapshot that may
predate another process marking the same credential exhausted or dead
(last-writer-wins lost update). Without this merge, process B's later
rewrite resurrects a rate-limited key as healthy and both processes
resume hammering it. Adopt the on-disk status fields only when they are
strictly more recent (by ``last_status_at``) AND still binding a DEAD
marker, or an EXHAUSTED cooldown that has not yet expired. Expired
cooldowns are not resurrected, so the pool's own expiry-clear (which
resets ``last_status_at`` to None) is never overridden.
"""
if not isinstance(disk_entry, dict):
return entry
try:
from agent.credential_pool import (
PooledCredential,
STATUS_DEAD,
STATUS_EXHAUSTED,
_exhausted_until,
_parse_absolute_timestamp,
)
disk_status = disk_entry.get("last_status")
if disk_status not in (STATUS_DEAD, STATUS_EXHAUSTED):
return entry
# A token change means the caller re-authed/refreshed this entry and
# intentionally cleared its status (e.g. _sync_codex_entry_from_
# auth_store after a fresh device-code login) — never resurrect the
# old cooldown onto fresh credentials.
mem_access = entry.get("access_token") or ""
disk_access = disk_entry.get("access_token") or ""
if mem_access and disk_access and mem_access != disk_access:
return entry
disk_ts = _parse_absolute_timestamp(disk_entry.get("last_status_at")) or 0.0
mem_ts = _parse_absolute_timestamp(entry.get("last_status_at")) or 0.0
if disk_ts <= mem_ts:
return entry
if disk_status == STATUS_EXHAUSTED:
until = _exhausted_until(
PooledCredential.from_dict(provider_id, disk_entry)
)
if until is None or until <= time.time():
return entry
merged_entry = dict(entry)
for status_field in _POOL_STATUS_FIELDS:
merged_entry[status_field] = disk_entry.get(status_field)
return merged_entry
except Exception: # pragma: no cover - best-effort merge
return entry
def write_credential_pool(
provider_id: str,
entries: List[Dict[str, Any]],
@ -1464,6 +1531,10 @@ def write_credential_pool(
the caller loaded its in-memory snapshot; without this merge a later
rotation/exhaustion rewrite drops the concurrent credential.
For entries present on BOTH sides, status fields are merged by
``last_status_at`` recency via ``_merge_disk_cooldown_state`` so a stale
snapshot cannot erase a cooldown/quarantine another process just wrote.
Pass ``removed_ids`` for entries the caller intentionally removed, so the
merge does not resurrect them from the on-disk copy.
"""
@ -1481,12 +1552,24 @@ def write_credential_pool(
]
existing = pool.get(provider_id)
existing_list = existing if isinstance(existing, list) else []
existing_by_id = {
entry.get("id"): entry
for entry in existing_list
if isinstance(entry, dict) and entry.get("id")
}
new_ids = {
entry.get("id")
for entry in sanitized_entries
if isinstance(entry, dict) and entry.get("id")
}
merged: List[Dict[str, Any]] = list(sanitized_entries)
merged: List[Dict[str, Any]] = [
_merge_disk_cooldown_state(
entry, existing_by_id.get(entry.get("id")), provider_id
)
if isinstance(entry, dict)
else entry
for entry in sanitized_entries
]
for disk_entry in existing_list:
if not isinstance(disk_entry, dict):
continue
@ -1588,7 +1671,7 @@ def is_provider_explicitly_configured(provider_id: str) -> bool:
except Exception:
pass
# 2. Check config.yaml model.provider
# 2. Check config.yaml model.provider and other explicit provider slots.
try:
from hermes_cli.config import load_config
cfg = load_config()
@ -1597,6 +1680,37 @@ def is_provider_explicitly_configured(provider_id: str) -> bool:
cfg_provider = (model_cfg.get("provider") or "").strip().lower()
if cfg_provider == normalized:
return True
# MoA presets are explicit model selections too. A user who configured
# ``provider: anthropic`` as a MoA advisor/aggregator has opted Hermes
# into using Anthropic credentials for that slot even when the main
# session model is another provider. Without this, Claude Code OAuth
# entries are pruned/ignored by credential_pool.load_pool("anthropic"),
# so MoA Anthropic advisors fail with "no ANTHROPIC_API_KEY" while the
# normal model picker says Anthropic is logged in.
def _slot_matches_provider(slot):
return (
isinstance(slot, dict)
and (slot.get("provider") or "").strip().lower() == normalized
)
moa_cfg = cfg.get("moa")
if isinstance(moa_cfg, dict):
for slot in moa_cfg.get("reference_models") or []:
if _slot_matches_provider(slot):
return True
if _slot_matches_provider(moa_cfg.get("aggregator")):
return True
presets = moa_cfg.get("presets")
if isinstance(presets, dict):
for preset in presets.values():
if not isinstance(preset, dict):
continue
for slot in preset.get("reference_models") or []:
if _slot_matches_provider(slot):
return True
if _slot_matches_provider(preset.get("aggregator")):
return True
except Exception:
pass

View file

@ -503,13 +503,21 @@ class CLIAgentSetupMixin:
if resolved_meta:
session_meta = resolved_meta
restored = self._session_db.get_messages_as_conversation(
self.session_id, repair_alternation=True
)
model_history, display_history = self._session_db.get_resume_conversations(self.session_id)
restored = model_history
if restored:
restored = [m for m in restored if m.get("role") != "session_meta"]
self.conversation_history = restored
msg_count = len([m for m in restored if m.get("role") == "user"])
self._resume_display_history = [
m for m in display_history if m.get("role") != "session_meta"
]
msg_count = len(
[
m
for m in self._resume_display_history
if m.get("role") == "user" and not m.get("display_kind")
]
)
title_part = ""
if session_meta.get("title"):
title_part = f' "{session_meta["title"]}"'
@ -552,7 +560,8 @@ class CLIAgentSetupMixin:
"""
from cli import CLI_CONFIG, _record_output_history_entry, _strip_reasoning_tags, _suspend_output_history
from tools.ansi_strip import sanitize_display_text as _sanitize_display_text
if not self.conversation_history:
display_history = getattr(self, "_resume_display_history", self.conversation_history)
if not display_history:
return
# Check config: resume_display setting
@ -571,11 +580,21 @@ class CLIAgentSetupMixin:
entries = [] # list of (role, display_text)
_last_asst_idx = None # index of last assistant entry
_last_asst_full = None # un-truncated display text for last assistant
for msg in self.conversation_history:
for msg in display_history:
role = msg.get("role", "")
display_kind = msg.get("display_kind")
content = msg.get("content")
tool_calls = msg.get("tool_calls") or []
if display_kind == "hidden":
continue
if display_kind == "model_switch":
entries.append(("event", "model changed"))
continue
if display_kind == "async_delegation_complete":
entries.append(("event", "background delegation completed"))
continue
if role == "system":
continue
if role == "tool":
@ -682,7 +701,9 @@ class CLIAgentSetupMixin:
)
for i, (role, text) in enumerate(entries):
if role == "user":
if role == "event":
lines.append(f"{text}\n", style="dim italic")
elif role == "user":
lines.append(" ● You: ", style=f"dim bold {_session_label_c}")
# Show first line inline, indent rest
msg_lines = text.splitlines()

View file

@ -780,11 +780,20 @@ class CLICommandsMixin:
# becomes ``self.conversation_history`` for subsequent turns. Heal a
# durable ``user;user`` violation once here instead of re-firing the
# pre-request repair on every request for the rest of the session.
restored = self._session_db.get_messages_as_conversation(
target_id, repair_alternation=True
#
# Both projections come from one lineage SELECT: model_history is
# alternation-repaired for live replay; display_history is the full
# lineage verbatim, used by _display_resumed_history() so timeline
# events and ancestor rows render correctly (matching the startup
# --resume path in _preload_resumed_session).
model_history, display_history = self._session_db.get_resume_conversations(
target_id
)
restored = [m for m in (restored or []) if m.get("role") != "session_meta"]
restored = [m for m in (model_history or []) if m.get("role") != "session_meta"]
self.conversation_history = restored
self._resume_display_history = [
m for m in (display_history or []) if m.get("role") != "session_meta"
]
# Re-open the target session so it's not marked as ended
try:
@ -824,7 +833,7 @@ class CLICommandsMixin:
pass
title_part = f" \"{session_meta['title']}\"" if session_meta.get("title") else ""
msg_count = len([m for m in self.conversation_history if m.get("role") == "user"])
msg_count = len([m for m in self._resume_display_history if m.get("role") == "user" and not m.get("display_kind")])
if self.conversation_history:
_cprint(
f" ↻ Resumed session {target_id}{title_part}"

Some files were not shown because too many files have changed in this diff Show more