mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
Merge remote-tracking branch 'origin/main' into merge/relay-metrics-upstream-20260723
This commit is contained in:
commit
fb1e417367
281 changed files with 26317 additions and 1673 deletions
|
|
@ -1936,8 +1936,8 @@ class HermesACPAgent(acp.Agent):
|
|||
return "No tools available."
|
||||
lines = [f"Available tools ({len(tools)}):"]
|
||||
for t in tools:
|
||||
name = t.get("function", {}).get("name", "?")
|
||||
desc = t.get("function", {}).get("description", "")
|
||||
name = (t.get("function") or {}).get("name", "?")
|
||||
desc = (t.get("function") or {}).get("description", "")
|
||||
# Truncate long descriptions
|
||||
if len(desc) > 80:
|
||||
desc = desc[:77] + "..."
|
||||
|
|
|
|||
|
|
@ -922,12 +922,17 @@ def recover_with_credential_pool(
|
|||
)
|
||||
return False, has_retried_429
|
||||
|
||||
# Capture the current API key before any rotation — needed to
|
||||
# identify which credential actually failed when
|
||||
# mark_exhausted_and_rotate is called. Without this hint the
|
||||
# pool falls back to current() or _select_unlocked(), which may
|
||||
# return the NEXT (healthy) entry after a prior rotation, marking
|
||||
# the wrong credential as exhausted (#43747).
|
||||
# Attribute the failure to the API key the agent actually dispatched the
|
||||
# request with, not to pool.current(). The current() pointer is shared,
|
||||
# mutable state — round-robin select() advances it on every call, and
|
||||
# concurrent turns or a second process (gateway/dashboard) reloading the
|
||||
# pool reset it to None — so by the time recovery runs it routinely points
|
||||
# at a DIFFERENT, healthy entry. Marking that entry exhausted copies this
|
||||
# request's error/reset time onto it and can take the whole pool offline
|
||||
# from a single rate-limited key (#43747). ``_swap_credential`` keeps
|
||||
# ``agent.api_key`` in sync with the entry in use, so it identifies the
|
||||
# failing entry exactly; fall back to current()'s key only when the agent
|
||||
# carries no key at all.
|
||||
_api_key_hint = getattr(agent, "api_key", None) or None
|
||||
if not _api_key_hint:
|
||||
_cur = pool.current()
|
||||
|
|
@ -987,7 +992,16 @@ def recover_with_credential_pool(
|
|||
# rotate immediately. This prevents the "cancel-between-429s" trap
|
||||
# where has_retried_429 (a local var) gets reset on each new prompt,
|
||||
# causing the pool to retry the same exhausted credential forever.
|
||||
current_entry = pool.current()
|
||||
# Prefer the entry matching the failing key over the shared current()
|
||||
# pointer, for the same attribution reason as above.
|
||||
current_entry = None
|
||||
if _api_key_hint:
|
||||
current_entry = next(
|
||||
(e for e in pool.entries() if e.runtime_api_key == _api_key_hint),
|
||||
None,
|
||||
)
|
||||
if current_entry is None:
|
||||
current_entry = pool.current()
|
||||
current_last_status = getattr(current_entry, "last_status", None) if current_entry else None
|
||||
if current_last_status == STATUS_EXHAUSTED:
|
||||
_ra().logger.info(
|
||||
|
|
@ -995,7 +1009,11 @@ def recover_with_credential_pool(
|
|||
current_last_status,
|
||||
)
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s",
|
||||
|
|
@ -1019,7 +1037,11 @@ def recover_with_credential_pool(
|
|||
if not has_retried_429 and not usage_limit_reached:
|
||||
return False, True
|
||||
rotate_status = status_code if status_code is not None else 429
|
||||
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (rate limit) — rotated to pool entry %s",
|
||||
|
|
@ -1034,7 +1056,7 @@ def recover_with_credential_pool(
|
|||
# Subscription/entitlement 403s look like auth failures on the wire
|
||||
# but refresh cannot fix them — the OAuth token is already valid,
|
||||
# the account simply lacks the entitlement. Without this guard,
|
||||
# ``try_refresh_current()`` keeps minting fresh tokens against the
|
||||
# the refresh path keeps minting fresh tokens against the
|
||||
# same unsubscribed account and the main agent loop spins re-issuing
|
||||
# the same 403 until the user Ctrl+C's.
|
||||
#
|
||||
|
|
@ -1087,9 +1109,13 @@ def recover_with_credential_pool(
|
|||
agent.provider or "provider",
|
||||
)
|
||||
return False, has_retried_429
|
||||
refreshed = pool.try_refresh_current()
|
||||
# Refresh the entry that supplied the failing key, not current():
|
||||
# the shared pointer can reference a different, healthy entry, and
|
||||
# refreshing it would consume that entry's single-use refresh token
|
||||
# (or mark it exhausted on failure) for a failure it never had.
|
||||
refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint)
|
||||
if refreshed is not None:
|
||||
# ``try_refresh_current()`` re-mints a fresh OAuth token and reports
|
||||
# ``try_refresh_matching()`` re-mints a fresh OAuth token and reports
|
||||
# success even when the upstream keeps rejecting it — a single-entry
|
||||
# pool (common for OAuth/Max subscribers) has nothing to rotate to,
|
||||
# so a bare "refreshed → retry" loop spins forever on the same dead
|
||||
|
|
@ -1117,9 +1143,13 @@ def recover_with_credential_pool(
|
|||
agent._swap_credential(refreshed)
|
||||
return True, has_retried_429
|
||||
# Refresh failed — rotate to next credential instead of giving up.
|
||||
# The failed entry is already marked exhausted by try_refresh_current().
|
||||
# The failed entry is already marked exhausted by the refresh attempt.
|
||||
rotate_status = status_code if status_code is not None else 401
|
||||
next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context, api_key_hint=_api_key_hint)
|
||||
next_entry = pool.mark_exhausted_and_rotate(
|
||||
status_code=rotate_status,
|
||||
error_context=error_context,
|
||||
api_key_hint=_api_key_hint,
|
||||
)
|
||||
if next_entry is not None:
|
||||
_ra().logger.info(
|
||||
"Credential %s (auth refresh failed) — rotated to pool entry %s",
|
||||
|
|
|
|||
|
|
@ -1881,6 +1881,28 @@ def _content_parts_to_anthropic_blocks(parts: Any) -> List[Dict[str, Any]]:
|
|||
return out
|
||||
|
||||
|
||||
_EMPTY_TEXT_PLACEHOLDER = "(empty)"
|
||||
|
||||
|
||||
def _safe_text(text: Any) -> str:
|
||||
"""Return ``text`` if it's non-whitespace, else a non-whitespace placeholder.
|
||||
|
||||
The Anthropic Messages API rejects requests where a text content block is
|
||||
empty or whitespace-only (HTTP 400 "text content blocks must contain
|
||||
non-whitespace text"). When such a block gets stored in session history —
|
||||
e.g. produced by context compression — it is replayed verbatim on every
|
||||
subsequent turn, permanently wedging the session. Coercing to a
|
||||
non-whitespace placeholder is self-healing: the next API call recovers.
|
||||
|
||||
Mirrors ``bedrock_adapter._safe_text`` (#9486); ref #69512.
|
||||
"""
|
||||
if text is None:
|
||||
return _EMPTY_TEXT_PLACEHOLDER
|
||||
if not isinstance(text, str):
|
||||
text = str(text)
|
||||
return text if text.strip() else _EMPTY_TEXT_PLACEHOLDER
|
||||
|
||||
|
||||
def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Strip output-only fields from a stored Anthropic content block so it is
|
||||
valid as REQUEST input on replay.
|
||||
|
|
@ -1898,7 +1920,10 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|||
return None
|
||||
btype = b.get("type")
|
||||
if btype == "text":
|
||||
out: Dict[str, Any] = {"type": "text", "text": b.get("text", "")}
|
||||
# Coerce empty/whitespace-only text to a non-whitespace placeholder;
|
||||
# the Messages input schema rejects blank text blocks (#69512), and a
|
||||
# blank block stored in history replays on every turn → permanent 400.
|
||||
out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))}
|
||||
# citations is input-valid ONLY when it's a non-empty list; the SDK
|
||||
# emits citations=None on responses, which the input schema rejects.
|
||||
cits = b.get("citations")
|
||||
|
|
@ -2058,7 +2083,16 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]:
|
|||
# Anthropic rejects empty assistant content
|
||||
effective = blocks or content
|
||||
if not effective or effective == "":
|
||||
effective = [{"type": "text", "text": "(empty)"}]
|
||||
effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
elif isinstance(effective, list):
|
||||
# The all-empty guard above misses a list that still contains a
|
||||
# whitespace-only text block (e.g. from a content array of blank parts,
|
||||
# or compression). Those also trip "text content blocks must contain
|
||||
# non-whitespace text" (#69512). Coerce text blocks in place; other
|
||||
# block types (thinking/tool_use/image) are left untouched.
|
||||
for blk in effective:
|
||||
if isinstance(blk, dict) and blk.get("type") == "text":
|
||||
blk["text"] = _safe_text(blk.get("text", ""))
|
||||
return {"role": "assistant", "content": effective}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2305,6 +2305,62 @@ def _read_main_base_url() -> str:
|
|||
return ""
|
||||
|
||||
|
||||
def _resolve_moa_aggregator(preset_name: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Resolve a MoA preset to its aggregator (provider, model) pair.
|
||||
|
||||
"moa" is a virtual provider — the acting model of a preset is its
|
||||
aggregator slot, and there is no real "moa" HTTP endpoint. Auxiliary
|
||||
tasks (title generation, compression, vision, commit messages, …) don't
|
||||
need the reference fan-out, so every aux resolution layer maps
|
||||
provider="moa"/model=<preset> to the aggregator's real provider+model
|
||||
through this single helper (shared by ``_resolve_auto``,
|
||||
``_resolve_task_provider_model``, and ``resolve_provider_client`` so the
|
||||
preset lookup and validation cannot drift between paths).
|
||||
|
||||
Args:
|
||||
preset_name: The MoA preset name (usually carried in the "model"
|
||||
field), or None/"" to resolve the user's default preset.
|
||||
|
||||
Returns:
|
||||
(aggregator_provider, aggregator_model), or (None, None) when the
|
||||
preset cannot be resolved (missing config, renamed/deleted preset,
|
||||
or a malformed aggregator slot).
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from hermes_cli.moa_config import resolve_moa_preset
|
||||
|
||||
preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None)
|
||||
agg = preset.get("aggregator") or {}
|
||||
agg_provider = str(agg.get("provider") or "").strip()
|
||||
agg_model = str(agg.get("model") or "").strip()
|
||||
if agg_provider and agg_model and agg_provider.lower() != "moa":
|
||||
return agg_provider, agg_model
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"MoA aggregator resolution failed for preset %r", preset_name, exc_info=True
|
||||
)
|
||||
return None, None
|
||||
|
||||
|
||||
def _read_main_model_for_aux() -> str:
|
||||
"""Main model with MoA presets unwrapped to the aggregator's model.
|
||||
|
||||
When the main provider is ``moa``, ``_read_main_model()`` returns a MoA
|
||||
*preset name* (e.g. "opus-gpt") — never a valid wire model id on any
|
||||
provider. Auxiliary fallback chains that pre-fill a missing model from
|
||||
the main model must use this reader instead, so unset aux models default
|
||||
to the preset's acting (aggregator) model. Returns "" when the main
|
||||
provider is moa but the preset cannot be resolved — sending nothing is
|
||||
strictly better than sending a preset name that 400s.
|
||||
"""
|
||||
model = _read_main_model()
|
||||
if (_read_main_provider() or "").strip().lower() == "moa":
|
||||
_, agg_model = _resolve_moa_aggregator(model)
|
||||
return agg_model or ""
|
||||
return model
|
||||
|
||||
|
||||
def _read_main_api_key_if_same_host(aux_base_url: str) -> str:
|
||||
"""Return the main api_key only when *aux_base_url* points at the same
|
||||
host as the main model's base_url.
|
||||
|
|
@ -2734,7 +2790,7 @@ def _try_custom_endpoint() -> Tuple[Optional[Any], Optional[str]]:
|
|||
return None, None
|
||||
if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()):
|
||||
return None, None
|
||||
model = _read_main_model() or "gpt-4o-mini"
|
||||
model = _read_main_model_for_aux() or "gpt-4o-mini"
|
||||
logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions")
|
||||
_clean_base, _dq = _extract_url_query_params(custom_base)
|
||||
_extra = {"default_query": _dq} if _dq else {}
|
||||
|
|
@ -4234,6 +4290,13 @@ def _try_main_agent_model_fallback(
|
|||
"""
|
||||
main_provider = (_read_main_provider() or "").strip()
|
||||
main_model = (_read_main_model() or "").strip()
|
||||
if main_provider.lower() == "moa":
|
||||
# MoA virtual provider: fall back to the preset's aggregator — the
|
||||
# acting model — instead of the unreachable "moa"/<preset-name> pair.
|
||||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||||
if not _agg_provider or not _agg_model:
|
||||
return None, None, ""
|
||||
main_provider, main_model = _agg_provider, _agg_model
|
||||
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
|
||||
return None, None, ""
|
||||
|
||||
|
|
@ -4634,26 +4697,17 @@ def _resolve_auto(
|
|||
# model. Resolve the MoA preset to its aggregator slot and continue Step 1
|
||||
# with that real provider+model. Mirrors the MoA context-length resolution.
|
||||
if main_provider == "moa":
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from hermes_cli.moa_config import resolve_moa_preset
|
||||
|
||||
_preset = resolve_moa_preset(load_config().get("moa") or {}, main_model)
|
||||
_agg = _preset.get("aggregator") or {}
|
||||
_agg_provider = str(_agg.get("provider") or "").strip()
|
||||
_agg_model = str(_agg.get("model") or "").strip()
|
||||
if _agg_provider and _agg_model and _agg_provider.lower() != "moa":
|
||||
main_provider = _agg_provider
|
||||
main_model = _agg_model
|
||||
# The MoA virtual runtime carries a non-HTTP base_url
|
||||
# ("moa://local") and a placeholder api_key; they belong to the
|
||||
# facade, not the aggregator's real provider. Drop them so the
|
||||
# aggregator resolves through its own provider credentials.
|
||||
runtime_base_url = ""
|
||||
runtime_api_key = ""
|
||||
runtime_api_mode = ""
|
||||
except Exception:
|
||||
logger.debug("MoA aux resolution to aggregator failed", exc_info=True)
|
||||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||||
if _agg_provider and _agg_model:
|
||||
main_provider = _agg_provider
|
||||
main_model = _agg_model
|
||||
# The MoA virtual runtime carries a non-HTTP base_url
|
||||
# ("moa://local") and a placeholder api_key; they belong to the
|
||||
# facade, not the aggregator's real provider. Drop them so the
|
||||
# aggregator resolves through its own provider credentials.
|
||||
runtime_base_url = ""
|
||||
runtime_api_key = ""
|
||||
runtime_api_mode = ""
|
||||
|
||||
if (main_provider and main_model
|
||||
and main_provider not in {"auto", ""}):
|
||||
|
|
@ -4908,6 +4962,27 @@ def resolve_provider_client(
|
|||
# Normalise aliases
|
||||
provider = _normalize_aux_provider(provider)
|
||||
|
||||
# MoA virtual provider chokepoint: "moa" is not a real HTTP provider —
|
||||
# its acting model is the preset's aggregator slot. The two resolver
|
||||
# layers above (_resolve_auto, _resolve_task_provider_model) already
|
||||
# unwrap their own paths, but callers that route here directly (vision
|
||||
# auto-detect, _try_main_agent_model_fallback, get_available_vision_backends,
|
||||
# plugin code) would otherwise dead-end in the unknown-provider branch.
|
||||
# ``model`` carries the preset name for moa calls; when the preset can't
|
||||
# be resolved we leave the call untouched and let the normal
|
||||
# missing-provider handling produce its diagnostic.
|
||||
if provider == "moa":
|
||||
_agg_provider, _agg_model = _resolve_moa_aggregator(model)
|
||||
if _agg_provider and _agg_model:
|
||||
original_provider = _agg_provider.strip().lower()
|
||||
provider = _normalize_aux_provider(_agg_provider)
|
||||
model = _agg_model
|
||||
# The moa:// facade endpoint and placeholder key belong to the
|
||||
# virtual runtime, not the aggregator's real provider.
|
||||
if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"):
|
||||
explicit_base_url = None
|
||||
explicit_api_key = None
|
||||
|
||||
# Universal model-resolution fallback for concrete providers. ``auto`` is
|
||||
# intentionally excluded: `_resolve_auto(main_runtime=...)` returns the
|
||||
# model paired with the provider it actually selected. Pre-filling an auto
|
||||
|
|
@ -4928,6 +5003,10 @@ def resolve_provider_client(
|
|||
# the load-bearing step for OAuth providers: an xai-oauth user
|
||||
# with grok-4.3 configured gets grok-4.3 for title generation
|
||||
# instead of silently dropping to whatever Step-2 fallback (#31845).
|
||||
# When the main provider is MoA, ``_read_main_model_for_aux()``
|
||||
# substitutes the preset's aggregator model — the preset NAME is
|
||||
# never a valid wire model id, so unset aux models default to the
|
||||
# preset's acting model instead.
|
||||
#
|
||||
# Each provider branch below sees a non-empty ``model`` whenever the
|
||||
# user has *anything* configured — no provider-specific empty-model
|
||||
|
|
@ -4944,7 +5023,7 @@ def resolve_provider_client(
|
|||
# return the actual current runtime model when the caller did not explicitly
|
||||
# request one. (# compression-current-model)
|
||||
if not model and provider != "auto":
|
||||
model = _get_aux_model_for_provider(provider) or _read_main_model() or model
|
||||
model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model
|
||||
|
||||
def _needs_codex_wrap(client_obj, base_url_str: str, model_str: str) -> bool:
|
||||
"""Decide if a plain OpenAI client should be wrapped for Responses API.
|
||||
|
|
@ -5218,7 +5297,7 @@ def resolve_provider_client(
|
|||
model
|
||||
or custom_entry.get("model")
|
||||
or (main_runtime.get("model") if main_runtime else None)
|
||||
or _read_main_model()
|
||||
or _read_main_model_for_aux()
|
||||
or "gpt-4o-mini",
|
||||
provider,
|
||||
)
|
||||
|
|
@ -5453,7 +5532,7 @@ def resolve_provider_client(
|
|||
final_model = _normalize_resolved_model(
|
||||
model
|
||||
or (main_runtime.get("model") if main_runtime else None)
|
||||
or _read_main_model(),
|
||||
or _read_main_model_for_aux(),
|
||||
provider,
|
||||
)
|
||||
if provider == "copilot-acp":
|
||||
|
|
@ -5821,7 +5900,24 @@ def resolve_vision_provider_client(
|
|||
# 5. Stop
|
||||
main_provider = str(runtime.get("provider") or _read_main_provider())
|
||||
main_model = str(runtime.get("model") or _read_main_model())
|
||||
if main_provider and main_provider not in {"auto", ""}:
|
||||
if main_provider.strip().lower() == "moa":
|
||||
# MoA virtual provider: main_model is a preset NAME, and every
|
||||
# capability probe below (_PROVIDERS_WITHOUT_VISION,
|
||||
# _main_model_supports_vision, _resolve_provider_vision_default)
|
||||
# would run against a provider/model pair that doesn't exist on
|
||||
# any wire. Unwrap to the preset's aggregator slot first so the
|
||||
# checks and the eventual client target the real acting model.
|
||||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||||
if _agg_provider and _agg_model:
|
||||
main_provider, main_model = _agg_provider, _agg_model
|
||||
# Drop the moa:// facade endpoint from the runtime view used
|
||||
# below — it belongs to the virtual provider, not the
|
||||
# aggregator's real provider.
|
||||
runtime = dict(runtime)
|
||||
runtime["base_url"] = ""
|
||||
runtime["api_key"] = ""
|
||||
runtime["api_mode"] = ""
|
||||
if main_provider and main_provider not in {"auto", "", "moa"}:
|
||||
# A provider-specific vision default wins over the user's chat model:
|
||||
# static overrides (xiaomi/zai) and catalog-backed discovery (the
|
||||
# DeepInfra profile hook) both yield a *known* vision-capable model,
|
||||
|
|
@ -6418,8 +6514,8 @@ def _resolve_task_provider_model(
|
|||
task: str = None,
|
||||
provider: str = None,
|
||||
model: str = None,
|
||||
base_url: str = None,
|
||||
api_key: str = None,
|
||||
base_url: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]:
|
||||
"""Determine provider + model for a call.
|
||||
|
||||
|
|
@ -6462,12 +6558,57 @@ def _resolve_task_provider_model(
|
|||
# which downstream consumers like ContextCompressor accept as the task output.
|
||||
# The provider-side 'auto' is handled in _resolve_auto() via main_runtime
|
||||
# fallback, so dropping cfg_model to None here lets that path do its job.
|
||||
#
|
||||
# The explicit `model` kwarg needs the identical normalization: MoA slots
|
||||
# (agent/moa_loop.py's _slot_runtime) forward a preset's `model:` field as
|
||||
# this explicit argument rather than through auxiliary.<task> config, so a
|
||||
# user-configured `model: auto` on a MoA reference/aggregator slot reaches
|
||||
# this function here, not as cfg_model. Only normalizing cfg_model let that
|
||||
# literal "auto" slip through via `model or cfg_model` below.
|
||||
if model and model.lower() == "auto":
|
||||
model = None
|
||||
if cfg_model and cfg_model.lower() == "auto":
|
||||
cfg_model = None
|
||||
|
||||
resolved_model = model or cfg_model
|
||||
resolved_api_mode = cfg_api_mode
|
||||
|
||||
# MoA virtual provider: an *explicit* `provider: moa` override (either the
|
||||
# caller-passed `provider` arg or `auxiliary.<task>.provider` in
|
||||
# config.yaml) reaches this function directly — it never goes through
|
||||
# _resolve_auto(), which only unwraps the *implicit* "main provider is
|
||||
# moa" case (#53827). Left as-is, "moa" is returned verbatim and
|
||||
# resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has
|
||||
# no "moa" entry — it's not a real HTTP provider), falls to the
|
||||
# unknown-provider dead end, and call_llm surfaces a nonsensical
|
||||
# "MOA_API_KEY environment variable" error for a provider that was never
|
||||
# meant to be reached over the wire. Auxiliary tasks don't need the
|
||||
# reference fan-out — resolve to the preset's aggregator slot instead,
|
||||
# exactly like the implicit path does (shared helper: _resolve_moa_aggregator).
|
||||
def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> Tuple[str, Optional[str]]:
|
||||
if prov.strip().lower() != "moa":
|
||||
return prov, mdl
|
||||
agg_provider, agg_model = _resolve_moa_aggregator(mdl)
|
||||
if agg_provider and agg_model:
|
||||
return agg_provider, agg_model
|
||||
return prov, mdl
|
||||
|
||||
if provider and str(provider).strip().lower() == "moa":
|
||||
provider, resolved_model = _unwrap_moa_provider(provider, resolved_model)
|
||||
# The moa:// virtual endpoint (if any explicit base_url/api_key was
|
||||
# passed alongside provider="moa") belongs to the facade, not the
|
||||
# aggregator's real provider — drop it so the aggregator resolves
|
||||
# through its own provider credentials, mirroring _resolve_auto().
|
||||
if provider and provider.lower() != "moa":
|
||||
base_url = None
|
||||
api_key = None
|
||||
elif cfg_provider and str(cfg_provider).strip().lower() == "moa":
|
||||
cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model)
|
||||
if cfg_provider and cfg_provider.lower() != "moa":
|
||||
resolved_model = cfg_model
|
||||
cfg_base_url = None
|
||||
cfg_api_key = None
|
||||
|
||||
# Convenience aliases for direct API-key endpoints that aren't first-class
|
||||
# providers (e.g. ``provider: openai`` → custom + api.openai.com/v1).
|
||||
# Applied to both explicit args and config-derived values. When the user
|
||||
|
|
|
|||
|
|
@ -433,6 +433,29 @@ def _model_supports_tool_use(model_id: str) -> bool:
|
|||
return not any(pattern in model_lower for pattern in _NON_TOOL_CALLING_PATTERNS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt-cache capability detection (Converse API cachePoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Claude on Bedrock already gets prompt caching through the AnthropicBedrock
|
||||
# SDK path (see is_anthropic_bedrock_model / runtime_provider.py's dual-path
|
||||
# routing) — it never reaches build_converse_kwargs unless bearer-token auth
|
||||
# forces the Converse path (#28156). This allowlist covers the Converse API
|
||||
# itself: sending an unsupported model a cachePoint block raises a
|
||||
# ValidationException, so — like _model_supports_tool_use but inverted —
|
||||
# unknown models default to NOT receiving cache markers until confirmed.
|
||||
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
|
||||
_CACHE_POINT_PATTERNS = [
|
||||
"anthropic.claude", # bearer-token fallback path
|
||||
"amazon.nova",
|
||||
]
|
||||
|
||||
|
||||
def _model_supports_prompt_cache(model_id: str) -> bool:
|
||||
"""Return True if the model accepts a Converse API cachePoint block."""
|
||||
model_lower = model_id.lower()
|
||||
return any(pattern in model_lower for pattern in _CACHE_POINT_PATTERNS)
|
||||
|
||||
|
||||
def is_anthropic_bedrock_model(model_id: str) -> bool:
|
||||
"""Return True if the model is an Anthropic Claude model on Bedrock.
|
||||
|
||||
|
|
@ -764,14 +787,22 @@ def normalize_converse_response(response: Dict) -> SimpleNamespace:
|
|||
reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None,
|
||||
)
|
||||
|
||||
# Build usage stats
|
||||
# Build usage stats. Converse's inputTokens excludes cache read/write
|
||||
# tokens (unlike OpenAI's prompt_tokens, which includes them) — restore
|
||||
# the OpenAI-style "total includes cache" convention here so downstream
|
||||
# normalize_usage() can subtract them back out consistently, and surface
|
||||
# the Anthropic-named fields it already falls back to for cache reads.
|
||||
usage_data = response.get("usage", {})
|
||||
input_tokens = usage_data.get("inputTokens", 0)
|
||||
cache_read_tokens = usage_data.get("cacheReadInputTokens", 0)
|
||||
cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0)
|
||||
output_tokens = usage_data.get("outputTokens", 0)
|
||||
usage = SimpleNamespace(
|
||||
prompt_tokens=usage_data.get("inputTokens", 0),
|
||||
completion_tokens=usage_data.get("outputTokens", 0),
|
||||
total_tokens=(
|
||||
usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0)
|
||||
),
|
||||
prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens,
|
||||
completion_tokens=output_tokens,
|
||||
total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens,
|
||||
cache_read_input_tokens=cache_read_tokens,
|
||||
cache_creation_input_tokens=cache_write_tokens,
|
||||
)
|
||||
|
||||
finish_reason = _converse_stop_reason_to_openai(stop_reason)
|
||||
|
|
@ -936,6 +967,8 @@ def stream_converse_with_callbacks(
|
|||
usage_data = {
|
||||
"inputTokens": meta_usage.get("inputTokens", 0),
|
||||
"outputTokens": meta_usage.get("outputTokens", 0),
|
||||
"cacheReadInputTokens": meta_usage.get("cacheReadInputTokens", 0),
|
||||
"cacheWriteInputTokens": meta_usage.get("cacheWriteInputTokens", 0),
|
||||
}
|
||||
|
||||
# Flush remaining text
|
||||
|
|
@ -949,12 +982,16 @@ def stream_converse_with_callbacks(
|
|||
reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None,
|
||||
)
|
||||
|
||||
input_tokens = usage_data.get("inputTokens", 0)
|
||||
cache_read_tokens = usage_data.get("cacheReadInputTokens", 0)
|
||||
cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0)
|
||||
output_tokens = usage_data.get("outputTokens", 0)
|
||||
usage = SimpleNamespace(
|
||||
prompt_tokens=usage_data.get("inputTokens", 0),
|
||||
completion_tokens=usage_data.get("outputTokens", 0),
|
||||
total_tokens=(
|
||||
usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0)
|
||||
),
|
||||
prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens,
|
||||
completion_tokens=output_tokens,
|
||||
total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens,
|
||||
cache_read_input_tokens=cache_read_tokens,
|
||||
cache_creation_input_tokens=cache_write_tokens,
|
||||
)
|
||||
|
||||
finish_reason = _converse_stop_reason_to_openai(stop_reason)
|
||||
|
|
@ -993,6 +1030,7 @@ def build_converse_kwargs(
|
|||
Converts OpenAI-format inputs to Converse API parameters.
|
||||
"""
|
||||
system_prompt, converse_messages = convert_messages_to_converse(messages)
|
||||
cache_enabled = _model_supports_prompt_cache(model)
|
||||
|
||||
kwargs: Dict[str, Any] = {
|
||||
"modelId": model,
|
||||
|
|
@ -1003,6 +1041,8 @@ def build_converse_kwargs(
|
|||
}
|
||||
|
||||
if system_prompt:
|
||||
if cache_enabled:
|
||||
system_prompt = system_prompt + [{"cachePoint": {"type": "default"}}]
|
||||
kwargs["system"] = system_prompt
|
||||
|
||||
from agent.anthropic_adapter import _forbids_sampling_params
|
||||
|
|
@ -1026,6 +1066,8 @@ def build_converse_kwargs(
|
|||
# Strip tools for known non-tool-calling models and warn the user.
|
||||
# Ref: PR #7920 feedback from @ptlally, pattern from PR #4346.
|
||||
if _model_supports_tool_use(model):
|
||||
if cache_enabled:
|
||||
converse_tools = converse_tools + [{"cachePoint": {"type": "default"}}]
|
||||
kwargs["toolConfig"] = {"tools": converse_tools}
|
||||
else:
|
||||
logger.warning(
|
||||
|
|
@ -1033,6 +1075,14 @@ def build_converse_kwargs(
|
|||
"The agent will operate in text-only mode.", model
|
||||
)
|
||||
|
||||
if cache_enabled and len(converse_messages) >= 2:
|
||||
# Checkpoint everything up to (not including) the newest turn, so the
|
||||
# marker survives unchanged across requests as only the tail grows —
|
||||
# mirroring the Anthropic system_and_3 strategy in prompt_caching.py.
|
||||
content = converse_messages[-2].get("content")
|
||||
if isinstance(content, list) and content:
|
||||
content.append({"cachePoint": {"type": "default"}})
|
||||
|
||||
if guardrail_config:
|
||||
kwargs["guardrailConfig"] = guardrail_config
|
||||
|
||||
|
|
|
|||
|
|
@ -335,6 +335,11 @@ _ACTIVE_TASK_MAX_CHARS = 1400
|
|||
# high for small/light tails, but using all 20 as a hard floor here would bring
|
||||
# back the old large-tool-output case where nothing can be compacted.
|
||||
_MAX_TAIL_MESSAGE_FLOOR = 8
|
||||
# Under context pressure (protected-tail tool bodies alone exceed the soft
|
||||
# tail budget), demote large completed tool/file outputs even inside the
|
||||
# protected region — but always keep this many trailing messages verbatim so
|
||||
# the active user ask / latest tool pair remain readable. Issue #61932.
|
||||
_PRESSURE_KEEP_RECENT_MESSAGES = 3
|
||||
|
||||
# Models with context windows below this get their compression threshold
|
||||
# floored at ``_SMALL_CTX_THRESHOLD_PERCENT`` (raise-only — an explicitly
|
||||
|
|
@ -1129,8 +1134,10 @@ class ContextCompressor(ContextEngine):
|
|||
self._last_summary_error = None
|
||||
self._consecutive_timeout_failures = 0
|
||||
self._fallback_compression_streak = 0
|
||||
self._ineffective_compression_count = 0
|
||||
self.get_active_compression_failure_cooldown()
|
||||
self._load_fallback_compression_streak()
|
||||
self._load_ineffective_compression_count()
|
||||
|
||||
def on_session_start(self, session_id: str, **kwargs) -> None:
|
||||
"""Bind session-scoped compression state for a new or resumed session."""
|
||||
|
|
@ -1139,6 +1146,7 @@ class ContextCompressor(ContextEngine):
|
|||
old_session_id = kwargs.get("old_session_id")
|
||||
session_db = kwargs.get("session_db", getattr(self, "_session_db", None))
|
||||
previous_fallback_streak = self._fallback_compression_streak
|
||||
previous_ineffective_count = self._ineffective_compression_count
|
||||
if boundary_reason == "compression" and old_session_id:
|
||||
getter = getattr(session_db, "get_compression_fallback_streak", None)
|
||||
if callable(getter):
|
||||
|
|
@ -1153,12 +1161,37 @@ class ContextCompressor(ContextEngine):
|
|||
"compression parent fallback streak lookup failed (non-sqlite): %s",
|
||||
exc,
|
||||
)
|
||||
count_getter = getattr(
|
||||
session_db, "get_compression_ineffective_count", None,
|
||||
)
|
||||
if callable(count_getter):
|
||||
try:
|
||||
stored_count = count_getter(old_session_id)
|
||||
if isinstance(stored_count, (int, float, str)):
|
||||
previous_ineffective_count = max(0, int(stored_count))
|
||||
except (TypeError, ValueError, sqlite3.Error) as exc:
|
||||
logger.debug(
|
||||
"compression parent ineffective count lookup failed: %s", exc,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"compression parent ineffective count lookup failed (non-sqlite): %s",
|
||||
exc,
|
||||
)
|
||||
self.bind_session_state(session_db, session_id)
|
||||
if boundary_reason == "compression":
|
||||
# Rotation creates a fresh child row before this callback. Preserve
|
||||
# the logical conversation's streak until boundary bookkeeping
|
||||
# persists the updated value onto the child row.
|
||||
self._fallback_compression_streak = previous_fallback_streak
|
||||
# Same for the anti-thrash strike counter — but unlike the streak,
|
||||
# no later boundary bookkeeping writes it, so persist the carried
|
||||
# value onto the (fresh) child row now. Otherwise a restart between
|
||||
# rotation and the next real-usage verdict would silently disarm
|
||||
# an armed guard (#54923).
|
||||
if self._ineffective_compression_count != previous_ineffective_count:
|
||||
self._ineffective_compression_count = previous_ineffective_count
|
||||
self._persist_ineffective_compression_count()
|
||||
|
||||
def _load_fallback_compression_streak(self) -> None:
|
||||
session_db = getattr(self, "_session_db", None)
|
||||
|
|
@ -1192,6 +1225,59 @@ class ContextCompressor(ContextEngine):
|
|||
except Exception as exc:
|
||||
logger.debug("compression fallback streak persist failed (non-sqlite): %s", exc)
|
||||
|
||||
def _load_ineffective_compression_count(self) -> None:
|
||||
"""Load the durable anti-thrash strike count for the bound session.
|
||||
|
||||
A fresh compressor on a resumed session starts with
|
||||
``compression_count == 0`` and, historically, an in-memory-only
|
||||
ineffective counter — so a guard armed (1 strike) or tripped
|
||||
(2 strikes) before a process restart silently disarmed, and a
|
||||
near-threshold session could re-compact once per restart forever
|
||||
(#54923). The counter now round-trips through the session row like
|
||||
the failure cooldown and the fallback streak.
|
||||
"""
|
||||
session_db = getattr(self, "_session_db", None)
|
||||
session_id = getattr(self, "_session_id", "")
|
||||
getter = getattr(session_db, "get_compression_ineffective_count", None)
|
||||
if not session_id or not callable(getter):
|
||||
return
|
||||
try:
|
||||
stored_count = getter(session_id)
|
||||
self._ineffective_compression_count = max(
|
||||
0,
|
||||
int(stored_count)
|
||||
if isinstance(stored_count, (int, float, str))
|
||||
else 0,
|
||||
)
|
||||
except (TypeError, ValueError, sqlite3.Error) as exc:
|
||||
logger.debug("compression ineffective count lookup failed: %s", exc)
|
||||
except Exception as exc:
|
||||
logger.debug("compression ineffective count lookup failed (non-sqlite): %s", exc)
|
||||
|
||||
def _persist_ineffective_compression_count(self) -> None:
|
||||
session_db = getattr(self, "_session_db", None)
|
||||
session_id = getattr(self, "_session_id", "")
|
||||
setter = getattr(session_db, "set_compression_ineffective_count", None)
|
||||
if not session_id or not callable(setter):
|
||||
return
|
||||
try:
|
||||
setter(session_id, self._ineffective_compression_count)
|
||||
except sqlite3.Error as exc:
|
||||
logger.debug("compression ineffective count persist failed: %s", exc)
|
||||
except Exception as exc:
|
||||
logger.debug("compression ineffective count persist failed (non-sqlite): %s", exc)
|
||||
|
||||
def _record_ineffective_compression_verdict(self, count: int) -> None:
|
||||
"""Set the anti-thrash strike counter, keeping the durable copy in sync.
|
||||
|
||||
Persists only on change so the reset issued by every ordinary fitting
|
||||
response (already-zero -> zero) never costs a DB write.
|
||||
"""
|
||||
if count == self._ineffective_compression_count:
|
||||
return
|
||||
self._ineffective_compression_count = count
|
||||
self._persist_ineffective_compression_count()
|
||||
|
||||
def record_completed_compaction(self, *, used_fallback: bool = False) -> None:
|
||||
"""Record one completed boundary and its summary quality."""
|
||||
self._verify_compaction_cleared_threshold = True
|
||||
|
|
@ -1400,7 +1486,10 @@ class ContextCompressor(ContextEngine):
|
|||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
self.last_compression_rough_tokens = 0
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
self._ineffective_compression_count = 0
|
||||
# Strikes were judged against the PREVIOUS threshold; a recomputed
|
||||
# trigger invalidates them. Keep the durable copy in sync so a
|
||||
# restart doesn't resurrect strikes this recalibration just voided.
|
||||
self._record_ineffective_compression_verdict(0)
|
||||
if runtime_changed:
|
||||
self._fallback_compression_streak = 0
|
||||
self._persist_fallback_compression_streak()
|
||||
|
|
@ -1724,7 +1813,7 @@ class ContextCompressor(ContextEngine):
|
|||
# when this response was not immediately after compaction. The
|
||||
# independent fallback streak is boundary-scoped and survives
|
||||
# ordinary fitting responses during context regrowth.
|
||||
self._ineffective_compression_count = 0
|
||||
self._record_ineffective_compression_verdict(0)
|
||||
else:
|
||||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
|
||||
|
|
@ -1746,7 +1835,9 @@ class ContextCompressor(ContextEngine):
|
|||
# per compaction.
|
||||
if self._verify_compaction_cleared_threshold:
|
||||
if self.last_prompt_tokens >= self.threshold_tokens:
|
||||
self._ineffective_compression_count += 1
|
||||
self._record_ineffective_compression_verdict(
|
||||
self._ineffective_compression_count + 1,
|
||||
)
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Compaction did not clear the threshold: %d real "
|
||||
|
|
@ -1758,7 +1849,7 @@ class ContextCompressor(ContextEngine):
|
|||
self._ineffective_compression_count,
|
||||
)
|
||||
else:
|
||||
self._ineffective_compression_count = 0
|
||||
self._record_ineffective_compression_verdict(0)
|
||||
# Consume the pending-verification flag once real usage arrives, whether
|
||||
# or not prompt_tokens was reported, so a usage-less response can't leave
|
||||
# it armed for a later, unrelated reading.
|
||||
|
|
@ -1810,23 +1901,83 @@ class ContextCompressor(ContextEngine):
|
|||
def should_compress(self, prompt_tokens: int = None) -> bool:
|
||||
"""Check if context exceeds the compression threshold.
|
||||
|
||||
Returns ``True`` when compression should run now. For the caller-facing
|
||||
*reason* (e.g. why compression is skipped while still over threshold),
|
||||
see :meth:`should_compress_info`, which returns a ``(bool, reason)``
|
||||
tuple without changing the decision logic here.
|
||||
|
||||
Includes anti-thrashing protection: if the last two compressions
|
||||
each saved less than 10%, skip compression to avoid infinite loops
|
||||
where each pass removes only 1-2 messages.
|
||||
"""
|
||||
decision, _reason = self.should_compress_info(prompt_tokens)
|
||||
return decision
|
||||
|
||||
def should_compress_info(
|
||||
self, prompt_tokens: int = None
|
||||
) -> "tuple[bool, str | None]":
|
||||
"""Check if context exceeds the compression threshold.
|
||||
|
||||
Returns a ``(should_compress, reason)`` tuple instead of a bare bool so
|
||||
callers can tell *why* compression is skipped when it is skipped while
|
||||
the context is already over threshold. ``reason`` is ``None`` unless
|
||||
compression is needed but blocked:
|
||||
|
||||
* ``"cooldown:<seconds>"`` — the summary LLM is recovering from a
|
||||
recent 429/transient failure; compression is deferred to avoid the
|
||||
freeze loop described in #11529.
|
||||
* ``"ineffective"`` — anti-thrashing has backed off because the last
|
||||
two compressions each saved <10%.
|
||||
|
||||
When ``reason`` is non-``None`` the session is over its compression
|
||||
threshold yet cannot shrink — callers should surface a warning so the
|
||||
user knows the model may silently stop answering (the context keeps
|
||||
growing until it hits the hard provider limit). Without this signal an
|
||||
over-threshold session fails opaquely.
|
||||
|
||||
Includes anti-thrashing protection: if the last two compressions
|
||||
each saved less than 10%, skip compression to avoid infinite loops
|
||||
where each pass removes only 1-2 messages.
|
||||
"""
|
||||
tokens = prompt_tokens if prompt_tokens is not None else self.last_prompt_tokens
|
||||
if tokens < self.threshold_tokens:
|
||||
return False
|
||||
return not self._automatic_compression_blocked()
|
||||
return False, None
|
||||
if self._automatic_compression_blocked():
|
||||
return False, self._compression_block_reason() or "blocked"
|
||||
return True, None
|
||||
|
||||
def _compression_block_reason(self) -> "str | None":
|
||||
"""Return a human-readable reason for the current automatic-compaction
|
||||
block, derived from the same in-memory state that
|
||||
:meth:`_automatic_compression_blocked_locally` evaluates.
|
||||
|
||||
* ``"cooldown:<seconds>"`` — the summary LLM is recovering from a
|
||||
recent 429/transient failure; compression is deferred to avoid the
|
||||
freeze loop described in #11529.
|
||||
* ``"ineffective"`` — anti-thrashing has backed off (the last two
|
||||
compressions each saved <10%, or the fallback streak tripped).
|
||||
* ``None`` — no block active.
|
||||
"""
|
||||
_cooldown_remaining = self._summary_failure_cooldown_until - time.monotonic()
|
||||
if _cooldown_remaining > 0:
|
||||
return f"cooldown:{_cooldown_remaining:.0f}"
|
||||
if (
|
||||
self._ineffective_compression_count >= 2
|
||||
or self._fallback_compression_streak >= 2
|
||||
):
|
||||
return "ineffective"
|
||||
return None
|
||||
|
||||
def _refresh_durable_guards(self) -> None:
|
||||
"""Re-read durable cooldown + fallback-streak state from the DB.
|
||||
"""Re-read durable cooldown + breaker state from the DB.
|
||||
|
||||
Cheap, best-effort, and only called when a gate is about to say
|
||||
"blocked": another agent on the same session may have cleared the
|
||||
durable rows (successful boundary, forced retry) after this
|
||||
compressor was bound, and a fallback streak has no timer — without
|
||||
a re-read the stale in-memory snapshot blocks forever.
|
||||
durable rows (successful boundary, forced retry, a real usage
|
||||
reading that dipped below the threshold) after this compressor was
|
||||
bound, and neither the fallback streak nor the ineffective-strike
|
||||
counter has a timer — without a re-read the stale in-memory
|
||||
snapshot blocks forever.
|
||||
"""
|
||||
try:
|
||||
self.get_active_compression_failure_cooldown(refresh=True)
|
||||
|
|
@ -1836,26 +1987,22 @@ class ContextCompressor(ContextEngine):
|
|||
self._load_fallback_compression_streak()
|
||||
except Exception as exc:
|
||||
logger.debug("compression fallback-streak refresh failed: %s", exc)
|
||||
try:
|
||||
self._load_ineffective_compression_count()
|
||||
except Exception as exc:
|
||||
logger.debug("compression ineffective-count refresh failed: %s", exc)
|
||||
|
||||
def _automatic_compression_blocked(self) -> bool:
|
||||
"""Return whether automatic compaction is in cooldown or tripped."""
|
||||
if not self._automatic_compression_blocked_locally():
|
||||
return False
|
||||
# Blocked on the in-memory snapshot. Durable guard rows may have
|
||||
# been cleared by another agent since bind_session_state(); refresh
|
||||
# and re-evaluate so a stale local block cannot outlive the durable
|
||||
# state that justified it. The unblocked hot path above never pays
|
||||
# for the DB reads.
|
||||
if (
|
||||
self._summary_failure_cooldown_until <= time.monotonic()
|
||||
and self._fallback_compression_streak < 2
|
||||
):
|
||||
# Blocked solely by the in-memory ineffective-compression
|
||||
# counter, which is not durable — there is nothing in the DB
|
||||
# that could unblock it, so skip the refresh (otherwise this
|
||||
# branch would re-read the DB on every gate check for the rest
|
||||
# of the session).
|
||||
return True
|
||||
# been cleared by another agent since bind_session_state() — a
|
||||
# successful boundary, a forced retry, or a real usage reading
|
||||
# below the threshold (which zeroes the durable ineffective
|
||||
# counter) — so refresh and re-evaluate before letting a stale
|
||||
# local block outlive the durable state that justified it. The
|
||||
# unblocked hot path above never pays for the DB reads.
|
||||
self._refresh_durable_guards()
|
||||
return self._automatic_compression_blocked_locally()
|
||||
|
||||
|
|
@ -1918,7 +2065,14 @@ class ContextCompressor(ContextEngine):
|
|||
fall within ``protect_tail_tokens`` (when provided) OR the last
|
||||
``protect_tail_count`` messages (backward-compatible default).
|
||||
When both are given, the token budget takes priority and the message
|
||||
count acts as a hard minimum floor.
|
||||
count acts as a hard minimum floor — capped at
|
||||
``_MAX_TAIL_MESSAGE_FLOOR`` so a default ``protect_last_n=20`` cannot
|
||||
freeze a whole run of bulky tool outputs against pruning.
|
||||
|
||||
When the protected region itself still exceeds the soft tail budget
|
||||
(``protect_tail_tokens * 1.5``), a pressure pass demotes large
|
||||
completed tool/file outputs *inside* that region while keeping a
|
||||
short recent floor verbatim (issue #61932).
|
||||
|
||||
Returns (pruned_messages, pruned_count).
|
||||
"""
|
||||
|
|
@ -1946,10 +2100,17 @@ class ContextCompressor(ContextEngine):
|
|||
|
||||
# Determine the prune boundary
|
||||
if protect_tail_tokens is not None and protect_tail_tokens > 0:
|
||||
# Token-budget approach: walk backward accumulating tokens
|
||||
# Token-budget approach: walk backward accumulating tokens.
|
||||
# Cap the message-count floor the same way tail-cut does so a
|
||||
# default protect_last_n=20 cannot lock a bulky recent tool run
|
||||
# outside the compressible / prunable window (#61932).
|
||||
accumulated = 0
|
||||
boundary = len(result)
|
||||
min_protect = min(protect_tail_count, len(result))
|
||||
min_protect = min(
|
||||
protect_tail_count,
|
||||
len(result),
|
||||
_MAX_TAIL_MESSAGE_FLOOR,
|
||||
)
|
||||
for i in range(len(result) - 1, -1, -1):
|
||||
msg = result[i]
|
||||
msg_tokens = _estimate_msg_budget_tokens(msg)
|
||||
|
|
@ -1997,54 +2158,53 @@ class ContextCompressor(ContextEngine):
|
|||
else:
|
||||
content_hashes[h] = (i, msg.get("tool_call_id", "?"))
|
||||
|
||||
# Pass 2: Replace old tool results with informative summaries
|
||||
for i in range(prune_boundary):
|
||||
msg = result[i]
|
||||
def _demote_tool_result_at(idx: int) -> bool:
|
||||
"""Replace a bulky tool result at ``idx`` with a 1-line summary.
|
||||
|
||||
Returns True when the message was modified.
|
||||
"""
|
||||
nonlocal pruned
|
||||
msg = result[idx]
|
||||
if msg.get("role") != "tool":
|
||||
continue
|
||||
return False
|
||||
content = msg.get("content", "")
|
||||
# Multimodal content (base64 screenshots etc.): strip the image
|
||||
# payload — keep a lightweight text placeholder in its place.
|
||||
# Without this, an old computer_use screenshot (~1MB base64 +
|
||||
# ~1500 real tokens) survives every compression pass forever.
|
||||
if isinstance(content, list):
|
||||
stripped = _strip_image_parts_from_parts(content)
|
||||
if stripped is not None:
|
||||
result[i] = {**msg, "content": stripped}
|
||||
result[idx] = {**msg, "content": stripped}
|
||||
pruned += 1
|
||||
continue
|
||||
return True
|
||||
return False
|
||||
if isinstance(content, dict) and content.get("_multimodal"):
|
||||
summary = content.get("text_summary") or "[screenshot removed to save context]"
|
||||
result[i] = {**msg, "content": f"[screenshot removed] {summary[:200]}"}
|
||||
result[idx] = {**msg, "content": f"[screenshot removed] {summary[:200]}"}
|
||||
pruned += 1
|
||||
continue
|
||||
return True
|
||||
if not isinstance(content, str):
|
||||
continue
|
||||
return False
|
||||
if not content or content == _PRUNED_TOOL_PLACEHOLDER:
|
||||
continue
|
||||
# Skip already-deduplicated or previously-summarized results
|
||||
return False
|
||||
if content.startswith("[Duplicate tool output"):
|
||||
continue
|
||||
# Only prune if the content is substantial (>200 chars)
|
||||
if len(content) > 200:
|
||||
call_id = msg.get("tool_call_id", "")
|
||||
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
|
||||
summary = _summarize_tool_result(tool_name, tool_args, content)
|
||||
result[i] = {**msg, "content": summary}
|
||||
pruned += 1
|
||||
return False
|
||||
# Already replaced by a prior prune/pressure pass (1-line summary).
|
||||
if content.startswith("[") and " chars)" in content and len(content) < 400:
|
||||
return False
|
||||
if content.startswith("[screenshot removed"):
|
||||
return False
|
||||
if len(content) <= 200:
|
||||
return False
|
||||
call_id = msg.get("tool_call_id", "")
|
||||
tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
|
||||
summary = _summarize_tool_result(tool_name, tool_args, content)
|
||||
result[idx] = {**msg, "content": summary}
|
||||
pruned += 1
|
||||
return True
|
||||
|
||||
# Pass 3: Truncate large tool_call arguments in assistant messages
|
||||
# outside the protected tail. write_file with 50KB content, for
|
||||
# example, survives pruning entirely without this.
|
||||
#
|
||||
# The shrinking is done inside the parsed JSON structure so the
|
||||
# result remains valid JSON — otherwise downstream providers 400
|
||||
# on every subsequent turn until the broken call falls out of
|
||||
# the window. See ``_truncate_tool_call_args_json`` docstring.
|
||||
for i in range(prune_boundary):
|
||||
msg = result[i]
|
||||
def _truncate_tool_call_args_at(idx: int) -> bool:
|
||||
"""Shrink large tool_call argument payloads at ``idx``."""
|
||||
msg = result[idx]
|
||||
if msg.get("role") != "assistant" or not msg.get("tool_calls"):
|
||||
continue
|
||||
return False
|
||||
new_tcs = []
|
||||
modified = False
|
||||
for tc in msg["tool_calls"]:
|
||||
|
|
@ -2057,7 +2217,93 @@ class ContextCompressor(ContextEngine):
|
|||
modified = True
|
||||
new_tcs.append(tc)
|
||||
if modified:
|
||||
result[i] = {**msg, "tool_calls": new_tcs}
|
||||
result[idx] = {**msg, "tool_calls": new_tcs}
|
||||
return modified
|
||||
|
||||
# Pass 2: Replace old tool results with informative summaries
|
||||
for i in range(max(0, prune_boundary)):
|
||||
_demote_tool_result_at(i)
|
||||
|
||||
# Pass 3: Truncate large tool_call arguments in assistant messages
|
||||
# outside the protected tail. write_file with 50KB content, for
|
||||
# example, survives pruning entirely without this.
|
||||
#
|
||||
# The shrinking is done inside the parsed JSON structure so the
|
||||
# result remains valid JSON — otherwise downstream providers 400
|
||||
# on every subsequent turn until the broken call falls out of
|
||||
# the window. See ``_truncate_tool_call_args_json`` docstring.
|
||||
for i in range(max(0, prune_boundary)):
|
||||
_truncate_tool_call_args_at(i)
|
||||
|
||||
# Pass 4 (issue #61932): protected-tail pressure demotion.
|
||||
# After multiple in-place compactions the transcript can be short
|
||||
# enough that nearly every remaining message sits inside the
|
||||
# protected floor, yet those messages are huge completed tool /
|
||||
# file outputs. Summarizing the (empty) middle does nothing and
|
||||
# preflight ends in "Cannot compress further". Demote bulky tool
|
||||
# bodies *inside* the protected region until the protected tail
|
||||
# fits the soft budget, always keeping a short recent floor
|
||||
# verbatim so the active ask stays readable.
|
||||
if protect_tail_tokens is not None and protect_tail_tokens > 0 and result:
|
||||
soft_ceiling = int(protect_tail_tokens * 1.5)
|
||||
keep_recent = min(_PRESSURE_KEEP_RECENT_MESSAGES, len(result))
|
||||
demote_end = len(result) - keep_recent
|
||||
|
||||
def _protected_region_tokens() -> int:
|
||||
start = max(0, prune_boundary)
|
||||
return sum(
|
||||
_estimate_msg_budget_tokens(result[i])
|
||||
for i in range(start, len(result))
|
||||
)
|
||||
|
||||
if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling:
|
||||
pressure_hits = 0
|
||||
for i in range(max(0, prune_boundary), demote_end):
|
||||
if _demote_tool_result_at(i):
|
||||
pressure_hits += 1
|
||||
if _truncate_tool_call_args_at(i):
|
||||
pressure_hits += 1
|
||||
if _protected_region_tokens() <= soft_ceiling:
|
||||
break
|
||||
# If the short recent floor itself is still dominated by a
|
||||
# stack of huge tool bodies, demote every protected tool
|
||||
# result except the single most recent one. The active
|
||||
# user message (usually the last row) stays untouched.
|
||||
if _protected_region_tokens() > soft_ceiling:
|
||||
last_tool_idx = None
|
||||
for i in range(len(result) - 1, -1, -1):
|
||||
if result[i].get("role") == "tool":
|
||||
last_tool_idx = i
|
||||
break
|
||||
for i in range(max(0, prune_boundary), len(result)):
|
||||
if last_tool_idx is not None and i == last_tool_idx:
|
||||
continue
|
||||
if result[i].get("role") == "tool":
|
||||
if _demote_tool_result_at(i):
|
||||
pressure_hits += 1
|
||||
elif result[i].get("role") == "assistant":
|
||||
if _truncate_tool_call_args_at(i):
|
||||
pressure_hits += 1
|
||||
# Absolute last resort: even the newest tool body can
|
||||
# be larger than the soft budget alone (one 200KB file
|
||||
# read). Summarize it so compression can still reclaim
|
||||
# enough headroom to continue the session.
|
||||
if (
|
||||
last_tool_idx is not None
|
||||
and last_tool_idx >= prune_boundary
|
||||
and _protected_region_tokens() > soft_ceiling
|
||||
):
|
||||
if _demote_tool_result_at(last_tool_idx):
|
||||
pressure_hits += 1
|
||||
if pressure_hits and not self.quiet_mode:
|
||||
logger.info(
|
||||
"Pre-compression pressure demotion: reclaimed protected-tail "
|
||||
"tool output (%d change(s); protected region now ~%s tokens, "
|
||||
"soft ceiling %s)",
|
||||
pressure_hits,
|
||||
f"{_protected_region_tokens():,}",
|
||||
f"{soft_ceiling:,}",
|
||||
)
|
||||
|
||||
return result, pruned
|
||||
|
||||
|
|
@ -4083,7 +4329,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
# threshold because of the incompressible floor (system prompt +
|
||||
# tool schemas), every subsequent turn re-fires a compaction that
|
||||
# returns here unchanged, and the CLI appears frozen.
|
||||
self._ineffective_compression_count += 1
|
||||
self._record_ineffective_compression_verdict(
|
||||
self._ineffective_compression_count + 1,
|
||||
)
|
||||
self._last_compression_savings_pct = 0.0
|
||||
telemetry["failure_class"] = "insufficient_messages"
|
||||
if not self.quiet_mode:
|
||||
|
|
@ -4151,7 +4399,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
|||
# an ineffective compression the anti-thrashing guard in
|
||||
# should_compress() never fires and every subsequent turn
|
||||
# re-triggers a no-op compression loop. (#40803)
|
||||
self._ineffective_compression_count += 1
|
||||
self._record_ineffective_compression_verdict(
|
||||
self._ineffective_compression_count + 1,
|
||||
)
|
||||
self._last_compression_savings_pct = 0.0
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
|
|
|
|||
|
|
@ -53,6 +53,39 @@ def sanitize_memory_context(memory_context: str) -> str:
|
|||
)
|
||||
|
||||
|
||||
def automatic_compaction_status_message(
|
||||
engine: Any,
|
||||
*,
|
||||
phase: str,
|
||||
default_message: str,
|
||||
**context: Any,
|
||||
) -> str | None:
|
||||
"""Resolve host-visible status for an automatic compaction event.
|
||||
|
||||
Engines can suppress routine automatic status with
|
||||
``emit_automatic_compaction_status = False`` or customize it by defining
|
||||
``get_automatic_compaction_status_message(...)``. Empty strings and
|
||||
``None`` mean "do not emit a lifecycle status".
|
||||
"""
|
||||
if not getattr(engine, "emit_automatic_compaction_status", True):
|
||||
return None
|
||||
|
||||
formatter = getattr(engine, "get_automatic_compaction_status_message", None)
|
||||
if callable(formatter):
|
||||
message = formatter(
|
||||
phase=phase,
|
||||
default_message=default_message,
|
||||
**context,
|
||||
)
|
||||
else:
|
||||
message = default_message
|
||||
|
||||
if message is None:
|
||||
return None
|
||||
message = str(message).strip()
|
||||
return message or None
|
||||
|
||||
|
||||
class ContextEngine(ABC):
|
||||
"""Base class all context engines must implement."""
|
||||
|
||||
|
|
@ -89,6 +122,12 @@ class ContextEngine(ABC):
|
|||
protect_first_n: int = 3
|
||||
protect_last_n: int = 6
|
||||
|
||||
# User-visible lifecycle status for automatic host-triggered compaction.
|
||||
# Alternative engines that treat compaction as routine background
|
||||
# maintenance can set this false to keep successful automatic passes silent;
|
||||
# warnings, errors, and explicit manual commands should still surface.
|
||||
emit_automatic_compaction_status: bool = True
|
||||
|
||||
# -- Core interface ----------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
|
|
@ -107,6 +146,19 @@ class ContextEngine(ABC):
|
|||
def should_compress(self, prompt_tokens: int = None) -> bool:
|
||||
"""Return True if compaction should fire this turn."""
|
||||
|
||||
def should_compress_info(self, prompt_tokens: int = None) -> "tuple[bool, str | None]":
|
||||
"""Return ``(should_compress, reason)``.
|
||||
|
||||
The base implementation is backward-compatible: engines that only
|
||||
implement ``should_compress`` get ``(should_compress(prompt_tokens),
|
||||
None)``. Concrete engines with richer block reasons (e.g. a
|
||||
summary-LLM cooldown or an anti-thrashing guard) override this to
|
||||
surface a human-readable reason so callers can warn the user instead
|
||||
of silently skipping compression. Added for the silent-overflow
|
||||
warning fix (#62625) so plugin engines don't raise AttributeError.
|
||||
"""
|
||||
return self.should_compress(prompt_tokens), None
|
||||
|
||||
@abstractmethod
|
||||
def compress(
|
||||
self,
|
||||
|
|
@ -156,6 +208,27 @@ class ContextEngine(ABC):
|
|||
"""
|
||||
return False
|
||||
|
||||
def get_automatic_compaction_status_message(
|
||||
self,
|
||||
*,
|
||||
phase: str,
|
||||
default_message: str,
|
||||
**context: Any,
|
||||
) -> str | None:
|
||||
"""Return user-visible status for automatic host-triggered compaction.
|
||||
|
||||
Return ``None`` to suppress successful automatic lifecycle status for
|
||||
this compaction event. ``phase`` identifies the host call site (for
|
||||
example ``"preflight"`` or ``"compress"``). ``context`` contains
|
||||
best-effort fields such as ``approx_tokens`` and ``threshold_tokens``.
|
||||
|
||||
This hook does not control warning/error messages or explicit manual
|
||||
commands such as ``/compress``.
|
||||
"""
|
||||
if not self.emit_automatic_compaction_status:
|
||||
return None
|
||||
return default_message
|
||||
|
||||
# -- Optional: manual /compress preflight ------------------------------
|
||||
|
||||
def has_content_to_compress(self, messages: List[Dict[str, Any]]) -> bool:
|
||||
|
|
|
|||
|
|
@ -42,7 +42,10 @@ from datetime import datetime
|
|||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
from agent.context_engine import sanitize_memory_context
|
||||
from agent.context_engine import (
|
||||
automatic_compaction_status_message,
|
||||
sanitize_memory_context,
|
||||
)
|
||||
from agent.model_metadata import estimate_request_tokens_rough
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -106,6 +109,22 @@ COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE = (
|
|||
"🗜️ Context reduced to {new_ctx:,} tokens (was {old_ctx:,}), retrying..."
|
||||
)
|
||||
|
||||
# FAILURE-CLASS notice — a deliberate carve-out from routine-compression
|
||||
# silence (#16775 class): the context is over the compression threshold but
|
||||
# compression is blocked (summary-LLM cooldown / anti-thrash breaker), so the
|
||||
# session will keep growing until the hard provider token limit kills it.
|
||||
# This MUST stay visible on chat gateways. Do NOT add it to
|
||||
# ROUTINE_COMPRESSION_STATUS_SAMPLES or the gateway noise regex
|
||||
# (_TELEGRAM_NOISY_STATUS_RE); it is pinned un-swallowed in
|
||||
# tests/gateway/test_telegram_noise_filter.py::VISIBLE_COMPRESSION_MESSAGES.
|
||||
CONTEXT_OVERFLOW_BLOCKED_WARNING_TEMPLATE = (
|
||||
"⚠ Context is over the compression threshold "
|
||||
"(~{tokens:,} tokens >= {threshold:,}) "
|
||||
"but compression is currently blocked ({reason}). "
|
||||
"The model may stop responding. Run /new to start a fresh "
|
||||
"session or /compress to retry immediately."
|
||||
)
|
||||
|
||||
# Sample-formatted instances of every routine compression status line, for
|
||||
# behavioral tests that iterate the ACTUAL emitted wording (formatted from the
|
||||
# same constants the emission sites use) through the gateway noise filter.
|
||||
|
|
@ -188,6 +207,64 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo
|
|||
return True
|
||||
|
||||
|
||||
class CompressionCommitFence:
|
||||
"""Fence timeout cancellation against post-summary session mutation.
|
||||
|
||||
Compression itself is synchronous and may be running in an executor thread.
|
||||
A caller can stop waiting for the summary, but it cannot kill that thread.
|
||||
This fence makes the commit boundary deterministic: cancellation either wins
|
||||
before session mutation starts, or waits until an already-started commit is
|
||||
fully complete before the caller proceeds.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._cancelled = False
|
||||
self._commit_started = False
|
||||
|
||||
def cancel_before_commit(self) -> bool:
|
||||
"""Cancel a pending commit, or wait for an active commit to finish.
|
||||
|
||||
Returns ``True`` when cancellation won before the commit boundary.
|
||||
Returns ``False`` when the worker had already entered the boundary; in
|
||||
that case acquiring this lock waits until all session mutation finishes.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._commit_started:
|
||||
return False
|
||||
self._cancelled = True
|
||||
return True
|
||||
|
||||
def try_cancel_before_commit(self) -> Optional[bool]:
|
||||
"""Non-blocking form of :meth:`cancel_before_commit`.
|
||||
|
||||
Returns ``None`` while an active commit owns the fence, allowing an
|
||||
async caller to yield instead of blocking its event loop.
|
||||
"""
|
||||
if not self._lock.acquire(blocking=False):
|
||||
return None
|
||||
try:
|
||||
if self._commit_started:
|
||||
return False
|
||||
self._cancelled = True
|
||||
return True
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def begin_commit(self) -> bool:
|
||||
"""Enter the commit boundary unless cancellation already won."""
|
||||
self._lock.acquire()
|
||||
if self._cancelled:
|
||||
self._lock.release()
|
||||
return False
|
||||
self._commit_started = True
|
||||
return True
|
||||
|
||||
def finish_commit(self) -> None:
|
||||
"""Leave a commit boundary entered by :meth:`begin_commit`."""
|
||||
self._lock.release()
|
||||
|
||||
|
||||
def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool:
|
||||
"""Whether the live in-memory SessionDB class structurally predates locks.
|
||||
|
||||
|
|
@ -216,6 +293,7 @@ def _refresh_persisted_compression_guards(compressor: Any) -> None:
|
|||
method_calls = (
|
||||
("get_active_compression_failure_cooldown", {"refresh": True}),
|
||||
("_load_fallback_compression_streak", {}),
|
||||
("_load_ineffective_compression_count", {}),
|
||||
)
|
||||
for method_name, kwargs in method_calls:
|
||||
method = getattr(type(compressor), method_name, None)
|
||||
|
|
@ -730,7 +808,11 @@ def replay_compression_warning(agent: Any) -> None:
|
|||
pass
|
||||
|
||||
|
||||
def conversation_history_after_compression(agent: Any, messages: list) -> Optional[list]:
|
||||
def conversation_history_after_compression(
|
||||
agent: Any,
|
||||
messages: list,
|
||||
previous_history: Optional[list] = None,
|
||||
) -> Optional[list]:
|
||||
"""Return the correct flush baseline after a compression boundary.
|
||||
|
||||
Legacy compression rotates to a fresh child session. That child has not
|
||||
|
|
@ -747,7 +829,19 @@ def conversation_history_after_compression(agent: Any, messages: list) -> Option
|
|||
|
||||
A shallow copy is intentional: it captures the current compacted dict
|
||||
identities as history while allowing later same-turn appends to remain new.
|
||||
|
||||
An aborted or no-op attempt after an earlier in-place compaction must retain
|
||||
the pre-attempt baseline. Treating all current messages as persisted would
|
||||
drop any later, unflushed turns on restart; clearing the baseline would
|
||||
append the already-persisted compacted rows a second time.
|
||||
"""
|
||||
if bool(getattr(agent, "_last_compression_attempt_recorded", False)):
|
||||
attempt_in_place = getattr(agent, "_last_compression_attempt_in_place", None)
|
||||
if attempt_in_place is True:
|
||||
return list(messages)
|
||||
if attempt_in_place is False:
|
||||
return None
|
||||
return previous_history
|
||||
if bool(getattr(agent, "_last_compaction_in_place", False)):
|
||||
return list(messages)
|
||||
return None
|
||||
|
|
@ -806,6 +900,37 @@ def _is_real_user_message(message: Any) -> bool:
|
|||
return not ContextCompressor._is_synthetic_compression_user_turn(message)
|
||||
|
||||
|
||||
def _strip_stale_todo_snapshot(content: Any) -> Any:
|
||||
"""Remove a previously merged todo-snapshot block from message content.
|
||||
|
||||
Snapshot merges (see the injection site in ``compress_context``) always
|
||||
append the block at the end of the trailing user turn, so a surviving
|
||||
header marks stale todo state from an earlier compaction boundary.
|
||||
Stripping before re-injection keeps repeated boundaries from
|
||||
accumulating outdated snapshots (#26981).
|
||||
"""
|
||||
from tools.todo_tool import TODO_INJECTION_HEADER
|
||||
|
||||
if isinstance(content, str):
|
||||
idx = content.find(TODO_INJECTION_HEADER)
|
||||
if idx == -1:
|
||||
return content
|
||||
return content[:idx].rstrip()
|
||||
if isinstance(content, list):
|
||||
return [
|
||||
part
|
||||
for part in content
|
||||
if not (
|
||||
isinstance(part, dict)
|
||||
and part.get("type") == "text"
|
||||
and str(part.get("text") or "")
|
||||
.lstrip()
|
||||
.startswith(TODO_INJECTION_HEADER)
|
||||
)
|
||||
]
|
||||
return content
|
||||
|
||||
|
||||
def _merge_anchor_into_user_message(target: dict, anchor: dict) -> None:
|
||||
"""Fold the human anchor into an existing user-role scaffolding turn.
|
||||
|
||||
|
|
@ -975,6 +1100,7 @@ def compress_context(
|
|||
focus_topic: Optional[str] = None,
|
||||
force: bool = False,
|
||||
defer_context_engine_notification: bool = False,
|
||||
commit_fence: Optional[CompressionCommitFence] = None,
|
||||
) -> Tuple[list, str]:
|
||||
"""Compress conversation context and split the session in SQLite.
|
||||
|
||||
|
|
@ -994,6 +1120,9 @@ def compress_context(
|
|||
callers use the default ``False``.
|
||||
defer_context_engine_notification: Delay the existing context-engine
|
||||
hook until a manual host commits its outer history transaction.
|
||||
commit_fence: Optional cooperative fence for executor callers that
|
||||
may time out. It prevents a late worker from mutating session state
|
||||
after its caller has moved on.
|
||||
|
||||
Returns:
|
||||
``(compressed_messages, new_system_prompt)`` tuple. When
|
||||
|
|
@ -1008,6 +1137,13 @@ def compress_context(
|
|||
):
|
||||
raise RuntimeError("a compression notification is already pending")
|
||||
|
||||
# ``conversation_history_after_compression()`` needs the latest attempt's
|
||||
# outcome, while ``_last_compaction_in_place`` remains the run-level signal
|
||||
# read by gateway callers. ``None`` means this attempt aborted or made no
|
||||
# boundary, so the previous flush baseline remains authoritative.
|
||||
agent._last_compression_attempt_recorded = True
|
||||
agent._last_compression_attempt_in_place = None
|
||||
|
||||
_attempt_started_at = time.monotonic()
|
||||
_attempt_id = uuid.uuid4().hex
|
||||
_trigger_source = "manual" if force else "auto"
|
||||
|
|
@ -1030,14 +1166,26 @@ def compress_context(
|
|||
# the app server does not expose its native summary prompt, so there is no
|
||||
# truthful injection point for ``on_pre_compress()`` return text here.
|
||||
if getattr(agent, "api_mode", None) == "codex_app_server":
|
||||
return _compress_context_via_codex_app_server(
|
||||
agent,
|
||||
messages,
|
||||
system_message,
|
||||
approx_tokens=approx_tokens,
|
||||
task_id=task_id,
|
||||
force=force,
|
||||
)
|
||||
_codex_fence_entered = False
|
||||
if commit_fence is not None:
|
||||
_codex_fence_entered = commit_fence.begin_commit()
|
||||
if not _codex_fence_entered:
|
||||
existing_prompt = getattr(agent, "_cached_system_prompt", None)
|
||||
if not existing_prompt:
|
||||
existing_prompt = agent._build_system_prompt(system_message)
|
||||
return messages, existing_prompt
|
||||
try:
|
||||
return _compress_context_via_codex_app_server(
|
||||
agent,
|
||||
messages,
|
||||
system_message,
|
||||
approx_tokens=approx_tokens,
|
||||
task_id=task_id,
|
||||
force=force,
|
||||
)
|
||||
finally:
|
||||
if _codex_fence_entered:
|
||||
commit_fence.finish_commit()
|
||||
|
||||
# Every automatic entrypoint must honor compressor-owned cooldown and
|
||||
# breaker state. Gateway hygiene constructs a fresh AIAgent, so the
|
||||
|
|
@ -1089,7 +1237,20 @@ def compress_context(
|
|||
f"{approx_tokens:,}" if approx_tokens else "unknown", agent.model,
|
||||
focus_topic,
|
||||
)
|
||||
agent._emit_status(COMPACTION_STATUS)
|
||||
_compaction_status = COMPACTION_STATUS
|
||||
if not force:
|
||||
_compaction_status = automatic_compaction_status_message(
|
||||
agent.context_compressor,
|
||||
phase="compress",
|
||||
default_message=_compaction_status,
|
||||
approx_tokens=approx_tokens,
|
||||
message_count=_pre_msg_count,
|
||||
model=agent.model,
|
||||
focus_topic=focus_topic,
|
||||
)
|
||||
_compaction_status_emitted = bool(_compaction_status)
|
||||
if _compaction_status:
|
||||
agent._emit_status(_compaction_status)
|
||||
_compaction_done_emitted = False
|
||||
|
||||
def _complete_compaction_lifecycle() -> None:
|
||||
|
|
@ -1097,7 +1258,11 @@ def compress_context(
|
|||
if _compaction_done_emitted:
|
||||
return
|
||||
_compaction_done_emitted = True
|
||||
_emit_compaction_done(agent)
|
||||
# A suppressed start (quiet context engine) opened no visible
|
||||
# compaction phase — emit no terminal edge either. Failure warnings
|
||||
# go through agent._emit_warning and are never suppressed here.
|
||||
if _compaction_status_emitted:
|
||||
_emit_compaction_done(agent)
|
||||
|
||||
# ── Compression lock ────────────────────────────────────────────────
|
||||
# Atomic, state.db-backed lock per session_id. Without this, two
|
||||
|
|
@ -1134,6 +1299,12 @@ def compress_context(
|
|||
_try_acquire_lock = None
|
||||
_lock_lookup_error: Optional[Exception] = None
|
||||
_legacy_session_db_without_lock_api = False
|
||||
# Clear any stale lock-skip signal from a prior call so this call's
|
||||
# outcome alone determines what callers see. Without this an
|
||||
# auto-compress lock-skip followed by a successful manual /compress
|
||||
# would falsely report "Compression already in progress" and discard
|
||||
# the compression results.
|
||||
agent._compression_skipped_due_to_lock = None
|
||||
if _lock_db is not None:
|
||||
try:
|
||||
_legacy_session_db_without_lock_api = _lock_api_is_absent_on_session_db(
|
||||
|
|
@ -1221,6 +1392,11 @@ def compress_context(
|
|||
_lock_sid, existing,
|
||||
)
|
||||
_lock_holder = None # don't release a lock we don't own
|
||||
# Signal to callers that this no-op is due to a concurrent lock,
|
||||
# not a genuine "nothing to compress" or aux-model failure.
|
||||
# Manual /compress callers can surface a clear status message
|
||||
# instead of the misleading "No changes from compression" text.
|
||||
agent._compression_skipped_due_to_lock = existing or True
|
||||
# Surface to the user once — quiet for downstream auto-compress loops
|
||||
if getattr(agent, "_last_compression_lock_warning_sid", None) != _lock_sid:
|
||||
agent._last_compression_lock_warning_sid = _lock_sid
|
||||
|
|
@ -1394,6 +1570,7 @@ def compress_context(
|
|||
if _activity_heartbeat is not None:
|
||||
_activity_heartbeat.stop("context compression completed")
|
||||
|
||||
_commit_fence_entered = False
|
||||
try:
|
||||
# Capture boundary quality before session-rotation callbacks run. Built-in
|
||||
# and plugin lifecycle hooks may reset per-session compressor fields while
|
||||
|
|
@ -1481,6 +1658,28 @@ def compress_context(
|
|||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
if commit_fence is not None:
|
||||
_commit_fence_entered = commit_fence.begin_commit()
|
||||
if not _commit_fence_entered:
|
||||
logger.info(
|
||||
"Compression commit cancelled before session mutation "
|
||||
"(session=%s).",
|
||||
agent.session_id or "none",
|
||||
)
|
||||
agent._last_compaction_in_place = False
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="commit_fence_cancelled",
|
||||
)
|
||||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
summary_error = getattr(agent.context_compressor, "_last_summary_error", None)
|
||||
if summary_error:
|
||||
if getattr(agent, "_last_compression_summary_warning", None) != summary_error:
|
||||
|
|
@ -1509,11 +1708,54 @@ def compress_context(
|
|||
|
||||
todo_snapshot = agent._todo_store.format_for_injection()
|
||||
if todo_snapshot:
|
||||
compressed.append({
|
||||
"role": "user",
|
||||
"content": todo_snapshot,
|
||||
"_todo_snapshot_synthetic": True,
|
||||
})
|
||||
# Fold the snapshot into a trailing REAL user message so
|
||||
# compression never introduces a synthetic user/user pair. Any
|
||||
# snapshot merged at an earlier boundary is stripped first so
|
||||
# repeated compactions refresh rather than accumulate todo state
|
||||
# (#26981). Scaffolding tails (continuation marker, summary
|
||||
# handoff, a bare stale snapshot row) must never absorb the
|
||||
# snapshot: merging would upgrade them to "real user" evidence
|
||||
# and break zero-user provenance (#69292), so those keep the
|
||||
# flagged standalone append and the real-user preservation pass
|
||||
# continues to see todo scaffolding, not human intent.
|
||||
from agent.context_compressor import _append_text_to_content
|
||||
|
||||
merged = False
|
||||
_tail = (
|
||||
compressed[-1]
|
||||
if compressed and isinstance(compressed[-1], dict)
|
||||
else None
|
||||
)
|
||||
if _tail is not None and _tail.get("role") == "user":
|
||||
_stripped = _strip_stale_todo_snapshot(_tail.get("content"))
|
||||
_probe = {
|
||||
key: value for key, value in _tail.items() if key != "content"
|
||||
}
|
||||
_probe["content"] = _stripped
|
||||
if _is_real_user_message(_probe):
|
||||
_snapshot_text = (
|
||||
f"\n\n{todo_snapshot}"
|
||||
if isinstance(_stripped, str) and _stripped
|
||||
else todo_snapshot
|
||||
)
|
||||
_tail["content"] = _append_text_to_content(
|
||||
_stripped, _snapshot_text
|
||||
)
|
||||
merged = True
|
||||
elif _stripped != _tail.get("content") and not _message_text(
|
||||
{"role": "user", "content": _stripped}
|
||||
).strip():
|
||||
# The tail was nothing but an earlier snapshot row —
|
||||
# refresh it in place instead of stacking a duplicate.
|
||||
_tail["content"] = todo_snapshot
|
||||
_tail["_todo_snapshot_synthetic"] = True
|
||||
merged = True
|
||||
if not merged:
|
||||
compressed.append({
|
||||
"role": "user",
|
||||
"content": todo_snapshot,
|
||||
"_todo_snapshot_synthetic": True,
|
||||
})
|
||||
_ensure_compressed_has_user_turn(messages, compressed)
|
||||
|
||||
cached_system_prompt = agent._cached_system_prompt
|
||||
|
|
@ -1823,6 +2065,7 @@ def compress_context(
|
|||
# via a rotation-independent flag. The gateway uses this — NOT an
|
||||
# id-change diff — to re-baseline transcript handling (history_offset=0 +
|
||||
# rewrite on the same id) when compaction happened in place. See #38763.
|
||||
agent._last_compression_attempt_in_place = compacted_in_place
|
||||
agent._last_compaction_in_place = compacted_in_place
|
||||
|
||||
# Keep the post-compression rough estimate for diagnostics, but do not
|
||||
|
|
@ -1888,7 +2131,11 @@ def compress_context(
|
|||
# file dedup) ran. A concurrent path that wakes up the moment we
|
||||
# release will see the NEW session_id in state.db / SessionEntry and
|
||||
# acquire on that — no race against our just-finished work.
|
||||
_release_lock()
|
||||
try:
|
||||
_release_lock()
|
||||
finally:
|
||||
if _commit_fence_entered:
|
||||
commit_fence.finish_commit()
|
||||
|
||||
|
||||
def _compress_context_via_codex_app_server(
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from agent.conversation_compression import (
|
|||
PRE_API_COMPRESSION_STATUS_TEMPLATE,
|
||||
conversation_history_after_compression,
|
||||
)
|
||||
from agent.context_engine import automatic_compaction_status_message
|
||||
from agent.display import KawaiiSpinner
|
||||
from agent.error_classifier import FailoverReason, classify_api_error
|
||||
from agent.iteration_budget import IterationBudget
|
||||
|
|
@ -710,6 +711,13 @@ def run_conversation(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# The gateway caches agents across user turns. Compression state is
|
||||
# per-turn: carrying a prior in-place boundary forward would make a later
|
||||
# uncompressed result look like a compacted transcript to gateway writers.
|
||||
agent._last_compaction_in_place = False
|
||||
agent._last_compression_attempt_recorded = False
|
||||
agent._last_compression_attempt_in_place = None
|
||||
|
||||
# ── Per-turn setup (the prologue) ──
|
||||
# All once-per-turn setup — stdio guarding, retry-counter resets, user
|
||||
# message sanitization, todo/nudge hydration, system-prompt restore-or-
|
||||
|
|
@ -981,6 +989,13 @@ def run_conversation(
|
|||
# outgoing copy.
|
||||
_api_content = api_msg.pop("api_content", None)
|
||||
|
||||
# Display-only timeline metadata. Never a provider field — strip
|
||||
# from every outgoing copy so strict OpenAI-compatible backends
|
||||
# don't reject the request after a model switch or resumed typed
|
||||
# event row enters the live history.
|
||||
api_msg.pop("display_kind", None)
|
||||
api_msg.pop("display_metadata", None)
|
||||
|
||||
# Inject ephemeral context into the current turn's user message.
|
||||
# Sources: memory manager prefetch + plugin pre_llm_call hooks
|
||||
# with target="user_message" (the default). Both are
|
||||
|
|
@ -1300,6 +1315,15 @@ def run_conversation(
|
|||
if _moa_prepared_request is not None:
|
||||
pending_moa_prepared_request = _moa_prepared_request
|
||||
compression_attempts += 1
|
||||
# Compression is actually running (block cleared / was never
|
||||
# blocked) — reset the blocked-overflow warning dedup so a future
|
||||
# blocked-over-threshold turn can warn again. Mirrors the
|
||||
# turn-context preflight reset (silent-overflow fix #62625).
|
||||
# getattr guard: test doubles built via object.__new__ lack the
|
||||
# method (gateway test-double pitfall) — treat absence as no-op.
|
||||
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
|
||||
if callable(_clear_warn):
|
||||
_clear_warn()
|
||||
logger.info(
|
||||
"Pre-API compression: ~%s request tokens >= %s threshold "
|
||||
"(context=%s, attempt=%s/%s)",
|
||||
|
|
@ -1310,11 +1334,25 @@ def run_conversation(
|
|||
compression_attempts,
|
||||
max_compression_attempts,
|
||||
)
|
||||
agent._emit_status(
|
||||
PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
|
||||
_pre_api_status = automatic_compaction_status_message(
|
||||
_compressor,
|
||||
phase="pre_api",
|
||||
default_message=PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
|
||||
tokens=request_pressure_tokens
|
||||
)
|
||||
),
|
||||
approx_tokens=request_pressure_tokens,
|
||||
threshold_tokens=int(
|
||||
getattr(_compressor, "threshold_tokens", 0) or 0
|
||||
),
|
||||
context_length=int(
|
||||
getattr(_compressor, "context_length", 0) or 0
|
||||
),
|
||||
model=agent.model,
|
||||
attempt=compression_attempts,
|
||||
max_attempts=max_compression_attempts,
|
||||
)
|
||||
if _pre_api_status:
|
||||
agent._emit_status(_pre_api_status)
|
||||
_last_preflight_pressure = request_pressure_tokens
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages,
|
||||
|
|
@ -1340,12 +1378,38 @@ def run_conversation(
|
|||
# and preflight compaction sites; see
|
||||
# conversation_history_after_compression().
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
api_call_count -= 1
|
||||
agent._api_call_count = api_call_count
|
||||
agent.iteration_budget.refund()
|
||||
continue
|
||||
elif (
|
||||
agent.compression_enabled
|
||||
and len(messages) > 1
|
||||
and compression_attempts < max_compression_attempts
|
||||
and not _defer_preflight(request_pressure_tokens)
|
||||
and _compression_cooldown
|
||||
):
|
||||
# Blocked by the summary-LLM cooldown. Surface a deduped warning
|
||||
# (only when actually over threshold — should_compress_info
|
||||
# returns a None reason below threshold) so the user isn't left
|
||||
# with a silently growing context. Mirrors the turn-context
|
||||
# preflight and the loop-compaction guards (silent-overflow fix
|
||||
# #62625).
|
||||
_block_reason = None
|
||||
try:
|
||||
_block_reason = _compressor.should_compress_info(
|
||||
request_pressure_tokens
|
||||
)[1]
|
||||
except Exception:
|
||||
_block_reason = None
|
||||
if _block_reason:
|
||||
agent._warn_context_overflow_blocked(
|
||||
_block_reason,
|
||||
request_pressure_tokens,
|
||||
int(getattr(_compressor, "threshold_tokens", 0) or 0),
|
||||
)
|
||||
|
||||
# Thinking spinner for quiet mode (animated during API call)
|
||||
thinking_spinner = None
|
||||
|
|
@ -3555,7 +3619,7 @@ def run_conversation(
|
|||
task_id=effective_task_id,
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
if len(messages) < original_len or old_ctx > _reduced_ctx:
|
||||
agent._buffer_status(
|
||||
|
|
@ -3810,7 +3874,7 @@ def run_conversation(
|
|||
task_id=effective_task_id,
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
|
||||
# Re-estimate tokens after compression. Same-message-count
|
||||
|
|
@ -4051,7 +4115,7 @@ def run_conversation(
|
|||
task_id=effective_task_id,
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
|
||||
# Re-estimate tokens after compression. Same-message-count
|
||||
|
|
@ -5442,6 +5506,15 @@ def run_conversation(
|
|||
and _compressor.should_compress(_real_tokens)
|
||||
):
|
||||
compression_attempts += 1
|
||||
# Compression is actually running (block cleared / was
|
||||
# never blocked) — reset the blocked-overflow warning
|
||||
# dedup so a future blocked-over-threshold turn can warn
|
||||
# again (silent-overflow fix #62625).
|
||||
# getattr guard: test doubles built via object.__new__ lack the
|
||||
# method (gateway test-double pitfall) — treat absence as no-op.
|
||||
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
|
||||
if callable(_clear_warn):
|
||||
_clear_warn()
|
||||
agent._safe_print(" ⟳ compacting context…")
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message,
|
||||
|
|
@ -5449,8 +5522,27 @@ def run_conversation(
|
|||
task_id=effective_task_id,
|
||||
)
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
elif agent.compression_enabled:
|
||||
# Over threshold but compression is blocked (summary-LLM
|
||||
# cooldown or anti-thrashing). Surface a deduped warning so
|
||||
# the user isn't left with a silently growing context that
|
||||
# eventually hits the hard provider limit. Mirrors the
|
||||
# turn-context preflight guard (silent-overflow fix #62625).
|
||||
_block_reason = None
|
||||
_info = getattr(_compressor, "should_compress_info", None)
|
||||
if _info is not None:
|
||||
try:
|
||||
_block_reason = _info(_real_tokens)[1]
|
||||
except Exception:
|
||||
_block_reason = None
|
||||
if _block_reason:
|
||||
agent._warn_context_overflow_blocked(
|
||||
_block_reason,
|
||||
_real_tokens,
|
||||
int(getattr(_compressor, "threshold_tokens", 0) or 0),
|
||||
)
|
||||
|
||||
# Save session log incrementally (so progress is visible even if interrupted)
|
||||
agent._session_messages = messages
|
||||
|
|
|
|||
|
|
@ -503,6 +503,10 @@ class CopilotACPClient:
|
|||
|
||||
def _run_prompt(self, prompt_text: str, *, timeout_seconds: float) -> tuple[str, str]:
|
||||
try:
|
||||
# Hide the console the CLI child would otherwise flash on Windows
|
||||
# (#56747). Hide-only — stdio pipes stay intact for the ACP wire.
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[self._acp_command] + self._acp_args,
|
||||
stdin=subprocess.PIPE,
|
||||
|
|
@ -512,6 +516,7 @@ class CopilotACPClient:
|
|||
bufsize=1,
|
||||
cwd=self._acp_cwd,
|
||||
env=_build_subprocess_env(),
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
|
|
|
|||
|
|
@ -596,20 +596,32 @@ class CredentialPool:
|
|||
self._last_no_entries_log_at: Optional[float] = None
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
return bool(self._entries)
|
||||
with self._lock:
|
||||
return bool(self._entries)
|
||||
|
||||
def has_available(self) -> bool:
|
||||
"""True if at least one entry is not currently in exhaustion cooldown."""
|
||||
return bool(self._available_entries())
|
||||
# ``_available_entries`` is not read-only: it prunes aged-out DEAD
|
||||
# manual entries (rebinding ``self._entries``) and persists. It must
|
||||
# run under ``self._lock`` like every other caller (``select`` etc.),
|
||||
# otherwise a status probe here can race a concurrent ``select`` /
|
||||
# rotation and tear ``self._entries`` or double-write auth.json.
|
||||
with self._lock:
|
||||
return bool(self._available_entries())
|
||||
|
||||
def entries(self) -> List[PooledCredential]:
|
||||
return list(self._entries)
|
||||
with self._lock:
|
||||
return list(self._entries)
|
||||
|
||||
def current(self) -> Optional[PooledCredential]:
|
||||
def _current_unlocked(self) -> Optional[PooledCredential]:
|
||||
if not self._current_id:
|
||||
return None
|
||||
return next((entry for entry in self._entries if entry.id == self._current_id), None)
|
||||
|
||||
def current(self) -> Optional[PooledCredential]:
|
||||
with self._lock:
|
||||
return self._current_unlocked()
|
||||
|
||||
def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
|
||||
"""Swap an entry in-place by id, preserving sort order."""
|
||||
for idx, entry in enumerate(self._entries):
|
||||
|
|
@ -652,6 +664,8 @@ class CredentialPool:
|
|||
entry: PooledCredential,
|
||||
status_code: Optional[int],
|
||||
error_context: Optional[Dict[str, Any]] = None,
|
||||
*,
|
||||
persist: bool = True,
|
||||
) -> PooledCredential:
|
||||
normalized_error = _normalize_error_context(error_context)
|
||||
# Permanent OAuth failures (token_invalidated, token_revoked, etc.)
|
||||
|
|
@ -675,7 +689,8 @@ class CredentialPool:
|
|||
last_error_reset_at=normalized_error.get("reset_at"),
|
||||
)
|
||||
self._replace_entry(entry, updated)
|
||||
self._persist()
|
||||
if persist:
|
||||
self._persist()
|
||||
return updated
|
||||
|
||||
def _sync_anthropic_entry_from_credentials_file(self, entry: PooledCredential) -> PooledCredential:
|
||||
|
|
@ -1726,18 +1741,21 @@ class CredentialPool:
|
|||
self._entries = [replace(candidate, priority=idx) for idx, candidate in enumerate(rotated)]
|
||||
self._persist()
|
||||
self._current_id = entry.id
|
||||
return self.current() or entry
|
||||
return self._current_unlocked() or entry
|
||||
|
||||
entry = available[0]
|
||||
self._current_id = entry.id
|
||||
return entry
|
||||
|
||||
def peek(self) -> Optional[PooledCredential]:
|
||||
current = self.current()
|
||||
if current is not None:
|
||||
return current
|
||||
available = self._available_entries()
|
||||
return available[0] if available else None
|
||||
# Single lock acquisition for the whole read; call the unlocked
|
||||
# helpers so we don't re-enter the non-reentrant ``self._lock``.
|
||||
with self._lock:
|
||||
current = self._current_unlocked()
|
||||
if current is not None:
|
||||
return current
|
||||
available = self._available_entries()
|
||||
return available[0] if available else None
|
||||
|
||||
def mark_exhausted_and_rotate(
|
||||
self,
|
||||
|
|
@ -1757,12 +1775,49 @@ class CredentialPool:
|
|||
(e for e in self._entries if e.runtime_api_key == api_key_hint),
|
||||
None,
|
||||
)
|
||||
if entry is None:
|
||||
# The failed key is identifiable but matches no entry
|
||||
# (rotated away, or a wrapper whose runtime key differs).
|
||||
# Falling through to current()/_select_unlocked() would
|
||||
# mark an INNOCENT healthy key exhausted for the full
|
||||
# cooldown TTL. Don't guess — just hand back a fresh
|
||||
# selection so the caller can retry.
|
||||
logger.info(
|
||||
"credential pool: failed key hint matched no %s entry; "
|
||||
"rotating without marking any credential exhausted",
|
||||
self.provider,
|
||||
)
|
||||
self._current_id = None
|
||||
return self._select_unlocked()
|
||||
if entry is None:
|
||||
entry = self.current() or self._select_unlocked()
|
||||
entry = self._current_unlocked() or self._select_unlocked()
|
||||
if entry is None:
|
||||
return None
|
||||
_label = entry.label or entry.id[:8]
|
||||
self._mark_exhausted(entry, status_code, error_context)
|
||||
# A 402/429/401 is an API-key–level failure: the account is out of
|
||||
# balance, rate-limited, or its key is rejected. The same key can
|
||||
# back more than one pool entry (e.g. an explicit pool entry plus a
|
||||
# ``model_config`` entry auto-seeded from ``model.api_key`` — both
|
||||
# carry the identical ``runtime_api_key``). Marking only the first
|
||||
# match leaves the sibling entries OK, so ``_select_unlocked()``
|
||||
# keeps handing back the same depleted key and rotation never
|
||||
# converges — the caller ``continue``s forever until the client
|
||||
# disconnects (a ~2.5min hang with no error surfaced to the user).
|
||||
# Mark every entry sharing the failed key so the pool can reach the
|
||||
# "no available entries" state and let the error propagate.
|
||||
if api_key_hint:
|
||||
siblings_marked = False
|
||||
for sibling in self._entries:
|
||||
if sibling.id == entry.id:
|
||||
continue
|
||||
if sibling.runtime_api_key == api_key_hint:
|
||||
self._mark_exhausted(
|
||||
sibling, status_code, error_context, persist=False
|
||||
)
|
||||
siblings_marked = True
|
||||
if siblings_marked:
|
||||
self._persist()
|
||||
# Re-read the updated entry to log the correct terminal state.
|
||||
updated_entry = next(
|
||||
(e for e in self._entries if e.id == entry.id), entry,
|
||||
|
|
@ -1852,14 +1907,14 @@ class CredentialPool:
|
|||
None,
|
||||
)
|
||||
else:
|
||||
entry = self.current() or self._select_unlocked(refresh=False)
|
||||
entry = self._current_unlocked() or self._select_unlocked(refresh=False)
|
||||
if entry is None:
|
||||
return None
|
||||
self._current_id = entry.id
|
||||
return self._try_refresh_current_unlocked()
|
||||
|
||||
def _try_refresh_current_unlocked(self) -> Optional[PooledCredential]:
|
||||
entry = self.current()
|
||||
entry = self._current_unlocked()
|
||||
if entry is None:
|
||||
return None
|
||||
refreshed = self._refresh_entry(entry, force=True)
|
||||
|
|
@ -1868,76 +1923,80 @@ class CredentialPool:
|
|||
return refreshed
|
||||
|
||||
def reset_statuses(self) -> int:
|
||||
count = 0
|
||||
new_entries = []
|
||||
for entry in self._entries:
|
||||
if entry.last_status or entry.last_status_at or entry.last_error_code:
|
||||
new_entries.append(
|
||||
replace(
|
||||
entry,
|
||||
last_status=None,
|
||||
last_status_at=None,
|
||||
last_error_code=None,
|
||||
last_error_reason=None,
|
||||
last_error_message=None,
|
||||
last_error_reset_at=None,
|
||||
with self._lock:
|
||||
count = 0
|
||||
new_entries = []
|
||||
for entry in self._entries:
|
||||
if entry.last_status or entry.last_status_at or entry.last_error_code:
|
||||
new_entries.append(
|
||||
replace(
|
||||
entry,
|
||||
last_status=None,
|
||||
last_status_at=None,
|
||||
last_error_code=None,
|
||||
last_error_reason=None,
|
||||
last_error_message=None,
|
||||
last_error_reset_at=None,
|
||||
)
|
||||
)
|
||||
)
|
||||
count += 1
|
||||
else:
|
||||
new_entries.append(entry)
|
||||
if count:
|
||||
self._entries = new_entries
|
||||
self._persist()
|
||||
return count
|
||||
count += 1
|
||||
else:
|
||||
new_entries.append(entry)
|
||||
if count:
|
||||
self._entries = new_entries
|
||||
self._persist()
|
||||
return count
|
||||
|
||||
def remove_index(self, index: int) -> Optional[PooledCredential]:
|
||||
if index < 1 or index > len(self._entries):
|
||||
return None
|
||||
removed = self._entries.pop(index - 1)
|
||||
self._entries = [
|
||||
replace(entry, priority=new_priority)
|
||||
for new_priority, entry in enumerate(self._entries)
|
||||
]
|
||||
write_credential_pool(
|
||||
self.provider,
|
||||
[entry.to_dict() for entry in self._entries],
|
||||
removed_ids=[removed.id],
|
||||
)
|
||||
if self._current_id == removed.id:
|
||||
self._current_id = None
|
||||
return removed
|
||||
with self._lock:
|
||||
if index < 1 or index > len(self._entries):
|
||||
return None
|
||||
removed = self._entries.pop(index - 1)
|
||||
self._entries = [
|
||||
replace(entry, priority=new_priority)
|
||||
for new_priority, entry in enumerate(self._entries)
|
||||
]
|
||||
write_credential_pool(
|
||||
self.provider,
|
||||
[entry.to_dict() for entry in self._entries],
|
||||
removed_ids=[removed.id],
|
||||
)
|
||||
if self._current_id == removed.id:
|
||||
self._current_id = None
|
||||
return removed
|
||||
|
||||
def resolve_target(self, target: Any) -> Tuple[Optional[int], Optional[PooledCredential], Optional[str]]:
|
||||
raw = str(target or "").strip()
|
||||
if not raw:
|
||||
return None, None, "No credential target provided."
|
||||
|
||||
for idx, entry in enumerate(self._entries, start=1):
|
||||
if entry.id == raw:
|
||||
return idx, entry, None
|
||||
with self._lock:
|
||||
for idx, entry in enumerate(self._entries, start=1):
|
||||
if entry.id == raw:
|
||||
return idx, entry, None
|
||||
|
||||
label_matches = [
|
||||
(idx, entry)
|
||||
for idx, entry in enumerate(self._entries, start=1)
|
||||
if entry.label.strip().lower() == raw.lower()
|
||||
]
|
||||
if len(label_matches) == 1:
|
||||
return label_matches[0][0], label_matches[0][1], None
|
||||
if len(label_matches) > 1:
|
||||
return None, None, f'Ambiguous credential label "{raw}". Use the numeric index or entry id instead.'
|
||||
if raw.isdigit():
|
||||
index = int(raw)
|
||||
if 1 <= index <= len(self._entries):
|
||||
return index, self._entries[index - 1], None
|
||||
return None, None, f"No credential #{index}."
|
||||
return None, None, f'No credential matching "{raw}".'
|
||||
label_matches = [
|
||||
(idx, entry)
|
||||
for idx, entry in enumerate(self._entries, start=1)
|
||||
if entry.label.strip().lower() == raw.lower()
|
||||
]
|
||||
if len(label_matches) == 1:
|
||||
return label_matches[0][0], label_matches[0][1], None
|
||||
if len(label_matches) > 1:
|
||||
return None, None, f'Ambiguous credential label "{raw}". Use the numeric index or entry id instead.'
|
||||
if raw.isdigit():
|
||||
index = int(raw)
|
||||
if 1 <= index <= len(self._entries):
|
||||
return index, self._entries[index - 1], None
|
||||
return None, None, f"No credential #{index}."
|
||||
return None, None, f'No credential matching "{raw}".'
|
||||
|
||||
def add_entry(self, entry: PooledCredential) -> PooledCredential:
|
||||
entry = replace(entry, priority=_next_priority(self._entries))
|
||||
self._entries.append(entry)
|
||||
self._persist()
|
||||
return entry
|
||||
with self._lock:
|
||||
entry = replace(entry, priority=_next_priority(self._entries))
|
||||
self._entries.append(entry)
|
||||
self._persist()
|
||||
return entry
|
||||
|
||||
|
||||
def _upsert_entry(entries: List[PooledCredential], provider: str, source: str, payload: Dict[str, Any]) -> bool:
|
||||
|
|
|
|||
85
agent/delegation_context.py
Normal file
85
agent/delegation_context.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""Context-local state for delegate_task child execution.
|
||||
|
||||
The parent Hermes process may itself be a Kanban dispatcher worker with
|
||||
HERMES_KANBAN_* variables in process env. delegate_task children run inside the
|
||||
same Python process, but they are not dispatcher-owned Kanban workers. This
|
||||
module lets code paths that resolve tool schemas or spawn subprocesses fail
|
||||
closed for delegated children without mutating global os.environ for the parent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Iterator, Mapping, MutableMapping
|
||||
|
||||
_DELEGATED_CHILD_CONTEXT: ContextVar[bool] = ContextVar(
|
||||
"hermes_delegated_child_context",
|
||||
default=False,
|
||||
)
|
||||
|
||||
DELEGATED_CHILD_ENV_MARKER = "HERMES_DELEGATED_CHILD_CONTEXT"
|
||||
|
||||
KANBAN_ENV_KEYS: tuple[str, ...] = (
|
||||
"HERMES_KANBAN_TASK",
|
||||
"HERMES_KANBAN_RUN_ID",
|
||||
"HERMES_KANBAN_WORKSPACE",
|
||||
"HERMES_KANBAN_WORKSPACES_ROOT",
|
||||
"HERMES_KANBAN_CLAIM_LOCK",
|
||||
"HERMES_KANBAN_BOARD",
|
||||
"HERMES_KANBAN_DB",
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def delegated_child_context() -> Iterator[None]:
|
||||
"""Mark the current execution context as a delegate_task child."""
|
||||
token = _DELEGATED_CHILD_CONTEXT.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_DELEGATED_CHILD_CONTEXT.reset(token)
|
||||
|
||||
|
||||
def is_delegated_child_context() -> bool:
|
||||
"""Return True while code is running for a delegate_task child."""
|
||||
return bool(_DELEGATED_CHILD_CONTEXT.get())
|
||||
|
||||
|
||||
def is_delegated_child_process_context() -> bool:
|
||||
"""Return True in this process or a subprocess spawned by a child."""
|
||||
import os
|
||||
|
||||
return bool(_DELEGATED_CHILD_CONTEXT.get()) or bool(
|
||||
os.environ.get(DELEGATED_CHILD_ENV_MARKER)
|
||||
)
|
||||
|
||||
|
||||
def scrub_kanban_env(env: Mapping[str, str] | MutableMapping[str, str]) -> dict[str, str]:
|
||||
"""Return *env* with dispatcher-only Kanban variables removed."""
|
||||
cleaned = dict(env)
|
||||
for key in KANBAN_ENV_KEYS:
|
||||
cleaned.pop(key, None)
|
||||
cleaned[DELEGATED_CHILD_ENV_MARKER] = "1"
|
||||
return cleaned
|
||||
|
||||
|
||||
def delegated_child_subprocess_env(
|
||||
env: Mapping[str, str] | MutableMapping[str, str] | None = None,
|
||||
) -> dict[str, str] | None:
|
||||
"""Return an env override only when delegated-child lineage must cross fork.
|
||||
|
||||
Most subprocess call sites historically used ``env=None`` to inherit the
|
||||
process environment. In a ``delegate_task`` child, inheriting as-is leaks
|
||||
parent dispatcher ``HERMES_KANBAN_*`` vars while losing the ContextVar in
|
||||
the new process. This helper preserves normal ``env=None`` semantics for
|
||||
non-delegated calls, and only materializes a scrubbed env when the lineage
|
||||
marker must be propagated across a child-process boundary.
|
||||
"""
|
||||
if not is_delegated_child_process_context():
|
||||
return None if env is None else dict(env)
|
||||
|
||||
if env is None:
|
||||
import os
|
||||
|
||||
env = os.environ
|
||||
return scrub_kanban_env(env)
|
||||
|
|
@ -413,6 +413,7 @@ _CONTENT_POLICY_BLOCKED_PATTERNS = [
|
|||
_AUTH_PATTERNS = [
|
||||
"invalid api key",
|
||||
"invalid_api_key",
|
||||
"gateway_auth_failed",
|
||||
"authentication",
|
||||
"unauthorized",
|
||||
"forbidden",
|
||||
|
|
|
|||
|
|
@ -194,6 +194,10 @@ def _supports_vision_override(
|
|||
and/or the user-declared name under ``model.provider``; all are
|
||||
tried. For ``custom:<name>`` syntax, the stripped ``<name>`` is also
|
||||
tried as a provider key.)
|
||||
2b. ``custom_providers`` (legacy list form) ``.models.<model>``
|
||||
|
||||
Under (2) and (2b), the per-model capability key may be written as
|
||||
either ``supports_vision`` or the shorter ``vision`` alias; both work.
|
||||
|
||||
Returns None when no override is set, so the caller falls through to
|
||||
models.dev. Returns False explicitly only when the user wrote a
|
||||
|
|
@ -234,7 +238,9 @@ def _supports_vision_override(
|
|||
models_cfg: Dict[str, Any] = models_raw if isinstance(models_raw, dict) else {}
|
||||
per_model_raw = models_cfg.get(model)
|
||||
per_model: Dict[str, Any] = per_model_raw if isinstance(per_model_raw, dict) else {}
|
||||
coerced = _coerce_capability_bool(per_model.get("supports_vision"))
|
||||
coerced = _coerce_capability_bool(
|
||||
per_model.get("supports_vision", per_model.get("vision"))
|
||||
)
|
||||
if coerced is not None:
|
||||
return coerced
|
||||
|
||||
|
|
@ -258,7 +264,9 @@ def _supports_vision_override(
|
|||
models_cfg = models_raw if isinstance(models_raw, dict) else {}
|
||||
per_model_raw = models_cfg.get(model)
|
||||
per_model = per_model_raw if isinstance(per_model_raw, dict) else {}
|
||||
coerced = _coerce_capability_bool(per_model.get("supports_vision"))
|
||||
coerced = _coerce_capability_bool(
|
||||
per_model.get("supports_vision", per_model.get("vision"))
|
||||
)
|
||||
if coerced is not None:
|
||||
return coerced
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,36 @@ from typing import Any, Sequence
|
|||
from agent.redact import redact_sensitive_text
|
||||
|
||||
|
||||
def describe_compression_lock_skip(lock_signal: Any) -> str:
|
||||
"""User-facing text for a manual /compress skipped by the compression lock.
|
||||
|
||||
``lock_signal`` is ``agent._compression_skipped_due_to_lock`` (or the
|
||||
``holder`` carried by the TUI's ``CompressionLockHeld``): a descriptive
|
||||
holder string when another compressor CONFIRMED holds the lock, or
|
||||
``True``/``None`` when acquisition failed without a confirmed holder
|
||||
(``hermes_state.try_acquire_compression_lock`` catches ``sqlite3.Error``
|
||||
internally and returns ``False``, so a failed acquire is NOT proof that
|
||||
another compression is running). The two cases must be worded
|
||||
differently: claiming "already in progress" on an unconfirmed failure
|
||||
misdirects the user when the real problem is a broken lock subsystem.
|
||||
"""
|
||||
holder = (
|
||||
lock_signal
|
||||
if isinstance(lock_signal, str) and lock_signal.strip()
|
||||
else None
|
||||
)
|
||||
if holder:
|
||||
return (
|
||||
f"⏳ Compression already in progress for this session "
|
||||
f"(holder: {holder}). Please wait for it to finish."
|
||||
)
|
||||
return (
|
||||
"⏳ Compression skipped: could not acquire this session's "
|
||||
"compression lock. Another compression may still be running, or "
|
||||
"the lock check failed — try again shortly."
|
||||
)
|
||||
|
||||
|
||||
def summarize_manual_compression(
|
||||
before_messages: Sequence[dict[str, Any]],
|
||||
after_messages: Sequence[dict[str, Any]],
|
||||
|
|
|
|||
|
|
@ -212,11 +212,30 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]:
|
|||
out["api_key"] = rt["api_key"]
|
||||
if rt.get("api_mode"):
|
||||
out["api_mode"] = rt["api_mode"]
|
||||
request_overrides = rt.get("request_overrides")
|
||||
if isinstance(request_overrides, dict):
|
||||
extra_body = request_overrides.get("extra_body")
|
||||
if isinstance(extra_body, dict) and extra_body:
|
||||
out["extra_body"] = dict(extra_body)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("MoA slot runtime resolution failed for %s: %s", _slot_label(slot), exc)
|
||||
return out
|
||||
|
||||
|
||||
def _merge_slot_extra_body(
|
||||
slot_extra_body: Any,
|
||||
caller_extra_body: Any,
|
||||
) -> Any:
|
||||
"""Merge slot defaults with a caller override for ``call_llm``."""
|
||||
if isinstance(slot_extra_body, dict) and slot_extra_body:
|
||||
if isinstance(caller_extra_body, dict):
|
||||
return {**slot_extra_body, **caller_extra_body}
|
||||
if caller_extra_body:
|
||||
return caller_extra_body
|
||||
return dict(slot_extra_body)
|
||||
return caller_extra_body
|
||||
|
||||
|
||||
def _maybe_apply_moa_cache_control(
|
||||
messages: list[dict[str, Any]],
|
||||
runtime: dict[str, Any],
|
||||
|
|
@ -976,18 +995,26 @@ class MoAChatCompletions:
|
|||
# actually governs the aggregator stream, not just call_llm's default.
|
||||
if api_kwargs.get("timeout") is not None:
|
||||
stream_kwargs["timeout"] = api_kwargs["timeout"]
|
||||
agg_runtime = _slot_runtime(aggregator)
|
||||
# _slot_runtime may carry the provider's request_overrides.extra_body;
|
||||
# pop it and merge with the caller's extra_body (caller wins) so the
|
||||
# explicit kwarg below never collides with **agg_runtime.
|
||||
agg_extra_body = _merge_slot_extra_body(
|
||||
agg_runtime.pop("extra_body", None),
|
||||
extra_body,
|
||||
)
|
||||
_agg_response = call_llm(
|
||||
task="moa_aggregator",
|
||||
messages=agg_messages,
|
||||
temperature=aggregator_temperature,
|
||||
max_tokens=max_tokens,
|
||||
tools=tools,
|
||||
extra_body=extra_body,
|
||||
extra_body=agg_extra_body,
|
||||
# Prepared requests must retain the acting aggregator's reasoning
|
||||
# policy exactly as the direct create() path does (#64187).
|
||||
reasoning_config=_aggregator_reasoning_config(aggregator),
|
||||
**stream_kwargs,
|
||||
**_slot_runtime(aggregator),
|
||||
**agg_runtime,
|
||||
)
|
||||
# Non-streaming path (quiet mode / eval / subagents): the aggregator
|
||||
# output is available inline, so capture it into the pending trace now.
|
||||
|
|
|
|||
|
|
@ -2201,11 +2201,18 @@ def get_model_context_length(
|
|||
# acting context, so they're ignored here.
|
||||
if (provider or "").strip().lower() == "moa":
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from hermes_cli.config import (
|
||||
get_compatible_custom_providers,
|
||||
load_config,
|
||||
)
|
||||
from hermes_cli.moa_config import resolve_moa_preset
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
||||
preset = resolve_moa_preset(load_config().get("moa") or {}, model)
|
||||
config = load_config()
|
||||
effective_custom_providers = custom_providers
|
||||
if effective_custom_providers is None:
|
||||
effective_custom_providers = get_compatible_custom_providers(config)
|
||||
preset = resolve_moa_preset(config.get("moa") or {}, model)
|
||||
agg = preset.get("aggregator") or {}
|
||||
agg_provider = str(agg.get("provider") or "").strip()
|
||||
agg_model = str(agg.get("model") or "").strip()
|
||||
|
|
@ -2215,7 +2222,8 @@ def get_model_context_length(
|
|||
agg_model,
|
||||
base_url=rt.get("base_url", "") or "",
|
||||
api_key=rt.get("api_key", "") or "",
|
||||
provider=agg_provider,
|
||||
provider=rt.get("provider") or agg_provider,
|
||||
custom_providers=effective_custom_providers,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("MoA aggregator context-length resolution failed", exc_info=True)
|
||||
|
|
|
|||
|
|
@ -884,7 +884,14 @@ PLATFORM_HINTS = {
|
|||
"You're responding through an API server. The rendering layer is unknown — "
|
||||
"assume plain text. No markdown formatting (no asterisks, bullets, headers, "
|
||||
"code fences). Treat this like a conversation, not a document. Keep responses "
|
||||
"brief and natural."
|
||||
"brief and natural. "
|
||||
"File/media delivery: images referenced as MEDIA:/absolute/path tags "
|
||||
"(.png/.jpg/.jpeg/.gif/.webp/.bmp, up to 5MB) are inlined as base64 data "
|
||||
"URLs in responses on the chat, completions, and responses endpoints. "
|
||||
"Non-image files are NOT intercepted anywhere, and the runs endpoint "
|
||||
"intercepts nothing — a MEDIA: tag there renders as literal text exposing "
|
||||
"a raw host filesystem path. For those cases, state the plain file path "
|
||||
"in your response text instead of a MEDIA: tag."
|
||||
),
|
||||
"webui": (
|
||||
"You are in the Hermes WebUI, a browser-based chat interface. "
|
||||
|
|
|
|||
|
|
@ -127,6 +127,10 @@ class CodexAppServerClient:
|
|||
# Codex emits tracing to stderr; default WARN keeps it quiet for users.
|
||||
spawn_env.setdefault("RUST_LOG", "warn")
|
||||
|
||||
# Hide the console the codex child would otherwise flash on Windows
|
||||
# (#56747). Hide-only — stdio pipes stay intact for the app-server wire.
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
|
||||
self._proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
|
|
@ -134,6 +138,7 @@ class CodexAppServerClient:
|
|||
stderr=subprocess.PIPE,
|
||||
bufsize=0,
|
||||
env=spawn_env,
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
self._next_id = 1
|
||||
self._pending: dict[int, _Pending] = {}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from agent.conversation_compression import (
|
|||
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
|
||||
conversation_history_after_compression,
|
||||
)
|
||||
from agent.context_engine import automatic_compaction_status_message
|
||||
from agent.iteration_budget import IterationBudget
|
||||
from agent.memory_manager import build_memory_context_block
|
||||
from agent.model_metadata import (
|
||||
|
|
@ -668,11 +669,18 @@ def build_turn_context(
|
|||
f"{_idle_floor:,}",
|
||||
agent.session_id or "none",
|
||||
)
|
||||
agent._emit_status(
|
||||
IDLE_COMPACTION_STATUS_TEMPLATE.format(
|
||||
_idle_status = automatic_compaction_status_message(
|
||||
_compressor,
|
||||
phase="idle",
|
||||
default_message=IDLE_COMPACTION_STATUS_TEMPLATE.format(
|
||||
idle_seconds=int(_idle_gap), tokens=_idle_tokens
|
||||
)
|
||||
),
|
||||
approx_tokens=_idle_tokens,
|
||||
idle_seconds=int(_idle_gap),
|
||||
model=agent.model,
|
||||
)
|
||||
if _idle_status:
|
||||
agent._emit_status(_idle_status)
|
||||
_idle_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message, approx_tokens=_idle_tokens,
|
||||
|
|
@ -686,7 +694,7 @@ def build_turn_context(
|
|||
# untouched.
|
||||
if messages is not _idle_input:
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
# Compaction rebuilt the list, so the index of this turn's
|
||||
# just-appended user message is stale — re-anchor it the
|
||||
|
|
@ -747,6 +755,8 @@ def build_turn_context(
|
|||
lambda: None,
|
||||
)()
|
||||
|
||||
_should_compress_now = False
|
||||
_compress_block_reason = None
|
||||
if _preflight_deferred:
|
||||
logger.info(
|
||||
"Skipping preflight compression: rough estimate ~%s >= %s, "
|
||||
|
|
@ -762,14 +772,42 @@ def build_turn_context(
|
|||
int(_compression_cooldown.get("remaining_seconds", 0.0)),
|
||||
agent.session_id or "none",
|
||||
)
|
||||
if _preflight_tokens >= _compressor.threshold_tokens:
|
||||
# Context is over threshold but compression is blocked by the
|
||||
# summary-LLM cooldown — surface a warning (see block below).
|
||||
_cooldown_secs = _compression_cooldown.get("remaining_seconds", 0.0)
|
||||
_compress_block_reason = f"cooldown:{_cooldown_secs:.0f}"
|
||||
elif _codex_native_auto:
|
||||
logger.info(
|
||||
"Skipping Hermes preflight compression for codex app-server "
|
||||
"(mode=%s); Hermes will not start thread compaction here.",
|
||||
getattr(agent, "codex_app_server_auto_compaction", "native"),
|
||||
)
|
||||
elif _compressor.should_compress(_preflight_tokens):
|
||||
else:
|
||||
_should_compress_now = _compressor.should_compress(_preflight_tokens)
|
||||
if not _should_compress_now:
|
||||
# Context is over threshold but compression is blocked
|
||||
# (summary-LLM cooldown or anti-thrashing). Ask should_compress_info
|
||||
# for the human-readable reason so we can surface a warning below.
|
||||
# getattr guard: minimal compressor doubles (SimpleNamespace in
|
||||
# the engine-preflight tests) and older plugin engines lack the
|
||||
# method — absence means no block reason, no warning.
|
||||
_info = getattr(_compressor, "should_compress_info", None)
|
||||
if callable(_info):
|
||||
try:
|
||||
_compress_block_reason = _info(_preflight_tokens)[1]
|
||||
except Exception:
|
||||
_compress_block_reason = None
|
||||
if _should_compress_now:
|
||||
_preflight_compressed = True
|
||||
# Compression is actually running (block cleared / was never
|
||||
# blocked) — reset the dedup so a future blocked-over-threshold
|
||||
# turn can warn again. Real session boundary.
|
||||
# getattr guard: test doubles built via object.__new__ lack the
|
||||
# method (gateway test-double pitfall) — treat absence as no-op.
|
||||
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
|
||||
if callable(_clear_warn):
|
||||
_clear_warn()
|
||||
logger.info(
|
||||
"Preflight compression: ~%s tokens >= %s threshold (model %s, ctx %s)",
|
||||
f"{_preflight_tokens:,}",
|
||||
|
|
@ -777,12 +815,20 @@ def build_turn_context(
|
|||
agent.model,
|
||||
f"{_compressor.context_length:,}",
|
||||
)
|
||||
agent._emit_status(
|
||||
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(
|
||||
_preflight_status = automatic_compaction_status_message(
|
||||
_compressor,
|
||||
phase="preflight",
|
||||
default_message=PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(
|
||||
tokens=_preflight_tokens,
|
||||
threshold=_compressor.threshold_tokens,
|
||||
)
|
||||
),
|
||||
approx_tokens=_preflight_tokens,
|
||||
threshold_tokens=_compressor.threshold_tokens,
|
||||
context_length=_compressor.context_length,
|
||||
model=agent.model,
|
||||
)
|
||||
if _preflight_status:
|
||||
agent._emit_status(_preflight_status)
|
||||
# Preflight passes honor the same configured per-turn cap
|
||||
# (compression.max_attempts) as the loop's compression sites;
|
||||
# default 3 preserves the prior hardcoded behavior.
|
||||
|
|
@ -811,7 +857,7 @@ def build_turn_context(
|
|||
_preflight_compression_blocked = True
|
||||
break # Cannot compress further: neither rows nor tokens moved
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
agent, messages, conversation_history
|
||||
)
|
||||
agent._empty_content_retries = 0
|
||||
agent._thinking_prefill_retries = 0
|
||||
|
|
@ -833,6 +879,100 @@ def build_turn_context(
|
|||
f"{_preflight_tokens:,}",
|
||||
)
|
||||
break
|
||||
elif _compress_block_reason:
|
||||
# Context is already over the compression threshold, but compression
|
||||
# is blocked (summary LLM cooldown or anti-thrashing). Without a
|
||||
# signal the session keeps growing until the model silently stops
|
||||
# answering — the conversation hits the hard provider token limit
|
||||
# with no explanation. Surface a deduped warning so the user can
|
||||
# take action (/new or /compress) instead of hitting a silent hang.
|
||||
agent._warn_context_overflow_blocked(
|
||||
_compress_block_reason,
|
||||
_preflight_tokens,
|
||||
_compressor.threshold_tokens,
|
||||
)
|
||||
else:
|
||||
# Sub-threshold and unblocked — allow the overflow warning to fire
|
||||
# again next time the context is over threshold but blocked.
|
||||
# getattr guard: test doubles built via object.__new__ lack the
|
||||
# method (gateway test-double pitfall) — treat absence as no-op.
|
||||
_clear_warn = getattr(agent, "_clear_context_overflow_warn", None)
|
||||
if callable(_clear_warn):
|
||||
_clear_warn()
|
||||
# Engine maintenance only when NO skip-branch fired: a failure
|
||||
# cooldown, deferred estimate, or codex-native route must keep
|
||||
# the engine hook un-consulted (#20316 contract — the cooldown
|
||||
# exists precisely because compression recently failed).
|
||||
if _compression_cooldown or _preflight_deferred or _codex_native_auto:
|
||||
_engine_preflight = None
|
||||
else:
|
||||
_engine_preflight = getattr(
|
||||
_compressor, "should_compress_preflight", None
|
||||
)
|
||||
# ── Engine-driven sub-threshold preflight maintenance (#20316) ──
|
||||
# None of the threshold-path branches fired (not deferred, no
|
||||
# failure cooldown, not codex-native, and should_compress() said
|
||||
# the request is under pressure). Context engines that override
|
||||
# ``should_compress_preflight()`` (e.g. LCM-style incremental
|
||||
# leaf-chunk compaction) can still request deferred maintenance
|
||||
# below the token threshold. The default
|
||||
# ``ContextEngine.should_compress_preflight()`` returns False, so
|
||||
# the built-in ``ContextCompressor`` path is byte-identical.
|
||||
#
|
||||
# Attempt-cap integration: the engine gets exactly ONE
|
||||
# ``compress()`` pass per turn. It is mutually exclusive with the
|
||||
# threshold multi-pass loop above (if/elif), so turn-start
|
||||
# preflight passes stay bounded by the resolved
|
||||
# ``compression.max_attempts`` cap (floor 1) in every case.
|
||||
#
|
||||
# No-op-blocking integration: a sub-threshold engine pass that
|
||||
# no-ops says nothing about over-threshold compressibility, so it
|
||||
# must neither set nor clear ``_preflight_compression_blocked``
|
||||
# (#64382) — and being in the ``else`` arm it can never run after
|
||||
# the threshold loop has proven a retry ineffective.
|
||||
# (resolved above, gated on no skip-branch having fired)
|
||||
_wants_engine_preflight = False
|
||||
if callable(_engine_preflight):
|
||||
try:
|
||||
_wants_engine_preflight = bool(_engine_preflight(messages))
|
||||
except Exception as _preflight_exc:
|
||||
# A buggy engine must never break an otherwise-healthy
|
||||
# turn: swallow at debug level and skip maintenance.
|
||||
logger.debug(
|
||||
"should_compress_preflight raised %s; skipping "
|
||||
"engine-driven preflight maintenance",
|
||||
_preflight_exc,
|
||||
)
|
||||
_wants_engine_preflight = False
|
||||
if _wants_engine_preflight:
|
||||
logger.info(
|
||||
"Engine-driven preflight maintenance: %s requested "
|
||||
"compress() at ~%s tokens (below %s threshold)",
|
||||
getattr(_compressor, "name", type(_compressor).__name__),
|
||||
f"{_preflight_tokens:,}",
|
||||
f"{getattr(_compressor, 'threshold_tokens', 0):,}",
|
||||
)
|
||||
_engine_input = messages
|
||||
messages, active_system_prompt = agent._compress_context(
|
||||
messages, system_message, approx_tokens=_preflight_tokens,
|
||||
task_id=effective_task_id,
|
||||
)
|
||||
# ``_compress_context`` returns the INPUT list object on every
|
||||
# skip path (per-session lock held elsewhere, cooldown,
|
||||
# anti-thrash breaker, codex-native routing) and an engine may
|
||||
# legitimately no-op. Only re-baseline the flush history and
|
||||
# re-anchor the user row after a REAL compaction — a skip must
|
||||
# leave the turn's bookkeeping untouched.
|
||||
if messages is not _engine_input:
|
||||
_preflight_compressed = True
|
||||
conversation_history = conversation_history_after_compression(
|
||||
agent, messages
|
||||
)
|
||||
agent._empty_content_retries = 0
|
||||
agent._thinking_prefill_retries = 0
|
||||
agent._last_content_with_tools = None
|
||||
agent._last_content_tools_all_housekeeping = False
|
||||
agent._mute_post_response = False
|
||||
|
||||
if _preflight_compressed:
|
||||
# Compression rebuilt the list (tail messages are fresh compaction
|
||||
|
|
|
|||
147
apps/desktop/e2e/hidden-history-messages.spec.ts
Normal file
147
apps/desktop/e2e/hidden-history-messages.spec.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* E2E regression: desktop resume must hide agent-only transcript rows.
|
||||
*
|
||||
* Compaction handoffs are active user rows because the model needs them for
|
||||
* context continuity. They are not authored chat content, so the desktop
|
||||
* transcript must never display them after a real compressor-generated resume.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { expect, test } from './test'
|
||||
|
||||
import {
|
||||
type MockBackendFixture,
|
||||
buildAppEnv,
|
||||
createSandbox,
|
||||
launchDesktop,
|
||||
waitForAppReady,
|
||||
writeEnvFile,
|
||||
writeMockProviderConfig,
|
||||
} from './fixtures'
|
||||
import {
|
||||
MOCK_REPLY,
|
||||
startMockServer,
|
||||
VERIFICATION_STOP_TEXT,
|
||||
VERIFICATION_STOP_TRIGGER,
|
||||
} from './mock-server'
|
||||
import { RealSessionBuilder } from './real-session-builder'
|
||||
|
||||
const SESSION_TITLE = 'E2E Hidden History Messages'
|
||||
const VISIBLE_USER_TEXT = 'E2E_VISIBLE_USER_HISTORY'
|
||||
const VISIBLE_POST_COMPACTION_TEXT = 'E2E_VISIBLE_POST_COMPACTION_HISTORY'
|
||||
const COMPACTION_TRIGGER_PADDING = ' force real context compression'.repeat(600)
|
||||
|
||||
async function setupSeededMockBackend(): Promise<MockBackendFixture> {
|
||||
const mock = await startMockServer()
|
||||
const sandbox = createSandbox('hidden-history')
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
fs.appendFileSync(
|
||||
path.join(sandbox.hermesHome, 'config.yaml'),
|
||||
'\ncompression:\n threshold_tokens: 1\n',
|
||||
'utf8',
|
||||
)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
|
||||
try {
|
||||
await builder.createSession({
|
||||
title: SESSION_TITLE,
|
||||
turns: [
|
||||
`${VISIBLE_USER_TEXT}${COMPACTION_TRIGGER_PADDING}`,
|
||||
VISIBLE_POST_COMPACTION_TEXT,
|
||||
],
|
||||
})
|
||||
} finally {
|
||||
await builder.close()
|
||||
}
|
||||
|
||||
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
mock,
|
||||
mockUrl: mock.url,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
test('resume hides real context-compaction handoffs', async ({}, testInfo) => {
|
||||
const fixture = await setupSeededMockBackend()
|
||||
|
||||
try {
|
||||
const { page } = fixture
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
|
||||
const sessionRow = page
|
||||
.locator('[data-slot="sidebar"] button')
|
||||
.filter({ hasText: SESSION_TITLE })
|
||||
.first()
|
||||
await sessionRow.click()
|
||||
|
||||
const transcript = page.locator('[data-slot="aui_thread-viewport"]')
|
||||
await expect(transcript).toContainText(VISIBLE_USER_TEXT)
|
||||
await expect(transcript).toContainText(VISIBLE_POST_COMPACTION_TEXT)
|
||||
await expect(transcript).toContainText(MOCK_REPLY)
|
||||
await expect(transcript).not.toContainText('[CONTEXT COMPACTION — REFERENCE ONLY]')
|
||||
await page.screenshot({ path: testInfo.outputPath('hidden-history-resume.png') })
|
||||
} finally {
|
||||
await fixture.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
test('live verify-on-stop continuations stay out of the transcript', async ({}, testInfo) => {
|
||||
const sandbox = createSandbox('live-verification-nudge')
|
||||
const projectRoot = path.join(sandbox.root, 'project')
|
||||
const changedFile = path.join(projectRoot, 'e2e-verification-target.py')
|
||||
fs.mkdirSync(projectRoot)
|
||||
fs.writeFileSync(
|
||||
path.join(projectRoot, 'pyproject.toml'),
|
||||
'[project]\nname = "e2e-verification-project"\nversion = "0.0.0"\n',
|
||||
'utf8',
|
||||
)
|
||||
|
||||
const mock = await startMockServer({ verificationWritePath: changedFile })
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
fs.appendFileSync(path.join(sandbox.hermesHome, 'config.yaml'), '\nagent:\n verify_on_stop: true\n', 'utf8')
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
|
||||
const fixture: MockBackendFixture = {
|
||||
app,
|
||||
page,
|
||||
mock,
|
||||
mockUrl: mock.url,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
|
||||
try {
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.click()
|
||||
await composer.type(VERIFICATION_STOP_TRIGGER)
|
||||
await page.keyboard.press('Enter')
|
||||
|
||||
const transcript = page.locator('[data-slot="aui_thread-viewport"]')
|
||||
await expect(transcript).toContainText(VERIFICATION_STOP_TEXT, { timeout: 60_000 })
|
||||
await expect.poll(
|
||||
() => mock.receivedPrompts.some(prompt => prompt.includes('[System: You edited code in this turn')),
|
||||
{ timeout: 30_000 },
|
||||
).toBe(true)
|
||||
expect(fs.existsSync(changedFile), 'The scripted write_file call should edit only the sandbox project').toBe(true)
|
||||
await expect(transcript).not.toContainText('[System: You edited code in this turn')
|
||||
await page.screenshot({ path: testInfo.outputPath('live-verification-nudge.png') })
|
||||
} finally {
|
||||
await fixture.cleanup()
|
||||
}
|
||||
})
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import { spawnSync } from 'node:child_process'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { type TestInfo } from '@playwright/test'
|
||||
|
|
@ -15,13 +14,16 @@ import {
|
|||
writeMockProviderConfig,
|
||||
} from './fixtures'
|
||||
import { MOCK_REPLY, startMockServer, type MockServer, type MockServerOptions } from './mock-server'
|
||||
import { RealSessionBuilder } from './real-session-builder'
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
const SEED_SCRIPT = path.resolve(import.meta.dirname, 'scripts', 'seed_large_session.py')
|
||||
const SESSION_TITLE = 'E2E large persisted session'
|
||||
const EXPECTED_TEXT = 'E2E persisted user message 52'
|
||||
const BACKGROUND_PROMPT = 'E2E background inference must remain attached across resume'
|
||||
const HISTORY_TURNS = Array.from(
|
||||
{ length: 27 },
|
||||
(_, index) => `E2E persisted user message ${index * 2}: audit the compatibility matrix`,
|
||||
)
|
||||
|
||||
interface SeededFixture {
|
||||
app: ElectronApplication
|
||||
|
|
@ -43,13 +45,11 @@ async function setupSeededDesktop(mockServer?: MockServerOptions): Promise<Seede
|
|||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
const seeded = spawnSync('python3', [SEED_SCRIPT, path.join(sandbox.hermesHome, 'state.db')], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, PYTHONPATH: REPO_ROOT },
|
||||
})
|
||||
if (seeded.status !== 0) {
|
||||
throw new Error(`large-session seed failed:\n${seeded.stdout}\n${seeded.stderr}`)
|
||||
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
|
||||
try {
|
||||
await builder.createSession({ title: SESSION_TITLE, turns: HISTORY_TURNS })
|
||||
} finally {
|
||||
await builder.close()
|
||||
}
|
||||
|
||||
const { app, page } = await launchDesktop(buildAppEnv(sandbox))
|
||||
|
|
@ -210,6 +210,7 @@ test.describe('large session resume', () => {
|
|||
await waitForAppReady(fixture, 120_000)
|
||||
|
||||
await openSeededSession(fixture.page)
|
||||
const initialMockReplyCount = await textNodeOccurrences(fixture.page, MOCK_REPLY)
|
||||
await submitPrompt(fixture.page, BACKGROUND_PROMPT)
|
||||
await fixture.mock.waitForHeldStream()
|
||||
await openNewSession(fixture.page)
|
||||
|
|
@ -229,7 +230,10 @@ test.describe('large session resume', () => {
|
|||
await fixture.page.screenshot({ path: testInfo.outputPath(`${resumeKind}-background-inference-resume.png`), fullPage: false })
|
||||
|
||||
expect(await textNodeOccurrences(fixture.page, BACKGROUND_PROMPT), 'the running user prompt should appear once').toBe(1)
|
||||
expect(await textNodeOccurrences(fixture.page, MOCK_REPLY), 'the completed assistant reply should appear once').toBe(1)
|
||||
expect(
|
||||
await textNodeOccurrences(fixture.page, MOCK_REPLY),
|
||||
'the completed assistant reply should add exactly one transcript row',
|
||||
).toBe(initialMockReplyCount + 1)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -23,8 +23,10 @@ export const MOCK_REPLY = 'Hello from the mock inference server! The full boot c
|
|||
export interface MockServerOptions {
|
||||
/** Pause the matching stream after its first token for session-switch E2E coverage. */
|
||||
holdFirstStreamForPrompt?: string
|
||||
/** Pause the first completion whose request JSON contains this text. */
|
||||
holdFirstCompletionContaining?: string
|
||||
/** Pause the first completion whose request JSON contains this text. */
|
||||
holdFirstCompletionContaining?: string
|
||||
/** Absolute sandbox path written by the verify-on-stop scripted tool call. */
|
||||
verificationWritePath?: string
|
||||
}
|
||||
|
||||
export interface MockServer {
|
||||
|
|
@ -104,6 +106,9 @@ let _queueStopIndex = 0
|
|||
/** Per-server counter for the correction/session-switch script. */
|
||||
let _correctionSwitchIndex = 0
|
||||
|
||||
/** Per-server counter for the verify-on-stop script. */
|
||||
let _verificationStopIndex = 0
|
||||
|
||||
/** User messages received by the mock, for E2E assertions on real submits. */
|
||||
const _receivedUserTexts: string[] = []
|
||||
|
||||
|
|
@ -114,6 +119,7 @@ function resetScriptIndex(): void {
|
|||
_sidebarCrossIndex = 0
|
||||
_queueStopIndex = 0
|
||||
_correctionSwitchIndex = 0
|
||||
_verificationStopIndex = 0
|
||||
_receivedUserTexts.length = 0
|
||||
}
|
||||
|
||||
|
|
@ -214,6 +220,32 @@ const CORRECTION_SWITCH_SCRIPT: ScriptedTurn[] = [
|
|||
|
||||
export const CORRECTION_SWITCH_TRIGGER = 'E2E_CORRECTION_SWITCH_TRIGGER'
|
||||
|
||||
/**
|
||||
* Drives a real code edit followed by two finish attempts. Hermes should add
|
||||
* its synthetic verify-on-stop continuation after each finish attempt until
|
||||
* the bounded verifier gives up. The mock's request capture proves the nudge
|
||||
* reached the model; desktop must never render it as chat content.
|
||||
*/
|
||||
function verificationStopScript(writePath: string): ScriptedTurn[] {
|
||||
return [
|
||||
{
|
||||
text: 'I will make the requested code change.',
|
||||
toolCalls: [{
|
||||
name: 'write_file',
|
||||
args: {
|
||||
path: writePath,
|
||||
content: 'def changed_by_e2e():\n return "changed"\n',
|
||||
},
|
||||
}],
|
||||
},
|
||||
{ text: 'The code edit is complete.' },
|
||||
{ text: 'I cannot provide fresh verification evidence for that edit.' },
|
||||
]
|
||||
}
|
||||
|
||||
export const VERIFICATION_STOP_TRIGGER = 'E2E_VERIFY_ON_STOP_TRIGGER'
|
||||
export const VERIFICATION_STOP_TEXT = 'I cannot provide fresh verification evidence for that edit.'
|
||||
|
||||
/**
|
||||
* A marker that makes the mock emit a real blocking clarify tool call. Tests
|
||||
* use it to hold a turn open while exercising busy-composer interactions.
|
||||
|
|
@ -340,6 +372,9 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
const isSidebarTrigger = userText.includes('E2E_SIDEBAR_TRIGGER')
|
||||
const isSidebarCrossTrigger = userText.includes('E2E_SIDEBAR_CROSS')
|
||||
const isQueueStopTrigger = userText.includes('E2E_QUEUE_STOP_TRIGGER')
|
||||
const isVerificationStopTrigger = messages.some(
|
||||
message => typeof message?.content === 'string' && message.content.includes(VERIFICATION_STOP_TRIGGER),
|
||||
)
|
||||
const isCorrectionSwitchTrigger = messages.some(
|
||||
message => typeof message?.content === 'string' && message.content.includes(CORRECTION_SWITCH_TRIGGER),
|
||||
)
|
||||
|
|
@ -364,6 +399,18 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
return
|
||||
}
|
||||
|
||||
if (isVerificationStopTrigger) {
|
||||
const script = verificationStopScript(options.verificationWritePath ?? 'e2e-verification-target.py')
|
||||
const turn = script[_verificationStopIndex] ?? script[script.length - 1]
|
||||
_verificationStopIndex++
|
||||
if (stream) {
|
||||
streamScriptedTurn(res, model, turn)
|
||||
} else {
|
||||
nonStreamingScriptedTurn(res, model, turn)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (isCorrectionSwitchTrigger) {
|
||||
const turn = CORRECTION_SWITCH_SCRIPT[_correctionSwitchIndex] ?? CORRECTION_SWITCH_SCRIPT[CORRECTION_SWITCH_SCRIPT.length - 1]
|
||||
_correctionSwitchIndex++
|
||||
|
|
|
|||
226
apps/desktop/e2e/real-session-builder.ts
Normal file
226
apps/desktop/e2e/real-session-builder.ts
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process'
|
||||
import * as path from 'node:path'
|
||||
import { createInterface } from 'node:readline'
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
const DEFAULT_TIMEOUT_MS = 60_000
|
||||
|
||||
interface JsonRpcError {
|
||||
code?: number
|
||||
message?: string
|
||||
}
|
||||
|
||||
interface JsonRpcFrame {
|
||||
error?: JsonRpcError
|
||||
id?: number
|
||||
method?: string
|
||||
params?: {
|
||||
payload?: unknown
|
||||
session_id?: string
|
||||
type?: string
|
||||
}
|
||||
result?: unknown
|
||||
}
|
||||
|
||||
interface CreatedSession {
|
||||
session_id: string
|
||||
stored_session_id: string
|
||||
}
|
||||
|
||||
export interface RealSessionSpec {
|
||||
/** Human-visible sidebar title, persisted by the first completed turn. */
|
||||
title: string
|
||||
/** Each item becomes one real user prompt followed by the mock provider's reply. */
|
||||
turns: readonly string[]
|
||||
}
|
||||
|
||||
export interface RealSession {
|
||||
/** Runtime-only TUI session id, valid only while the builder process is alive. */
|
||||
runtimeId: string
|
||||
/** Durable SessionDB id that desktop resumes after the builder exits. */
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates durable desktop session history through the real TUI gateway and
|
||||
* AIAgent loop, using the E2E mock provider configured in `hermesHome`.
|
||||
*
|
||||
* This intentionally uses the shipped stdio JSON-RPC transport instead of
|
||||
* importing SessionDB or launching Electron. The desktop's WebSocket backend
|
||||
* dispatches the same `tui_gateway.server` methods.
|
||||
*/
|
||||
export class RealSessionBuilder {
|
||||
private readonly child: ChildProcessWithoutNullStreams
|
||||
private nextRequestId = 0
|
||||
private readonly pending = new Map<number, { reject: (reason: Error) => void; resolve: (value: unknown) => void }>()
|
||||
private readonly events: JsonRpcFrame[] = []
|
||||
private readonly eventWaiters: Array<{
|
||||
predicate: (frame: JsonRpcFrame) => boolean
|
||||
reject: (reason: Error) => void
|
||||
resolve: (frame: JsonRpcFrame) => void
|
||||
}> = []
|
||||
private readonly stderr: string[] = []
|
||||
private closed = false
|
||||
|
||||
private constructor(hermesHome: string) {
|
||||
this.child = spawn('uv', ['run', '--active', '--no-sync', 'python', '-m', 'tui_gateway.entry'], {
|
||||
cwd: REPO_ROOT,
|
||||
env: {
|
||||
...process.env,
|
||||
HERMES_HOME: hermesHome,
|
||||
PYTHONPATH: REPO_ROOT,
|
||||
},
|
||||
stdio: 'pipe',
|
||||
})
|
||||
|
||||
createInterface({ input: this.child.stdout }).on('line', line => this.handleLine(line))
|
||||
createInterface({ input: this.child.stderr }).on('line', line => {
|
||||
this.stderr.push(line)
|
||||
if (this.stderr.length > 80) this.stderr.shift()
|
||||
})
|
||||
this.child.once('error', error => this.failAll(new Error(`real-session gateway failed to start: ${error.message}`)))
|
||||
this.child.once('exit', (code, signal) => {
|
||||
if (!this.closed) {
|
||||
this.failAll(new Error(`real-session gateway exited unexpectedly (${signal ?? code ?? 'unknown'}):\n${this.stderr.join('\n')}`))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
static async start(hermesHome: string): Promise<RealSessionBuilder> {
|
||||
const builder = new RealSessionBuilder(hermesHome)
|
||||
await builder.waitForEvent(frame => frame.params?.type === 'gateway.ready')
|
||||
return builder
|
||||
}
|
||||
|
||||
async createSession(spec: RealSessionSpec): Promise<RealSession> {
|
||||
if (spec.turns.length === 0) {
|
||||
throw new Error('RealSessionBuilder requires at least one turn so the real agent creates a durable session row')
|
||||
}
|
||||
|
||||
const created = await this.request<CreatedSession>('session.create', {
|
||||
cols: 120,
|
||||
cwd: REPO_ROOT,
|
||||
source: 'desktop',
|
||||
title: spec.title,
|
||||
})
|
||||
const runtimeId = requireString(created, 'session_id')
|
||||
const sessionId = requireString(created, 'stored_session_id')
|
||||
|
||||
for (const text of spec.turns) {
|
||||
const completion = this.waitForEvent(
|
||||
frame => frame.params?.type === 'message.complete' && frame.params.session_id === runtimeId,
|
||||
)
|
||||
await this.request('prompt.submit', { session_id: runtimeId, text })
|
||||
const frame = await completion
|
||||
const status = readString(frame.params?.payload, 'status')
|
||||
if (status !== 'complete') {
|
||||
throw new Error(`real session turn failed with status ${status ?? 'unknown'}: ${JSON.stringify(frame.params?.payload)}`)
|
||||
}
|
||||
}
|
||||
|
||||
await this.request('session.close', { session_id: runtimeId })
|
||||
return { runtimeId, sessionId }
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closed) return
|
||||
this.closed = true
|
||||
this.child.stdin.end()
|
||||
await new Promise<void>(resolve => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.child.kill('SIGTERM')
|
||||
resolve()
|
||||
}, 5_000)
|
||||
this.child.once('exit', () => {
|
||||
clearTimeout(timeout)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private request<T = unknown>(method: string, params: Record<string, unknown>): Promise<T> {
|
||||
const id = ++this.nextRequestId
|
||||
return this.withTimeout(new Promise<T>((resolve, reject) => {
|
||||
this.pending.set(id, { resolve: value => resolve(value as T), reject })
|
||||
this.child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`, error => {
|
||||
if (error) {
|
||||
this.pending.delete(id)
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
}), `request ${method}`)
|
||||
}
|
||||
|
||||
private waitForEvent(predicate: (frame: JsonRpcFrame) => boolean): Promise<JsonRpcFrame> {
|
||||
const index = this.events.findIndex(predicate)
|
||||
if (index >= 0) {
|
||||
return Promise.resolve(this.events.splice(index, 1)[0])
|
||||
}
|
||||
return this.withTimeout(new Promise<JsonRpcFrame>((resolve, reject) => {
|
||||
this.eventWaiters.push({ predicate, resolve, reject })
|
||||
}), 'gateway event')
|
||||
}
|
||||
|
||||
private handleLine(line: string): void {
|
||||
let frame: JsonRpcFrame
|
||||
try {
|
||||
frame = JSON.parse(line) as JsonRpcFrame
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
if (typeof frame.id === 'number') {
|
||||
const pending = this.pending.get(frame.id)
|
||||
if (!pending) return
|
||||
this.pending.delete(frame.id)
|
||||
if (frame.error) {
|
||||
pending.reject(new Error(`JSON-RPC error ${frame.error.code ?? 'unknown'}: ${frame.error.message ?? 'unknown error'}`))
|
||||
} else {
|
||||
pending.resolve(frame.result)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (frame.method !== 'event') return
|
||||
const waiter = this.eventWaiters.find(candidate => candidate.predicate(frame))
|
||||
if (!waiter) {
|
||||
this.events.push(frame)
|
||||
return
|
||||
}
|
||||
this.eventWaiters.splice(this.eventWaiters.indexOf(waiter), 1)
|
||||
waiter.resolve(frame)
|
||||
}
|
||||
|
||||
private withTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`Timed out after ${DEFAULT_TIMEOUT_MS / 1000}s waiting for ${operation}:\n${this.stderr.join('\n')}`)), DEFAULT_TIMEOUT_MS)
|
||||
promise.then(value => {
|
||||
clearTimeout(timer)
|
||||
resolve(value)
|
||||
}, error => {
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private failAll(error: Error): void {
|
||||
for (const pending of this.pending.values()) pending.reject(error)
|
||||
this.pending.clear()
|
||||
for (const waiter of this.eventWaiters) waiter.reject(error)
|
||||
this.eventWaiters.length = 0
|
||||
}
|
||||
}
|
||||
|
||||
function readString(value: unknown, key: string): string | undefined {
|
||||
if (!value || typeof value !== 'object') return undefined
|
||||
const candidate = (value as Record<string, unknown>)[key]
|
||||
return typeof candidate === 'string' ? candidate : undefined
|
||||
}
|
||||
|
||||
function requireString(value: unknown, key: string): string {
|
||||
const candidate = readString(value, key)
|
||||
if (!candidate) throw new Error(`Gateway response omitted required ${key}: ${JSON.stringify(value)}`)
|
||||
return candidate
|
||||
}
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Seed a deterministic, tool-free large session into an isolated state.db."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
sys.path.insert(0, str(repo_root))
|
||||
|
||||
from hermes_state import SessionDB # noqa: E402
|
||||
|
||||
SESSION_ID = "e2e-large-session"
|
||||
SESSION_TITLE = "E2E large persisted session"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit(f"usage: {sys.argv[0]} <state.db>")
|
||||
|
||||
messages = []
|
||||
for index in range(53):
|
||||
role = "user" if index % 2 == 0 else "assistant"
|
||||
content = (
|
||||
f"E2E persisted user message {index}: audit the compatibility matrix"
|
||||
if role == "user"
|
||||
else f"E2E persisted assistant reply {index}: recorded the audit result"
|
||||
)
|
||||
messages.append({"role": role, "content": content, "timestamp": 1_700_000_000 + index})
|
||||
|
||||
database = SessionDB(db_path=Path(sys.argv[1]))
|
||||
result = database.import_sessions(
|
||||
[
|
||||
{
|
||||
"id": SESSION_ID,
|
||||
"source": "desktop",
|
||||
"model": "mock-model",
|
||||
"started_at": 1_700_000_000,
|
||||
"title": SESSION_TITLE,
|
||||
"cwd": str(repo_root),
|
||||
"system_prompt": "",
|
||||
"messages": messages,
|
||||
}
|
||||
]
|
||||
)
|
||||
database.close()
|
||||
|
||||
if not result.get("ok") or result.get("imported") != 1:
|
||||
raise SystemExit(f"failed to seed large session: {result}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Seed a Hermes state.db with a session exported from a real conversation.
|
||||
|
||||
Usage: seed_session_db.py <state_db_path> <fixture_json_path>
|
||||
|
||||
Creates the database with the full SessionDB schema (if it doesn't exist)
|
||||
and imports the session from the JSON fixture. Uses the real
|
||||
SessionDB.import_sessions() so the data shape matches what the desktop
|
||||
backend expects.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add the repo root to sys.path so we can import hermes_state.
|
||||
# The script is invoked from apps/desktop/e2e/ — repo root is ../../..
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
sys.path.insert(0, str(repo_root))
|
||||
|
||||
from hermes_state import SessionDB # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} <state_db_path> <fixture_json_path>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
db_path = Path(sys.argv[1])
|
||||
fixture_path = Path(sys.argv[2])
|
||||
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(fixture_path, "r", encoding="utf-8") as f:
|
||||
session_data = json.load(f)
|
||||
|
||||
db = SessionDB(db_path=db_path)
|
||||
result = db.import_sessions([session_data])
|
||||
|
||||
if not result.get("ok"):
|
||||
print(f"Import failed: {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
imported = result.get("imported", 0)
|
||||
skipped = result.get("skipped", 0)
|
||||
errors = result.get("errors", [])
|
||||
|
||||
if errors:
|
||||
print(f"Import had errors: {errors}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Seeded {imported} session(s), skipped {skipped} → {db_path}")
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -28,11 +28,6 @@
|
|||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { expect, test } from './test'
|
||||
|
||||
import {
|
||||
|
|
@ -45,12 +40,9 @@ import {
|
|||
launchDesktop,
|
||||
} from './fixtures'
|
||||
import { startMockServer } from './mock-server'
|
||||
import { RealSessionBuilder } from './real-session-builder'
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
const SEED_SCRIPT = path.join(DESKTOP_ROOT, 'e2e', 'scripts', 'seed_session_db.py')
|
||||
const SESSION_TITLE = 'E2E Warm Resume Jitter Test'
|
||||
const SESSION_ID = 'e2e-warm-resume-session'
|
||||
/** 32 messages (16 user/assistant pairs) — enough DOM churn for detection. */
|
||||
const MESSAGE_COUNT = 32
|
||||
/** Seeded PRNG so the generated content is deterministic across runs. */
|
||||
|
|
@ -82,54 +74,27 @@ function gibberish(rng: () => number): string {
|
|||
const FIRST_USER_MSG = gibberish(mulberry32(RNG_SEED))
|
||||
|
||||
/**
|
||||
* Generate a session fixture with MESSAGE_COUNT messages (user/assistant
|
||||
* pairs) of seeded gibberish — just role + content, enough for SessionDB
|
||||
* to import and the transcript to render. Written to a temp file for the
|
||||
* seed script.
|
||||
* Generate the user turns for a real session. The mock provider produces the
|
||||
* assistant side of each pair through the normal AIAgent persistence path.
|
||||
*/
|
||||
function generateSessionFixture(fixturePath: string): void {
|
||||
function generateSessionTurns(): string[] {
|
||||
const rng = mulberry32(RNG_SEED)
|
||||
const messages: Array<{ role: string; content: string }> = []
|
||||
const turns: string[] = []
|
||||
|
||||
for (let i = 0; i < MESSAGE_COUNT / 2; i++) {
|
||||
messages.push({ role: 'user', content: gibberish(rng) })
|
||||
messages.push({ role: 'assistant', content: gibberish(rng) })
|
||||
turns.push(gibberish(rng))
|
||||
gibberish(rng)
|
||||
}
|
||||
|
||||
const session = {
|
||||
id: SESSION_ID,
|
||||
source: 'cli',
|
||||
model: 'mock-model',
|
||||
system_prompt: '',
|
||||
started_at: 1721692800.0,
|
||||
message_count: MESSAGE_COUNT,
|
||||
title: SESSION_TITLE,
|
||||
cwd: '/tmp',
|
||||
archived: 0,
|
||||
rewind_count: 0,
|
||||
compression_fallback_streak: 0,
|
||||
messages,
|
||||
}
|
||||
|
||||
fs.writeFileSync(fixturePath, JSON.stringify(session), 'utf8')
|
||||
}
|
||||
|
||||
/** Resolve the python binary from the nix devshell (falls back to python3). */
|
||||
function findPython(): string {
|
||||
const result = spawnSync('which', ['python'], { encoding: 'utf8' })
|
||||
if (result.status === 0 && result.stdout.trim()) {
|
||||
return result.stdout.trim()
|
||||
}
|
||||
return 'python3'
|
||||
return turns
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a mock-backend sandbox with a pre-seeded session in state.db.
|
||||
* Set up a mock-backend sandbox with a real persisted session in state.db.
|
||||
*
|
||||
* Unlike the shared `setupMockBackend()`, this variant seeds the DB
|
||||
* BEFORE launching the app so the session appears in the sidebar on first
|
||||
* load — exercising the real `resumeSession()` cold path without needing
|
||||
* to send a message first.
|
||||
* Unlike the shared `setupMockBackend()`, this variant creates the session
|
||||
* through the real stdio gateway before launching desktop so the session is
|
||||
* visible in the sidebar on first load.
|
||||
*/
|
||||
async function setupSeededMockBackend(): Promise<MockBackendFixture> {
|
||||
// 1. Start mock server
|
||||
|
|
@ -140,28 +105,13 @@ async function setupSeededMockBackend(): Promise<MockBackendFixture> {
|
|||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
// 3. Pre-seed state.db: generate a fixture JSON to a temp file, then
|
||||
// run the seed script to import it into state.db BEFORE launching.
|
||||
const stateDbPath = path.join(sandbox.hermesHome, 'state.db')
|
||||
const fixturePath = path.join(os.tmpdir(), `hermes-e2e-warm-resume-${Date.now()}.json`)
|
||||
generateSessionFixture(fixturePath)
|
||||
const python = findPython()
|
||||
const seedResult = spawnSync(
|
||||
python,
|
||||
[SEED_SCRIPT, stateDbPath, fixturePath],
|
||||
{
|
||||
cwd: REPO_ROOT,
|
||||
env: { ...process.env, PYTHONPATH: REPO_ROOT },
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
},
|
||||
)
|
||||
fs.unlinkSync(fixturePath)
|
||||
|
||||
if (seedResult.status !== 0) {
|
||||
throw new Error(
|
||||
`Failed to seed state.db:\nstdout: ${seedResult.stdout}\nstderr: ${seedResult.stderr}`,
|
||||
)
|
||||
// 3. Produce all 16 user/assistant pairs through the real TUI gateway,
|
||||
// AIAgent, mock provider, and SessionDB persistence path before desktop starts.
|
||||
const builder = await RealSessionBuilder.start(sandbox.hermesHome)
|
||||
try {
|
||||
await builder.createSession({ title: SESSION_TITLE, turns: generateSessionTurns() })
|
||||
} finally {
|
||||
await builder.close()
|
||||
}
|
||||
|
||||
// 4. Build env + launch
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ const originalWebSocket = globalThis.WebSocket
|
|||
beforeEach(() => {
|
||||
// Drop any parked gateway left by a prior file/case (globalThis slot).
|
||||
const leftover = takeGatewaySurvivor()
|
||||
|
||||
if (leftover) {
|
||||
try {
|
||||
leftover.gateway.close()
|
||||
|
|
@ -141,6 +142,7 @@ beforeEach(() => {
|
|||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
vi.useFakeTimers()
|
||||
FakeWebSocket.mode = 'open'
|
||||
FakeWebSocket.instances = []
|
||||
|
|
@ -166,6 +168,7 @@ afterEach(() => {
|
|||
// open gateway instead of tearing it down (the real HMR path). Drain + close
|
||||
// that survivor so the next test boots a fresh socket instead of adoptBoot().
|
||||
const survivor = takeGatewaySurvivor()
|
||||
|
||||
if (survivor) {
|
||||
try {
|
||||
survivor.gateway.close()
|
||||
|
|
@ -173,6 +176,7 @@ afterEach(() => {
|
|||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
vi.useRealTimers()
|
||||
;(globalThis as { WebSocket: unknown }).WebSocket = originalWebSocket
|
||||
delete (window as { hermesDesktop?: unknown }).hermesDesktop
|
||||
|
|
|
|||
|
|
@ -501,27 +501,36 @@ export function useMessageStream({
|
|||
const existing = prev[index]
|
||||
const existingText = chatMessageText(existing).trim()
|
||||
|
||||
// The last assistant row is a sealed interim (a tool-call turn or a
|
||||
// verify-on-stop candidate — `message.interim` fires for BOTH, see
|
||||
// tui_gateway `_load_interim_assistant_messages`). When the final
|
||||
// completion is the SAME turn's reply, settle it onto that interim
|
||||
// instead of appending a second bubble. Continuity, not exact
|
||||
// equality: streaming can drop characters and the final may add a
|
||||
// trailing delta, so treat prefix-either-way as the same message.
|
||||
// (mergeFinalAssistantText, via completeMessage, does the real
|
||||
// text merge — replaces the interim's text with the full final.)
|
||||
const finalContinuesInterim = Boolean(
|
||||
existing.interim &&
|
||||
finalText &&
|
||||
existingText &&
|
||||
(finalText === existingText || finalText.startsWith(existingText) || existingText.startsWith(finalText))
|
||||
)
|
||||
|
||||
if (existing.pending || (!interimBoundaryPending && finalText && existingText === finalText)) {
|
||||
nextMessages = prev.map((message, messageIndex) =>
|
||||
messageIndex === index ? completeMessage(message) : message
|
||||
)
|
||||
} else if (
|
||||
interimBoundaryPending &&
|
||||
responsePreviewed &&
|
||||
finalText &&
|
||||
existingText &&
|
||||
finalText.startsWith(existingText)
|
||||
) {
|
||||
// The verification candidate was published provisionally as an
|
||||
// interim message and then reused as the terminal response
|
||||
// (continuation-budget fallback). Settle the interim in place
|
||||
// instead of creating a duplicate — the DB has one row, so the
|
||||
// live UI must agree. (#65919 review: duplicate-message blocker)
|
||||
//
|
||||
// Prefix match (not exact equality): the final response may be
|
||||
// the streamed text plus a trailing delta. mergeFinalAssistantText
|
||||
// (called via completeMessage) handles the actual merge — it
|
||||
// strips the old text parts and appends the full final text.
|
||||
} else if (interimBoundaryPending && (responsePreviewed || finalContinuesInterim)) {
|
||||
// Settle the interim in place instead of creating a duplicate —
|
||||
// the DB has one row, so the live UI must agree. Previously this
|
||||
// was gated on `responsePreviewed` alone, so a NON-previewed
|
||||
// tool-call turn whose final matched its sealed interim appended a
|
||||
// second bubble (the "renders twice: partial first copy + clean
|
||||
// final" bug, #63679). `finalContinuesInterim` closes that gap
|
||||
// for ordinary tool-call turns while `responsePreviewed` still
|
||||
// covers the verify-on-stop continuation-budget case even when the
|
||||
// final text was rewritten and no longer shares a prefix.
|
||||
nextMessages = prev.map((message, messageIndex) =>
|
||||
messageIndex === index ? completeMessage(message) : message
|
||||
)
|
||||
|
|
|
|||
|
|
@ -186,18 +186,51 @@ describe('useMessageStream interim text sealing', () => {
|
|||
expect(getState().interimBoundaryPending).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps an identical final completion distinct from an interim reply without response_previewed', async () => {
|
||||
it('settles an identical final onto a non-previewed interim (tool-call turn) instead of duplicating (#63679)', async () => {
|
||||
await mountStream()
|
||||
await start()
|
||||
|
||||
// A plain tool-call turn: the streamed text is sealed as an interim at the
|
||||
// tool boundary (no response_previewed — that flag is only for verify-on-
|
||||
// stop). The final completion is the SAME turn's reply. It must settle onto
|
||||
// the interim, not append a second bubble — the DB has one row. This is the
|
||||
// "renders twice" bug: partial streamed copy + clean final copy side by side.
|
||||
await interim('same reply')
|
||||
await complete('same reply')
|
||||
|
||||
// Without response_previewed, the interim and terminal replies are
|
||||
// distinct messages — the gateway didn't signal that the final reuses
|
||||
// the provisional candidate.
|
||||
const texts = assistantMessages()
|
||||
expect(texts.filter(t => t === 'same reply')).toHaveLength(2)
|
||||
expect(texts.filter(t => t === 'same reply')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('settles a prefix-extended final onto a non-previewed interim (streamed + trailing delta)', async () => {
|
||||
await mountStream()
|
||||
await start()
|
||||
|
||||
// The stream dropped/settled early at the tool boundary; the final adds a
|
||||
// trailing delta. Same turn — one bubble with the full final text.
|
||||
await delta('partial')
|
||||
await interim('partial')
|
||||
await complete('partial answer continued')
|
||||
|
||||
const texts = assistantMessages()
|
||||
expect(texts.filter(t => t.includes('partial'))).toHaveLength(1)
|
||||
expect(texts[0]).toBe('partial answer continued')
|
||||
})
|
||||
|
||||
it('appends a genuinely different final as its own bubble (two real assistant segments)', async () => {
|
||||
await mountStream()
|
||||
await start()
|
||||
|
||||
// The interim is one segment (pre-tool commentary); the final is different
|
||||
// content, not a continuation of it. These are two real messages and must
|
||||
// both render — the fix must not over-collapse distinct replies.
|
||||
await interim('let me check the files')
|
||||
await complete('the answer is 42')
|
||||
|
||||
const texts = assistantMessages()
|
||||
expect(texts).toContain('let me check the files')
|
||||
expect(texts).toContain('the answer is 42')
|
||||
expect(texts).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('settles an identical final completion onto the interim when response_previewed', async () => {
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ export const en: Translations = {
|
|||
errors: {
|
||||
elevenLabsNeedsKey: 'ElevenLabs STT needs ELEVENLABS_API_KEY.',
|
||||
elevenLabsRejectedKey: 'ElevenLabs rejected the API key (401).',
|
||||
gatewayAuthFailed: 'Gateway authentication failed — check your API_SERVER_KEY.',
|
||||
methodNotAllowed:
|
||||
'The desktop backend rejected that request (405 Method Not Allowed). Try restarting Hermes Desktop.',
|
||||
microphonePermission: 'Microphone permission was denied.',
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ export const ja = defineLocale({
|
|||
errors: {
|
||||
elevenLabsNeedsKey: 'ElevenLabs STT には ELEVENLABS_API_KEY が必要です。',
|
||||
elevenLabsRejectedKey: 'ElevenLabs が API キーを拒否しました (401)。',
|
||||
gatewayAuthFailed: 'ゲートウェイ認証に失敗しました — API_SERVER_KEY を確認してください。',
|
||||
methodNotAllowed:
|
||||
'デスクトップバックエンドがそのリクエストを拒否しました (405 Method Not Allowed)。Hermes Desktop を再起動してください。',
|
||||
microphonePermission: 'マイクのアクセス許可が拒否されました。',
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ export interface Translations {
|
|||
errors: {
|
||||
elevenLabsNeedsKey: string
|
||||
elevenLabsRejectedKey: string
|
||||
gatewayAuthFailed: string
|
||||
methodNotAllowed: string
|
||||
microphonePermission: string
|
||||
openaiRejectedApiKey: string
|
||||
|
|
|
|||
|
|
@ -129,6 +129,7 @@ export const zhHant = defineLocale({
|
|||
errors: {
|
||||
elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。',
|
||||
elevenLabsRejectedKey: 'ElevenLabs 拒絕了該 API 金鑰 (401)。',
|
||||
gatewayAuthFailed: '閘道認證失敗 — 請檢查你的 API_SERVER_KEY。',
|
||||
methodNotAllowed: '桌面後端拒絕了該請求 (405 Method Not Allowed)。請嘗試重新啟動 Hermes Desktop。',
|
||||
microphonePermission: '麥克風權限已被拒絕。',
|
||||
openaiRejectedApiKey: 'OpenAI 拒絕了該 API 金鑰。',
|
||||
|
|
|
|||
|
|
@ -129,6 +129,7 @@ export const zh: Translations = {
|
|||
errors: {
|
||||
elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。',
|
||||
elevenLabsRejectedKey: 'ElevenLabs 拒绝了该 API key (401)。',
|
||||
gatewayAuthFailed: '网关认证失败 — 请检查你的 API_SERVER_KEY。',
|
||||
methodNotAllowed: '桌面后端拒绝了该请求 (405 Method Not Allowed)。请尝试重启 Hermes Desktop。',
|
||||
microphonePermission: '麦克风权限已被拒绝。',
|
||||
openaiRejectedApiKey: 'OpenAI 拒绝了该 API key。',
|
||||
|
|
|
|||
|
|
@ -158,6 +158,39 @@ describe('toChatMessages', () => {
|
|||
|
||||
expect(chatMessageText(message)).toBe('@file:foo.ts\n\nlook')
|
||||
})
|
||||
|
||||
it('projects durable timeline kinds without inspecting their text', () => {
|
||||
const messages = toChatMessages([
|
||||
{ role: 'user', content: 'real user turn', timestamp: 1 },
|
||||
{ role: 'assistant', content: 'real assistant reply', timestamp: 2 },
|
||||
{
|
||||
role: 'user',
|
||||
content: 'opaque compaction payload',
|
||||
display_kind: 'hidden',
|
||||
timestamp: 3
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: 'opaque model context payload',
|
||||
display_kind: 'model_switch',
|
||||
timestamp: 4
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: 'opaque delegation context payload',
|
||||
display_kind: 'async_delegation_complete',
|
||||
timestamp: 5
|
||||
}
|
||||
])
|
||||
|
||||
expect(messages.map(message => message.role)).toEqual(['user', 'assistant', 'system', 'system'])
|
||||
expect(messages.map(chatMessageText)).toEqual([
|
||||
'real user turn',
|
||||
'real assistant reply',
|
||||
'model changed',
|
||||
'background agent work finished'
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('renderMediaTags', () => {
|
||||
|
|
|
|||
|
|
@ -303,6 +303,29 @@ function displayContentForMessage(role: SessionMessage['role'], content: unknown
|
|||
return [refs.join('\n'), visibleText].filter(Boolean).join('\n\n') || visibleText
|
||||
}
|
||||
|
||||
function transcriptContent(displayKind: SessionMessage['display_kind'], content: string): string | null {
|
||||
return displayKind === 'hidden' ? null : content
|
||||
}
|
||||
|
||||
function timelineDisplayContent(message: SessionMessage, content: string): string {
|
||||
if (message.display_kind === 'model_switch') {
|
||||
return 'model changed'
|
||||
}
|
||||
|
||||
if (message.display_kind === 'async_delegation_complete') {
|
||||
const count =
|
||||
message.display_metadata && 'task_count' in message.display_metadata
|
||||
? message.display_metadata.task_count
|
||||
: undefined
|
||||
|
||||
return count === undefined
|
||||
? 'background agent work finished'
|
||||
: `${count} background agent${count === 1 ? '' : 's'} finished`
|
||||
}
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
const STREAM_PART: Record<'reasoning' | 'text', (text: string) => ChatMessagePart> = {
|
||||
reasoning: reasoningPart,
|
||||
text: textPart
|
||||
|
|
@ -884,7 +907,17 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
|||
}
|
||||
|
||||
const content = message.content || message.text || message.context || message.name
|
||||
const displayContent = displayContentForMessage(message.role, content)
|
||||
|
||||
const displayContent = transcriptContent(
|
||||
message.display_kind,
|
||||
timelineDisplayContent(message, displayContentForMessage(message.role, content))
|
||||
)
|
||||
|
||||
const displayRole =
|
||||
message.display_kind === 'model_switch' || message.display_kind === 'async_delegation_complete'
|
||||
? 'system'
|
||||
: message.role
|
||||
|
||||
const parts: ChatMessagePart[] = []
|
||||
|
||||
const reasoning =
|
||||
|
|
@ -897,7 +930,7 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
|||
}
|
||||
|
||||
if (displayContent) {
|
||||
parts.push(message.role === 'assistant' ? assistantTextPart(displayContent) : textPart(displayContent))
|
||||
parts.push(displayRole === 'assistant' ? assistantTextPart(displayContent) : textPart(displayContent))
|
||||
}
|
||||
|
||||
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
|
||||
|
|
@ -951,8 +984,8 @@ export function toChatMessages(messages: SessionMessage[]): ChatMessage[] {
|
|||
}
|
||||
|
||||
result.push({
|
||||
id: `${message.timestamp || Date.now()}-${index}-${message.role}`,
|
||||
role: message.role,
|
||||
id: `${message.timestamp || Date.now()}-${index}-${displayRole}`,
|
||||
role: displayRole,
|
||||
parts,
|
||||
timestamp: message.timestamp
|
||||
})
|
||||
|
|
|
|||
34
apps/desktop/src/store/notifications.test.ts
Normal file
34
apps/desktop/src/store/notifications.test.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { beforeEach, expect, test } from 'vitest'
|
||||
|
||||
import { $notifications, clearNotifications, notifyError } from './notifications'
|
||||
|
||||
beforeEach(() => {
|
||||
clearNotifications()
|
||||
})
|
||||
|
||||
function lastMessage(): string {
|
||||
return $notifications.get()[0]?.message ?? ''
|
||||
}
|
||||
|
||||
// Regression for #39365: a gateway auth 401 (bad API_SERVER_KEY) must not be
|
||||
// summarized as a provider (OpenAI/OpenRouter) API key problem.
|
||||
test('gateway_auth_failed error is summarized as gateway auth, not provider key', () => {
|
||||
notifyError(
|
||||
new Error(
|
||||
'401 {"error": {"message": "Invalid gateway API key (API_SERVER_KEY)", "type": "gateway_auth_error", "code": "gateway_auth_failed"}}'
|
||||
),
|
||||
'Request failed'
|
||||
)
|
||||
|
||||
expect(lastMessage()).toContain('API_SERVER_KEY')
|
||||
expect(lastMessage()).not.toMatch(/OpenAI/i)
|
||||
})
|
||||
|
||||
test('provider invalid_api_key error still maps to the OpenAI summary', () => {
|
||||
notifyError(
|
||||
new Error('401 {"error": {"message": "Incorrect API key provided", "code": "invalid_api_key"}}'),
|
||||
'Request failed'
|
||||
)
|
||||
|
||||
expect(lastMessage()).toMatch(/OpenAI rejected the API key/i)
|
||||
})
|
||||
|
|
@ -77,6 +77,10 @@ function cleanErrorText(value: string) {
|
|||
}
|
||||
|
||||
const ERROR_SUMMARIES: { test: (msg: string) => boolean; summarize: (msg: string) => string }[] = [
|
||||
{
|
||||
test: msg => /['"]code['"]\s*:\s*['"]gateway_auth_failed['"]/i.test(msg),
|
||||
summarize: () => translateNow('notifications.errors.gatewayAuthFailed')
|
||||
},
|
||||
{
|
||||
test: msg => /incorrect api key provided/i.test(msg) || /['"]code['"]\s*:\s*['"]invalid_api_key['"]/i.test(msg),
|
||||
summarize: msg => {
|
||||
|
|
|
|||
|
|
@ -423,6 +423,16 @@ export interface SessionInfo {
|
|||
is_default_profile?: boolean
|
||||
}
|
||||
|
||||
export type TimelineDisplayMetadata =
|
||||
| { model: string; provider?: string }
|
||||
| {
|
||||
delegation_id: string
|
||||
task_count: number
|
||||
completed_count?: number
|
||||
failed_count?: number
|
||||
duration_seconds?: number
|
||||
}
|
||||
|
||||
export interface SessionMessage {
|
||||
codex_reasoning_items?: unknown
|
||||
content: unknown
|
||||
|
|
@ -431,6 +441,8 @@ export interface SessionMessage {
|
|||
reasoning?: null | string
|
||||
reasoning_content?: null | string
|
||||
reasoning_details?: unknown
|
||||
display_kind?: 'async_delegation_complete' | 'hidden' | 'model_switch' | string
|
||||
display_metadata?: TimelineDisplayMetadata
|
||||
role: 'assistant' | 'system' | 'tool' | 'user'
|
||||
text?: unknown
|
||||
timestamp?: number
|
||||
|
|
|
|||
195
cli.py
195
cli.py
|
|
@ -1743,6 +1743,66 @@ def _worktree_is_dirty(worktree_path: str, timeout: int = 10) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _worktree_commits_all_merged_upstream(
|
||||
worktree_path: str, timeout: int = 30, max_ahead: int = 20
|
||||
) -> bool:
|
||||
"""Return whether every local-only commit is patch-equivalent to a commit
|
||||
already on the default upstream branch.
|
||||
|
||||
The dominant ``.worktrees/`` leak: a branch is pushed, its PR is
|
||||
squash-merged (or cherry-picked), and the remote branch is deleted. The
|
||||
local commits are then unreachable from ``refs/remotes/*`` forever, so the
|
||||
unpushed-commits guard preserves the worktree indefinitely even though its
|
||||
content is fully merged. ``git cherry`` detects patch-equivalence, letting
|
||||
the pruner reap these.
|
||||
|
||||
Bounded: skips (returns False) when the branch is more than ``max_ahead``
|
||||
commits ahead — a stale-base tree, too expensive to diff-hash and unlikely
|
||||
to be a merged scratch branch. Fails SAFE toward False (preserve).
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
base = None
|
||||
for candidate in ("origin/HEAD", "origin/main", "origin/master"):
|
||||
try:
|
||||
probe = subprocess.run(
|
||||
["git", "rev-parse", "--verify", "--quiet", candidate],
|
||||
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
|
||||
)
|
||||
if probe.returncode == 0 and probe.stdout.strip():
|
||||
base = candidate
|
||||
break
|
||||
except Exception:
|
||||
return False
|
||||
if base is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
ahead = subprocess.run(
|
||||
["git", "rev-list", "--count", f"{base}..HEAD"],
|
||||
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
|
||||
)
|
||||
if ahead.returncode != 0:
|
||||
return False
|
||||
count = int(ahead.stdout.strip() or "0")
|
||||
if count == 0:
|
||||
return True
|
||||
if count > max_ahead:
|
||||
return False
|
||||
|
||||
cherry = subprocess.run(
|
||||
["git", "cherry", base, "HEAD"],
|
||||
capture_output=True, text=True, timeout=timeout, cwd=worktree_path,
|
||||
)
|
||||
if cherry.returncode != 0:
|
||||
return False
|
||||
lines = [ln for ln in cherry.stdout.splitlines() if ln.strip()]
|
||||
# "-" = patch-equivalent commit exists upstream; "+" = unique local work
|
||||
return bool(lines) and all(ln.startswith("-") for ln in lines)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _worktree_lock_is_live(repo_root: str, worktree_path: str, timeout: int = 10):
|
||||
"""Classify a worktree's git lock as live, dead, or absent.
|
||||
|
||||
|
|
@ -1949,11 +2009,21 @@ def _run_checkpoint_auto_maintenance() -> None:
|
|||
def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
|
||||
"""Remove stale worktrees and orphaned branches on startup.
|
||||
|
||||
Age-based tiers (aggressive cleanup keeps ``.worktrees/`` from growing
|
||||
unbounded):
|
||||
- Under max_age_hours (24h): skip — session may still be active.
|
||||
- 24h–72h: remove if no unpushed commits.
|
||||
- Over 72h: force remove regardless (nothing should sit this long).
|
||||
Covers EVERY directory under ``.worktrees/`` except kanban task trees
|
||||
(``t_<hex>`` — owned by the kanban dispatcher's own gc). Scratch trees
|
||||
created by ``hermes -w`` (``hermes-*``) age out fast; named trees created
|
||||
manually for salvage/review lanes age out on a slower schedule:
|
||||
|
||||
- ``hermes-*``: skip under 24h; reap 24h+ when clean and merged/pushed;
|
||||
72h+ is the aggressive tier (still never deletes real work).
|
||||
- named trees: same logic at 3x the timeline (72h soft / 9d hard).
|
||||
|
||||
Work-preservation guards (all tiers, any age):
|
||||
- uncommitted changes (dirty) — never removed;
|
||||
- unpushed commits — never removed, UNLESS every local-only commit is
|
||||
patch-equivalent to a commit already on upstream (``git cherry``): the
|
||||
squash-merged-PR case, which is the dominant ``.worktrees/`` leak since
|
||||
those commits stay unreachable from ``refs/remotes/*`` forever.
|
||||
|
||||
Lock handling (orthogonal to age): ``hermes -w`` locks each worktree with
|
||||
reason ``hermes pid=<pid>`` so a concurrent hermes process leaves an in-use
|
||||
|
|
@ -1966,9 +2036,14 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
|
|||
removal never orphans the branch (which would drop easy reachability of any
|
||||
commits still in the worktree).
|
||||
|
||||
Preserved-work visibility: trees skipped for unpushed/dirty reasons that
|
||||
are older than 7 days are listed in a single WARNING so real in-flight
|
||||
work can't rot silently.
|
||||
|
||||
Also prunes orphaned ``hermes/*`` and ``pr-*`` local branches that
|
||||
have no corresponding worktree.
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
|
@ -1978,13 +2053,23 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
|
|||
return
|
||||
|
||||
now = time.time()
|
||||
soft_cutoff = now - (max_age_hours * 3600) # 24h default
|
||||
hard_cutoff = now - (max_age_hours * 3 * 3600) # 72h default
|
||||
stale_work_cutoff = now - (7 * 24 * 3600)
|
||||
preserved_stale: list = []
|
||||
# Kanban task worktrees (<repo>/.worktrees/t_<hex>) have their own
|
||||
# dispatcher-driven lifecycle (hermes kanban gc) — never touch them here.
|
||||
kanban_re = re.compile(r"^t_[0-9a-f]+$")
|
||||
|
||||
for entry in worktrees_dir.iterdir():
|
||||
if not entry.is_dir() or not entry.name.startswith("hermes-"):
|
||||
if not entry.is_dir() or kanban_re.match(entry.name):
|
||||
continue
|
||||
|
||||
# Scratch trees (hermes-*) age out on the default schedule; named
|
||||
# trees (salvage/review lanes someone created deliberately) get 3x.
|
||||
scratch = entry.name.startswith("hermes-")
|
||||
tier_hours = max_age_hours if scratch else max_age_hours * 3
|
||||
soft_cutoff = now - (tier_hours * 3600)
|
||||
hard_cutoff = now - (tier_hours * 3 * 3600)
|
||||
|
||||
# Check age
|
||||
try:
|
||||
mtime = entry.stat().st_mtime
|
||||
|
|
@ -1993,19 +2078,24 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
|
|||
except Exception:
|
||||
continue
|
||||
|
||||
force = mtime <= hard_cutoff # Over 72h — reap aggressively
|
||||
force = mtime <= hard_cutoff # Aggressive tier — reap clean trees
|
||||
|
||||
# Never delete real work, regardless of age. Unpushed commits and
|
||||
# uncommitted changes may be a crashed session's in-flight work; the
|
||||
# >72h tier reaps only abandoned *clean, fully-pushed* worktrees (the
|
||||
# scratch trees that actually cause .worktrees/ bloat).
|
||||
# Never delete real work, regardless of age or tier. Uncommitted
|
||||
# changes and unpushed commits may be a crashed session's in-flight
|
||||
# work; only clean, fully-merged/pushed trees (the scratch trees that
|
||||
# actually cause .worktrees/ bloat) are ever reaped.
|
||||
if _worktree_is_dirty(str(entry), timeout=5):
|
||||
if mtime <= stale_work_cutoff:
|
||||
preserved_stale.append(f"{entry.name} (uncommitted changes)")
|
||||
continue
|
||||
if _worktree_has_unpushed_commits(str(entry), timeout=5):
|
||||
continue # Has unpushed commits or can't check — skip
|
||||
if not force:
|
||||
# 24h–72h tier is conservative: unpushed check above is enough.
|
||||
pass
|
||||
elif _worktree_is_dirty(str(entry), timeout=5):
|
||||
continue # >72h but dirty — preserve uncommitted work
|
||||
# Squash-merge escape hatch: commits unreachable from any remote
|
||||
# ref but patch-equivalent to upstream commits are merged work,
|
||||
# not unpushed work.
|
||||
if not _worktree_commits_all_merged_upstream(str(entry), timeout=30):
|
||||
if mtime <= stale_work_cutoff:
|
||||
preserved_stale.append(f"{entry.name} (unpushed commits)")
|
||||
continue
|
||||
|
||||
# Respect git-native session locks. A lock owned by a still-running
|
||||
# hermes process means the worktree is actively in use — never touch
|
||||
|
|
@ -2054,6 +2144,13 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
|
|||
except Exception as e:
|
||||
logger.debug("Failed to prune worktree %s: %s", entry.name, e)
|
||||
|
||||
if preserved_stale:
|
||||
logger.warning(
|
||||
"Preserving %d worktree(s) older than 7 days with unmerged work "
|
||||
"(push or remove them to reclaim disk): %s",
|
||||
len(preserved_stale), ", ".join(sorted(preserved_stale)),
|
||||
)
|
||||
|
||||
_prune_orphaned_branches(repo_root)
|
||||
|
||||
|
||||
|
|
@ -4168,6 +4265,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._pending_tool_info: dict = {} # function_name -> list of (preview, args) for stacked scrollback
|
||||
self._last_scrollback_tool: str = "" # last tool name printed to scrollback (for "new" dedup)
|
||||
self._command_running = False
|
||||
self._command_blocks_input = False
|
||||
self._command_status = ""
|
||||
# Petdex mascot (opt-in via display.pet). The base CLI mirrors the TUI's
|
||||
# PetPane: a half-block sprite above the prompt that reacts to agent
|
||||
|
|
@ -6181,9 +6279,17 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
return _COMMAND_SPINNER_FRAMES[frame_idx]
|
||||
|
||||
@contextmanager
|
||||
def _busy_command(self, status: str):
|
||||
"""Expose a temporary busy state in the TUI while a slash command runs."""
|
||||
def _busy_command(self, status: str, *, blocks_input: bool = True):
|
||||
"""Expose a temporary busy state in the TUI while a slash command runs.
|
||||
|
||||
Most synchronous slash commands must reserve the composer because their
|
||||
completion changes the active session state. Manual compression is safe
|
||||
to draft through: the queued input is processed against the compacted
|
||||
history after the command completes.
|
||||
"""
|
||||
previous_blocks_input = getattr(self, "_command_blocks_input", False)
|
||||
self._command_running = True
|
||||
self._command_blocks_input = blocks_input
|
||||
self._command_status = status
|
||||
self._invalidate(min_interval=0.0)
|
||||
try:
|
||||
|
|
@ -6191,6 +6297,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
yield
|
||||
finally:
|
||||
self._command_running = False
|
||||
self._command_blocks_input = previous_blocks_input
|
||||
self._command_status = ""
|
||||
self._invalidate(min_interval=0.0)
|
||||
|
||||
|
|
@ -9381,9 +9488,12 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
# has all API keys in os.environ.
|
||||
from tools.environments.local import _sanitize_subprocess_env
|
||||
sanitized_env = _sanitize_subprocess_env(os.environ.copy())
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
result = subprocess.run(
|
||||
exec_cmd, shell=True, capture_output=True,
|
||||
text=True, timeout=30, env=sanitized_env
|
||||
text=True, timeout=30, env=sanitized_env,
|
||||
# No console flash on Windows (#56747).
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
output = result.stdout.strip() or result.stderr.strip()
|
||||
if output:
|
||||
|
|
@ -10017,7 +10127,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
return
|
||||
|
||||
original_count = len(self.conversation_history)
|
||||
with self._busy_command("Compressing context..."):
|
||||
with self._busy_command("Compressing context...", blocks_input=False):
|
||||
try:
|
||||
from agent.model_metadata import estimate_request_tokens_rough
|
||||
from agent.manual_compression_feedback import summarize_manual_compression
|
||||
|
|
@ -10073,6 +10183,39 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
force=True,
|
||||
defer_context_engine_notification=True,
|
||||
)
|
||||
|
||||
# If _compress_context returned unchanged because a
|
||||
# concurrent compression lock is held, tell the user
|
||||
# clearly instead of showing the misleading
|
||||
# "No changes from compression" no-op text. The wording
|
||||
# distinguishes a confirmed holder from an unconfirmed
|
||||
# acquisition failure (describe_compression_lock_skip).
|
||||
# Type-pinned check (is True / str): the flag's only real
|
||||
# values are None/True/holder-string, and a bare getattr
|
||||
# truthiness test is fooled by MagicMock auto-attributes on
|
||||
# test-double agents (skill pitfall: MagicMock vs hasattr).
|
||||
_lock_skip_signal = getattr(
|
||||
self.agent, "_compression_skipped_due_to_lock", None
|
||||
)
|
||||
if _lock_skip_signal is True or isinstance(_lock_skip_signal, str):
|
||||
from agent.manual_compression_feedback import (
|
||||
describe_compression_lock_skip,
|
||||
)
|
||||
print(
|
||||
" "
|
||||
+ describe_compression_lock_skip(
|
||||
self.agent._compression_skipped_due_to_lock
|
||||
)
|
||||
)
|
||||
self.agent._compression_skipped_due_to_lock = None
|
||||
# No boundary was committed on a lock-skip; discard the
|
||||
# deferred context-engine notification (exactly-once).
|
||||
finalize_context_engine_compression_notification(
|
||||
self.agent,
|
||||
committed=False,
|
||||
)
|
||||
return
|
||||
|
||||
if partial and tail:
|
||||
compressed = rejoin_compressed_head_and_tail(compressed, tail)
|
||||
self.conversation_history = compressed
|
||||
|
|
@ -13443,6 +13586,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
|
||||
# Slash command loading state
|
||||
self._command_running = False
|
||||
self._command_blocks_input = False
|
||||
self._command_status = ""
|
||||
|
||||
# Secure secret capture state for skill setup
|
||||
|
|
@ -14404,7 +14548,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
style='class:input-area',
|
||||
multiline=True,
|
||||
wrap_lines=True,
|
||||
read_only=Condition(lambda: bool(cli_ref._command_running)),
|
||||
read_only=Condition(lambda: bool(cli_ref._command_blocks_input)),
|
||||
history=FileHistory(str(self._history_file)),
|
||||
# complete_while_typing fires the completer on every keystroke. The
|
||||
# completer does blocking work — fuzzy @-file indexing shells out to
|
||||
|
|
@ -14615,8 +14759,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
|
||||
if cli_ref._command_running:
|
||||
frame = cli_ref._command_spinner_frame()
|
||||
detail = "input temporarily disabled" if cli_ref._command_blocks_input else "input stays active; Enter queues"
|
||||
return [
|
||||
('class:hint', f' {frame} command in progress · input temporarily disabled'),
|
||||
('class:hint', f' {frame} command in progress · {detail}'),
|
||||
]
|
||||
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
2001Y
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
metamon-p
|
||||
# PR #36220 salvage (channel continuity hint)
|
||||
1
contributors/emails/akitani@akitaninoMac-mini.local
Normal file
1
contributors/emails/akitani@akitaninoMac-mini.local
Normal file
|
|
@ -0,0 +1 @@
|
|||
2001Y
|
||||
|
|
@ -0,0 +1 @@
|
|||
benjamin2026-dot
|
||||
1
contributors/emails/boumagent@gmail.com
Normal file
1
contributors/emails/boumagent@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
patp
|
||||
1
contributors/emails/dickson.neoh@gmail.com
Normal file
1
contributors/emails/dickson.neoh@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
dnth
|
||||
|
|
@ -0,0 +1 @@
|
|||
gonzalofrancoceballos
|
||||
1
contributors/emails/hang.li@tcredit.com
Normal file
1
contributors/emails/hang.li@tcredit.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
airclear
|
||||
2
contributors/emails/harrison@medmetricsrx.com
Normal file
2
contributors/emails/harrison@medmetricsrx.com
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
harrisonmedmedmetrics
|
||||
# C17 Slack reaction events salvage (#33111/#44508/#45265)
|
||||
1
contributors/emails/hello@ianks.com
Normal file
1
contributors/emails/hello@ianks.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
ianks
|
||||
1
contributors/emails/hello@jeromeiveson.com
Normal file
1
contributors/emails/hello@jeromeiveson.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
Trantor-develops
|
||||
2
contributors/emails/john.kattenhorn.personal@gmail.com
Normal file
2
contributors/emails/john.kattenhorn.personal@gmail.com
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
johnkattenhorn
|
||||
# C17 Slack reaction events salvage (#33111/#44508/#45265)
|
||||
1
contributors/emails/jordanh@nvidia.com
Normal file
1
contributors/emails/jordanh@nvidia.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
jordanhubbard
|
||||
2
contributors/emails/kevin@fleetsmarts.net
Normal file
2
contributors/emails/kevin@fleetsmarts.net
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
Kev-fs
|
||||
# C17 Slack reaction events salvage (#33111/#44508/#45265)
|
||||
1
contributors/emails/lanyusea@gmail.com
Normal file
1
contributors/emails/lanyusea@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
lanyusea
|
||||
1
contributors/emails/lg_329@163.com
Normal file
1
contributors/emails/lg_329@163.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
cifangyiquan
|
||||
1
contributors/emails/lucas@policastromd.com
Normal file
1
contributors/emails/lucas@policastromd.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
enzo2
|
||||
1
contributors/emails/mattshapsss@gmail.com
Normal file
1
contributors/emails/mattshapsss@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
mattshapsss
|
||||
1
contributors/emails/mbrooks@slack-corp.com
Normal file
1
contributors/emails/mbrooks@slack-corp.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
mwbrooks
|
||||
2
contributors/emails/mehrzad.karami@gmail.com
Normal file
2
contributors/emails/mehrzad.karami@gmail.com
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
mzkarami
|
||||
# PR #66204 contributor identity
|
||||
1
contributors/emails/mycodeisbad@gmail.com
Normal file
1
contributors/emails/mycodeisbad@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
peterw
|
||||
1
contributors/emails/nawfal.fardana@dana.id
Normal file
1
contributors/emails/nawfal.fardana@dana.id
Normal file
|
|
@ -0,0 +1 @@
|
|||
arimu1
|
||||
1
contributors/emails/panding99@outlook.com
Normal file
1
contributors/emails/panding99@outlook.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
panDing19
|
||||
1
contributors/emails/rg@replygirl.club
Normal file
1
contributors/emails/rg@replygirl.club
Normal file
|
|
@ -0,0 +1 @@
|
|||
replygirl
|
||||
1
contributors/emails/rmk799@outlook.com
Normal file
1
contributors/emails/rmk799@outlook.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
MustafaK99
|
||||
1
contributors/emails/rt.cms012@gmail.com
Normal file
1
contributors/emails/rt.cms012@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
trac3r00
|
||||
1
contributors/emails/schattenan@kagaku.eu
Normal file
1
contributors/emails/schattenan@kagaku.eu
Normal file
|
|
@ -0,0 +1 @@
|
|||
schattenan
|
||||
1
contributors/emails/sdevinarayanan@asymbl.com
Normal file
1
contributors/emails/sdevinarayanan@asymbl.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
shivasymbl
|
||||
1
contributors/emails/shubhambc09@gmail.com
Normal file
1
contributors/emails/shubhambc09@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
navahc09
|
||||
1
contributors/emails/skool@doctablade.com
Normal file
1
contributors/emails/skool@doctablade.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
drleadflow
|
||||
1
contributors/emails/skywind5487@gmail.com
Normal file
1
contributors/emails/skywind5487@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
Skywind5487
|
||||
1
contributors/emails/stanislav@local
Normal file
1
contributors/emails/stanislav@local
Normal file
|
|
@ -0,0 +1 @@
|
|||
sl4m3
|
||||
1
contributors/emails/team@williepeacock.com
Normal file
1
contributors/emails/team@williepeacock.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
peacockesq
|
||||
1
contributors/emails/trkim@vms-solutions.com
Normal file
1
contributors/emails/trkim@vms-solutions.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
ddifa86
|
||||
1
contributors/emails/wilgefortz@gmail.com
Normal file
1
contributors/emails/wilgefortz@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
elphamale
|
||||
1
contributors/emails/yemi@lagosinternationalmarket.com
Normal file
1
contributors/emails/yemi@lagosinternationalmarket.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
yemi-lagosinternationalmarket
|
||||
1
contributors/emails/z23@users.noreply.github.com
Normal file
1
contributors/emails/z23@users.noreply.github.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
z23
|
||||
1
contributors/emails/zhangk1985@gmail.com
Normal file
1
contributors/emails/zhangk1985@gmail.com
Normal file
|
|
@ -0,0 +1 @@
|
|||
kylezh
|
||||
|
|
@ -9,6 +9,7 @@ action="list" and for resolving human-friendly channel names to numeric IDs.
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
|
@ -18,6 +19,14 @@ from utils import atomic_json_write
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
DIRECTORY_PATH = get_hermes_home() / "channel_directory.json"
|
||||
# Throttle window for repeated Slack channel-directory refresh failures.
|
||||
# The directory rebuilds on a timer, so a persistent workspace error (e.g.
|
||||
# missing scope, revoked token) would otherwise re-log the same warning on
|
||||
# every refresh. Warn once per (team, error detail) per interval; repeats
|
||||
# drop to DEBUG.
|
||||
_SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS = 3600
|
||||
_slack_directory_warning_last: Dict[tuple[str, str], float] = {}
|
||||
|
||||
# User-maintained friendly-name overlay. The directory is fully regenerated
|
||||
# from live adapters + session data on a timer, so hand-edits to
|
||||
# channel_directory.json don't survive. Aliases declared here are re-applied
|
||||
|
|
@ -105,6 +114,27 @@ def _session_entry_name(origin: Dict[str, Any]) -> str:
|
|||
return f"{base_name} / {topic_label}"
|
||||
|
||||
|
||||
def _warn_slack_directory(team_id: str, detail: str) -> None:
|
||||
"""Warn once per team/error per interval for recurring Slack refresh failures."""
|
||||
key = (str(team_id), str(detail))
|
||||
now = time.monotonic()
|
||||
last = _slack_directory_warning_last.get(key)
|
||||
if last is None or now - last >= _SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS:
|
||||
_slack_directory_warning_last[key] = now
|
||||
logger.warning(
|
||||
"Channel directory: failed to list Slack channels for team %s: %s",
|
||||
team_id,
|
||||
detail,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Channel directory: suppressed repeated Slack channel list failure "
|
||||
"for team %s: %s",
|
||||
team_id,
|
||||
detail,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build / refresh
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -214,13 +244,29 @@ def _build_discord(adapter) -> List[Dict[str, str]]:
|
|||
return channels
|
||||
|
||||
|
||||
def _slack_api_error_code(error: Exception) -> Optional[str]:
|
||||
"""Return Slack Web API error code from SlackApiError-like exceptions."""
|
||||
response = getattr(error, "response", None)
|
||||
if isinstance(response, dict):
|
||||
value = response.get("error")
|
||||
return str(value) if value else None
|
||||
if response is not None:
|
||||
try:
|
||||
value = response.get("error")
|
||||
return str(value) if value else None
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
async def _build_slack(adapter) -> List[Dict[str, Any]]:
|
||||
"""List Slack channels the bot has joined across all workspaces.
|
||||
|
||||
Uses ``users.conversations`` against each workspace's web client. Pulls
|
||||
public + private channels the bot is a member of, then merges in DMs
|
||||
discovered from session history (IMs aren't useful to enumerate
|
||||
proactively).
|
||||
proactively). If the Slack app lacks channels:read, fall back to session
|
||||
history quietly instead of logging a recurring warning every refresh.
|
||||
"""
|
||||
team_clients = getattr(adapter, "_team_clients", None) or {}
|
||||
if not team_clients:
|
||||
|
|
@ -240,11 +286,15 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]:
|
|||
cursor=cursor,
|
||||
)
|
||||
if not response.get("ok"):
|
||||
logger.warning(
|
||||
"Channel directory: users.conversations not ok for team %s: %s",
|
||||
team_id,
|
||||
response.get("error", "unknown"),
|
||||
)
|
||||
error_code = response.get("error", "unknown")
|
||||
if error_code == "missing_scope":
|
||||
logger.debug(
|
||||
"Channel directory: Slack team %s lacks channels:read; using session history only",
|
||||
team_id,
|
||||
)
|
||||
else:
|
||||
detail = f"users.conversations not ok: {error_code}"
|
||||
_warn_slack_directory(team_id, detail)
|
||||
break
|
||||
for ch in response.get("channels", []):
|
||||
cid = ch.get("id")
|
||||
|
|
@ -261,17 +311,59 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]:
|
|||
if not cursor:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Channel directory: failed to list Slack channels for team %s: %s",
|
||||
team_id, e,
|
||||
)
|
||||
if _slack_api_error_code(e) == "missing_scope":
|
||||
logger.debug(
|
||||
"Channel directory: Slack team %s lacks channels:read; using session history only",
|
||||
team_id,
|
||||
)
|
||||
else:
|
||||
_warn_slack_directory(team_id, str(e))
|
||||
continue
|
||||
|
||||
# Merge in DM/group entries discovered from session history.
|
||||
# Build a lookup from API-discovered channels so we can enrich session entries.
|
||||
api_name_lookup = {ch["id"]: ch["name"] for ch in channels}
|
||||
|
||||
for entry in await asyncio.to_thread(_build_from_sessions, "slack"):
|
||||
if entry.get("id") not in seen_ids:
|
||||
eid = entry.get("id")
|
||||
if eid not in seen_ids:
|
||||
# If the entry name is still a raw Slack ID (e.g. C0xxx / D0xxx),
|
||||
# try to resolve it from the API lookup first.
|
||||
if entry.get("name", "").startswith(("C0", "D0", "G0")):
|
||||
if eid in api_name_lookup:
|
||||
entry["name"] = api_name_lookup[eid]
|
||||
channels.append(entry)
|
||||
seen_ids.add(entry.get("id"))
|
||||
seen_ids.add(eid)
|
||||
|
||||
# Resolve remaining raw-ID entries (DMs, private channels not in bot scope)
|
||||
# by calling conversations.info + users.info for each.
|
||||
unresolved = [ch for ch in channels if ch.get("name", "").startswith(("C0", "D0", "G0"))]
|
||||
if unresolved and team_clients:
|
||||
client = next(iter(team_clients.values()))
|
||||
for entry in unresolved:
|
||||
try:
|
||||
resp = await client.conversations_info(channel=entry["id"])
|
||||
if not resp.get("ok"):
|
||||
continue
|
||||
ch_info = resp.get("channel", {})
|
||||
if ch_info.get("is_im"):
|
||||
peer_user = ch_info.get("user", "")
|
||||
if peer_user:
|
||||
user_resp = await client.users_info(user=peer_user)
|
||||
if user_resp.get("ok"):
|
||||
u = user_resp["user"]
|
||||
entry["name"] = (
|
||||
u.get("profile", {}).get("display_name")
|
||||
or u.get("real_name")
|
||||
or u.get("name")
|
||||
or entry["id"]
|
||||
)
|
||||
entry["type"] = "dm"
|
||||
else:
|
||||
entry["name"] = ch_info.get("name") or ch_info.get("name_normalized") or entry["id"]
|
||||
except Exception as e:
|
||||
logger.debug("Channel directory: failed to resolve %s: %s", entry["id"], e)
|
||||
continue
|
||||
|
||||
return channels
|
||||
|
||||
|
|
|
|||
|
|
@ -789,6 +789,22 @@ class StreamingConfig:
|
|||
# platform is sufficiently configured to be considered "connected". Platforms
|
||||
# that rely on the generic ``token or api_key`` check (Telegram, Discord,
|
||||
# Slack, Matrix, Mattermost, HomeAssistant) do not need an entry here.
|
||||
def _has_usable_api_server_key(key: object) -> bool:
|
||||
"""True when API_SERVER_KEY is present and strong enough to be usable.
|
||||
|
||||
Mirrors the startup guard in ``gateway/platforms/api_server.py``
|
||||
(``has_usable_secret`` with ``min_length=16``) so the platform is only
|
||||
enrolled at load time when the adapter would actually agree to start.
|
||||
"""
|
||||
if not key:
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.auth import has_usable_secret
|
||||
except ImportError:
|
||||
return len(str(key).strip()) >= 16
|
||||
return has_usable_secret(key, min_length=16)
|
||||
|
||||
|
||||
_PLATFORM_CONNECTED_CHECKERS: dict[Platform, Callable[[PlatformConfig], bool]] = {
|
||||
Platform.WEIXIN: lambda cfg: bool(
|
||||
cfg.extra.get("account_id") and (cfg.token or cfg.extra.get("token"))
|
||||
|
|
@ -797,7 +813,9 @@ _PLATFORM_CONNECTED_CHECKERS: dict[Platform, Callable[[PlatformConfig], bool]] =
|
|||
cfg.extra.get("phone_number_id") and cfg.extra.get("access_token")
|
||||
),
|
||||
Platform.SIGNAL: lambda cfg: bool(cfg.extra.get("http_url")),
|
||||
Platform.API_SERVER: lambda cfg: True,
|
||||
Platform.API_SERVER: lambda cfg: _has_usable_api_server_key(
|
||||
cfg.extra.get("key") if cfg else None
|
||||
),
|
||||
Platform.WEBHOOK: lambda cfg: True,
|
||||
Platform.MSGRAPH_WEBHOOK: lambda cfg: bool(
|
||||
str(cfg.extra.get("client_state") or "").strip()
|
||||
|
|
@ -1386,6 +1404,41 @@ def load_gateway_config() -> GatewayConfig:
|
|||
|
||||
_merge_platform_map(gateway_platforms)
|
||||
_merge_platform_map(yaml_cfg.get("platforms"))
|
||||
|
||||
# Also merge platform configs placed directly under ``gateway.*``
|
||||
# (e.g. ``gateway.api_server``) so subsections are discovered the
|
||||
# same way ``gateway.streaming`` is handled elsewhere. Iterate
|
||||
# all ``gateway:*`` keys and merge only those that match a known
|
||||
# platform value, skipping reserved keys like ``platforms``.
|
||||
if isinstance(gateway_cfg, dict):
|
||||
_nested_platforms: dict = {}
|
||||
for _k, _v in gateway_cfg.items():
|
||||
if _k == "platforms":
|
||||
continue
|
||||
try:
|
||||
Platform(_k)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
if isinstance(_v, dict):
|
||||
_nested_platforms[_k] = _v
|
||||
if _nested_platforms:
|
||||
_merge_platform_map(_nested_platforms)
|
||||
|
||||
# Bridge api_server-specific keys (port, key, host, cors_origins,
|
||||
# model_name) into extra so PlatformConfig.from_dict preserves
|
||||
# them — adapting what _apply_env_overrides does for env vars to
|
||||
# the YAML path. Users writing ``gateway.api_server.port: 8642``
|
||||
# expect these to end up in the platform's extra dict.
|
||||
_api_plat = platforms_data.get("api_server")
|
||||
if isinstance(_api_plat, dict):
|
||||
_api_extra = _api_plat.get("extra")
|
||||
if not isinstance(_api_extra, dict):
|
||||
_api_extra = {}
|
||||
_api_plat["extra"] = _api_extra
|
||||
for _bridge_key in ("port", "key", "host", "cors_origins", "model_name"):
|
||||
if _bridge_key in _api_plat and _bridge_key not in _api_extra:
|
||||
_api_extra[_bridge_key] = _api_plat.pop(_bridge_key)
|
||||
|
||||
if platforms_data:
|
||||
gw_data["platforms"] = platforms_data
|
||||
# Iterate built-in platforms plus any registered plugin platforms
|
||||
|
|
@ -1497,6 +1550,24 @@ def load_gateway_config() -> GatewayConfig:
|
|||
bridged["typing_indicator"] = platform_cfg["typing_indicator"]
|
||||
if "typing_status_text" in platform_cfg:
|
||||
bridged["typing_status_text"] = platform_cfg["typing_status_text"]
|
||||
# Bridge top-level port/host/secret into extra for platforms
|
||||
# whose adapters read these from config.extra (webhook,
|
||||
# msgraph_webhook, api_server). Without this, YAML like:
|
||||
# platforms:
|
||||
# webhook:
|
||||
# enabled: true
|
||||
# port: 8649
|
||||
# silently falls back to the hardcoded DEFAULT_PORT because
|
||||
# PlatformConfig.from_dict only extracts ``extra`` from the
|
||||
# ``extra:`` sub-key, not from the top level.
|
||||
if plat in {Platform.WEBHOOK, Platform.MSGRAPH_WEBHOOK}:
|
||||
for _bridge_key in ("port", "host", "secret"):
|
||||
if _bridge_key in platform_cfg and _bridge_key not in platform_cfg.get("extra", {}):
|
||||
bridged[_bridge_key] = platform_cfg[_bridge_key]
|
||||
if plat == Platform.API_SERVER:
|
||||
for _bridge_key in ("port", "host"):
|
||||
if _bridge_key in platform_cfg and _bridge_key not in platform_cfg.get("extra", {}):
|
||||
bridged[_bridge_key] = platform_cfg[_bridge_key]
|
||||
has_channel_overrides = "channel_overrides" in platform_cfg
|
||||
if has_channel_overrides:
|
||||
raw_overrides = platform_cfg.get("channel_overrides")
|
||||
|
|
@ -2008,10 +2079,27 @@ def _apply_env_overrides(config: GatewayConfig) -> None:
|
|||
api_server_cors_origins = getenv("API_SERVER_CORS_ORIGINS", "")
|
||||
api_server_port = getenv("API_SERVER_PORT")
|
||||
api_server_host = getenv("API_SERVER_HOST")
|
||||
if api_server_enabled or api_server_key:
|
||||
# Require a usable key: API_SERVER_ENABLED alone would load an
|
||||
# unauthenticated platform whose adapter refuses to start at connect()
|
||||
# anyway (startup guard in gateway/platforms/api_server.py), leaving the
|
||||
# reconnect watcher spinning and logging errors forever. Same strength
|
||||
# bar as the startup guard (has_usable_secret, min_length=16).
|
||||
if _has_usable_api_server_key(api_server_key):
|
||||
if Platform.API_SERVER not in config.platforms:
|
||||
config.platforms[Platform.API_SERVER] = PlatformConfig()
|
||||
config.platforms[Platform.API_SERVER].enabled = True
|
||||
# Respect an explicit ``enabled: false`` in config.yaml (flagged by
|
||||
# ``_enabled_explicit``). In multiplex mode a secondary profile's
|
||||
# config.yaml pins ``platforms.api_server.enabled: false`` so it shares
|
||||
# the default profile's listener instead of binding its own port. That
|
||||
# profile still inherits the process-level env (including
|
||||
# ``API_SERVER_KEY``); without this guard the env-var presence would
|
||||
# force-enable the listener and trip the MultiplexConfigError check.
|
||||
# Pop (don't read) the marker — the api_server branch is terminal (no
|
||||
# later registry pass re-enables it), so this both consumes the flag and
|
||||
# avoids reading it twice, matching the pop convention used elsewhere.
|
||||
api_server_explicit = config.platforms[Platform.API_SERVER].extra.pop("_enabled_explicit", False)
|
||||
if not api_server_explicit or config.platforms[Platform.API_SERVER].enabled:
|
||||
config.platforms[Platform.API_SERVER].enabled = True
|
||||
if api_server_key:
|
||||
config.platforms[Platform.API_SERVER].extra["key"] = api_server_key
|
||||
if api_server_cors_origins:
|
||||
|
|
|
|||
|
|
@ -140,7 +140,12 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
|
|||
# Tier 2 — edit support, often customer/workspace channels
|
||||
# Slack: tool_progress off by default — Bolt posts cannot be edited like CLI;
|
||||
# "new"/"all" spam permanent lines in channels (hermes-agent#14663).
|
||||
"slack": {**_TIER_MEDIUM, "tool_progress": "off"},
|
||||
"slack": {
|
||||
**_TIER_MEDIUM,
|
||||
"tool_progress": "off",
|
||||
"long_running_notifications": False,
|
||||
"busy_ack_detail": False,
|
||||
},
|
||||
"mattermost": _TIER_MEDIUM,
|
||||
"matrix": _TIER_MEDIUM,
|
||||
"feishu": _TIER_MEDIUM,
|
||||
|
|
@ -154,6 +159,13 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
|
|||
# status update as a separate message. Promote to TIER_MEDIUM once
|
||||
# Cloud's edit_message lands.
|
||||
"whatsapp_cloud": _TIER_LOW,
|
||||
# Photon (managed iMessage over the gRPC sidecar) and BlueBubbles are both
|
||||
# permanent-message iMessage inboxes with no message-edit support, so both
|
||||
# stay TIER_LOW. This keeps tool progress, interim scratch commentary,
|
||||
# "still working" heartbeats, and busy-ack iteration detail out of the
|
||||
# user's iMessage thread. Without this entry Photon inherited the noisy
|
||||
# global ("all") defaults and compacted/narrated on nearly every turn.
|
||||
"photon": _TIER_LOW,
|
||||
"bluebubbles": _TIER_LOW,
|
||||
"weixin": _TIER_LOW,
|
||||
"wecom": _TIER_LOW,
|
||||
|
|
|
|||
|
|
@ -336,6 +336,14 @@ class GatewayKanbanWatchersMixin:
|
|||
continue
|
||||
title = (task.title if task else sub["task_id"])[:120]
|
||||
board_tag = f"[{board_slug}] " if board_slug else ""
|
||||
# Per-subscription failure-counter key. Hoisted out of the
|
||||
# event loop: the wake self-post path (in the loop's
|
||||
# ``else`` clause) needs it even when every event in the
|
||||
# claim was skipped before reaching the send site.
|
||||
sub_key = (
|
||||
sub["task_id"], sub["platform"],
|
||||
sub["chat_id"], sub.get("thread_id") or "",
|
||||
)
|
||||
for ev in d["events"]:
|
||||
kind = ev.kind
|
||||
# Identity prefix: attribute terminal pings to the
|
||||
|
|
@ -408,14 +416,49 @@ class GatewayKanbanWatchersMixin:
|
|||
metadata: dict[str, Any] = {}
|
||||
if sub.get("thread_id"):
|
||||
metadata["thread_id"] = sub["thread_id"]
|
||||
sub_key = (
|
||||
sub["task_id"], sub["platform"],
|
||||
sub["chat_id"], sub.get("thread_id") or "",
|
||||
)
|
||||
# Adapters with no push channel (the API server —
|
||||
# ``supports_async_delivery = False``) can NEVER
|
||||
# satisfy a text-send: ``send()`` always reports
|
||||
# SendResult(success=False) by design (see
|
||||
# ApiServerAdapter.send()). Treating that as a
|
||||
# delivery failure would rewind/drop the subscription
|
||||
# forever and — because the wake dispatch below lives
|
||||
# in this loop's ``else`` clause — would also make the
|
||||
# wake-on-completion path (the actual fix for the
|
||||
# api_server wrong-session bug) unreachable. So for
|
||||
# non-push adapters, skip the doomed send attempt
|
||||
# entirely: there is nothing to text-notify, the
|
||||
# creator is woken via the self-post below instead.
|
||||
from gateway.wake import adapter_supports_push
|
||||
|
||||
if not adapter_supports_push(adapter):
|
||||
logger.debug(
|
||||
"kanban notifier: adapter %s has no push "
|
||||
"channel; skipping text ping for %s, relying "
|
||||
"on wake self-post instead",
|
||||
platform_str, sub["task_id"],
|
||||
)
|
||||
# Do NOT reset the failure counter here: on this
|
||||
# path the wake self-post below IS the delivery,
|
||||
# so the counter is resolved (reset or bumped) by
|
||||
# the self-post outcome, not by skipping the send.
|
||||
continue
|
||||
try:
|
||||
await adapter.send(
|
||||
_send_res = await adapter.send(
|
||||
sub["chat_id"], msg, metadata=metadata,
|
||||
)
|
||||
# A SendResult(success=False) without an exception
|
||||
# (returned by push-capable adapters on a genuine
|
||||
# transient failure) must count as a FAILED
|
||||
# delivery — otherwise the cursor advances and the
|
||||
# event is permanently lost. Adapters returning
|
||||
# None (or anything non-SendResult shaped) keep
|
||||
# the legacy "no exception == delivered" contract.
|
||||
if getattr(_send_res, "success", True) is False:
|
||||
raise RuntimeError(
|
||||
"adapter send() reported failure: "
|
||||
f"{getattr(_send_res, 'error', None) or 'unknown error'}"
|
||||
)
|
||||
logger.debug(
|
||||
"kanban notifier: delivered %s event for %s to %s/%s on board %s",
|
||||
kind, sub["task_id"], platform_str, sub["chat_id"], board_slug,
|
||||
|
|
@ -475,12 +518,108 @@ class GatewayKanbanWatchersMixin:
|
|||
# dropping the subscription is the terminal action.
|
||||
break
|
||||
else:
|
||||
# All events delivered; advance cursor. The cursor
|
||||
# All text pings delivered (or intentionally skipped
|
||||
# for non-push adapters, whose delivery is the wake
|
||||
# self-post below). Whether the cursor may advance now
|
||||
# depends on the adapter class:
|
||||
#
|
||||
# * push-capable: the text send WAS the delivery, so
|
||||
# advance immediately (pre-existing behavior); the
|
||||
# wake injection below stays best-effort.
|
||||
# * non-push (api_server): the wake self-post IS the
|
||||
# delivery. Advancing first would let a failed /
|
||||
# retry-exhausted self-post (swallowed by the
|
||||
# best-effort except) permanently lose the event.
|
||||
# So the self-post runs FIRST and the cursor only
|
||||
# advances after it succeeds — a failure rewinds the
|
||||
# claim exactly like a failed send() above, so the
|
||||
# next tick retries.
|
||||
task_terminal = task and task.status in {"done", "archived"}
|
||||
_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked")
|
||||
_wake_kinds = {ev.kind for ev in d["events"] if ev.kind in _WAKE_KINDS}
|
||||
from gateway.wake import adapter_supports_push as _adapter_push_ok
|
||||
|
||||
_is_push_adapter = _adapter_push_ok(adapter)
|
||||
_session_key = ""
|
||||
_synth = ""
|
||||
if _wake_kinds:
|
||||
_session_key = getattr(task, "session_id", None) or ""
|
||||
if _wake_kinds and _session_key:
|
||||
_title = (task.title if task else sub["task_id"])[:120]
|
||||
_assignee = task.assignee if task else ""
|
||||
_parts = []
|
||||
if "completed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.completed"))
|
||||
if "gave_up" in _wake_kinds: _parts.append(t("gateway.kanban.wake.gave_up"))
|
||||
if "crashed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.crashed"))
|
||||
if "timed_out" in _wake_kinds: _parts.append(t("gateway.kanban.wake.timed_out"))
|
||||
if "blocked" in _wake_kinds: _parts.append(t("gateway.kanban.wake.blocked"))
|
||||
_status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
|
||||
_synth = t(
|
||||
"gateway.kanban.wake.message",
|
||||
task_id=sub["task_id"],
|
||||
status=_status,
|
||||
title=_title,
|
||||
assignee=_assignee,
|
||||
board=board_slug,
|
||||
)
|
||||
|
||||
if not _is_push_adapter and _wake_kinds and _session_key:
|
||||
# Wake self-post IS the delivery on this path —
|
||||
# it must succeed BEFORE the cursor advances.
|
||||
from gateway.wake import deliver_wake
|
||||
|
||||
try:
|
||||
await deliver_wake(
|
||||
adapter,
|
||||
text=_synth,
|
||||
session_id=_session_key,
|
||||
)
|
||||
logger.info(
|
||||
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
|
||||
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
|
||||
)
|
||||
sub_fail_counts.pop(sub_key, None)
|
||||
except Exception as _wk_err:
|
||||
fails = sub_fail_counts.get(sub_key, 0) + 1
|
||||
sub_fail_counts[sub_key] = fails
|
||||
logger.warning(
|
||||
"kanban notifier: wake self-post failed "
|
||||
"for %s (attempt %d/%d): %s",
|
||||
sub["task_id"], fails,
|
||||
MAX_SEND_FAILURES, _wk_err, exc_info=True,
|
||||
)
|
||||
if fails >= MAX_SEND_FAILURES:
|
||||
logger.warning(
|
||||
"kanban notifier: dropping subscription "
|
||||
"%s on %s after %d consecutive wake failures",
|
||||
sub["task_id"], platform_str, fails,
|
||||
)
|
||||
await asyncio.to_thread(self._kanban_unsub, sub, board_slug)
|
||||
sub_fail_counts.pop(sub_key, None)
|
||||
else:
|
||||
# Rewind the pre-send claim so the next
|
||||
# tick retries the self-post — the event
|
||||
# is NOT lost.
|
||||
await asyncio.to_thread(
|
||||
self._kanban_rewind,
|
||||
sub,
|
||||
d["cursor"],
|
||||
d.get("old_cursor", 0),
|
||||
board_slug,
|
||||
)
|
||||
continue
|
||||
|
||||
# Delivery complete (text ping for push adapters, wake
|
||||
# self-post for non-push): advance cursor. The cursor
|
||||
# is the dedup mechanism — it prevents re-delivery
|
||||
# of the same event on subsequent ticks.
|
||||
await asyncio.to_thread(
|
||||
self._kanban_advance, sub, d["cursor"], board_slug,
|
||||
)
|
||||
if not _is_push_adapter:
|
||||
# Nothing left to deliver on this path (the wake,
|
||||
# if any, already succeeded above).
|
||||
sub_fail_counts.pop(sub_key, None)
|
||||
# Unsubscribe only when the task has reached a truly
|
||||
# final status (done / archived). For blocked /
|
||||
# gave_up / crashed / timed_out the subscription is
|
||||
|
|
@ -488,68 +627,50 @@ class GatewayKanbanWatchersMixin:
|
|||
# dispatcher respawns the task and it cycles into the
|
||||
# same state. See the longer comment on TERMINAL_KINDS
|
||||
# above for the failure mode this prevents.
|
||||
task_terminal = task and task.status in {"done", "archived"}
|
||||
_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked")
|
||||
_wake_kinds = {ev.kind for ev in d["events"] if ev.kind in _WAKE_KINDS}
|
||||
if _wake_kinds:
|
||||
if _is_push_adapter and _wake_kinds and _session_key:
|
||||
try:
|
||||
_session_key = getattr(task, "session_id", None) or ""
|
||||
if _session_key:
|
||||
_title = (task.title if task else sub["task_id"])[:120]
|
||||
_assignee = task.assignee if task else ""
|
||||
_parts = []
|
||||
if "completed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.completed"))
|
||||
if "gave_up" in _wake_kinds: _parts.append(t("gateway.kanban.wake.gave_up"))
|
||||
if "crashed" in _wake_kinds: _parts.append(t("gateway.kanban.wake.crashed"))
|
||||
if "timed_out" in _wake_kinds: _parts.append(t("gateway.kanban.wake.timed_out"))
|
||||
if "blocked" in _wake_kinds: _parts.append(t("gateway.kanban.wake.blocked"))
|
||||
_status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
|
||||
_synth = t(
|
||||
"gateway.kanban.wake.message",
|
||||
task_id=sub["task_id"],
|
||||
status=_status,
|
||||
title=_title,
|
||||
assignee=_assignee,
|
||||
board=board_slug,
|
||||
)
|
||||
from gateway.session import SessionSource
|
||||
from gateway.platforms.base import MessageEvent, MessageType
|
||||
# KNOWN LIMITATION (tracked follow-up): the
|
||||
# subscription row does not persist the
|
||||
# creator's chat_type, and it is not carried
|
||||
# on the session-context bridge, so we cannot
|
||||
# faithfully reconstruct the creator's real
|
||||
# session key here. build_session_key() keys
|
||||
# DMs (":dm:<chat_id>") on a wholly different
|
||||
# shape from group/thread, so any hardcoded
|
||||
# value mis-routes some creators. "group" is
|
||||
# the least-surprising default for the
|
||||
# dashboard/group flows this wake primarily
|
||||
# serves; DM-originated creators are handled
|
||||
# by the follow-up that stamps + persists
|
||||
# chat_type end-to-end. handle_message()
|
||||
# get_or_create_session's the target, so a
|
||||
# mismatch degrades to "wake lands in a fresh
|
||||
# group session" — never an exception.
|
||||
_source = SessionSource(
|
||||
platform=plat,
|
||||
chat_id=sub["chat_id"],
|
||||
chat_type="group",
|
||||
thread_id=sub.get("thread_id") or None,
|
||||
user_id=sub.get("user_id"),
|
||||
profile=sub_profile or None,
|
||||
)
|
||||
_synth_event = MessageEvent(
|
||||
text=_synth,
|
||||
message_type=MessageType.TEXT,
|
||||
source=_source,
|
||||
internal=True,
|
||||
)
|
||||
await adapter.handle_message(_synth_event)
|
||||
logger.info(
|
||||
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
|
||||
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
|
||||
)
|
||||
from gateway.session import SessionSource
|
||||
from gateway.wake import deliver_wake
|
||||
# KNOWN LIMITATION (tracked follow-up): the
|
||||
# subscription row does not persist the
|
||||
# creator's chat_type, and it is not carried
|
||||
# on the session-context bridge, so we cannot
|
||||
# faithfully reconstruct the creator's real
|
||||
# session key here. build_session_key() keys
|
||||
# DMs (":dm:<chat_id>") on a wholly different
|
||||
# shape from group/thread, so any hardcoded
|
||||
# value mis-routes some creators. "group" is
|
||||
# the least-surprising default for the
|
||||
# dashboard/group flows this wake primarily
|
||||
# serves; DM-originated creators are handled
|
||||
# by the follow-up that stamps + persists
|
||||
# chat_type end-to-end. handle_message()
|
||||
# get_or_create_session's the target, so a
|
||||
# mismatch degrades to "wake lands in a fresh
|
||||
# group session" — never an exception.
|
||||
_source = SessionSource(
|
||||
platform=plat,
|
||||
chat_id=sub["chat_id"],
|
||||
chat_type="group",
|
||||
thread_id=sub.get("thread_id") or None,
|
||||
user_id=sub.get("user_id"),
|
||||
profile=sub_profile or None,
|
||||
)
|
||||
# deliver_wake preserves the synthetic
|
||||
# MessageEvent/handle_message path for
|
||||
# push-capable adapters (the non-push /
|
||||
# self-post branch is handled BEFORE the
|
||||
# cursor advance above).
|
||||
await deliver_wake(
|
||||
adapter,
|
||||
text=_synth,
|
||||
session_id=_session_key,
|
||||
source=_source,
|
||||
)
|
||||
logger.info(
|
||||
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
|
||||
sub["task_id"], platform_str, sub["chat_id"], sub_profile or "default", _wake_kinds,
|
||||
)
|
||||
except Exception as _wk_err:
|
||||
# Best-effort: the notification itself already
|
||||
# delivered and the cursor has advanced, so a
|
||||
|
|
|
|||
|
|
@ -1335,7 +1335,7 @@ class APIServerAdapter(BasePlatformAdapter):
|
|||
self._request_audit_log_suffix(request),
|
||||
)
|
||||
return web.json_response(
|
||||
{"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": "invalid_api_key"}},
|
||||
{"error": {"message": "Invalid gateway API key (API_SERVER_KEY)", "type": "gateway_auth_error", "code": "gateway_auth_failed"}},
|
||||
status=401,
|
||||
)
|
||||
|
||||
|
|
@ -5101,7 +5101,17 @@ class APIServerAdapter(BasePlatformAdapter):
|
|||
# environment state.
|
||||
approval_token = set_current_session_key(approval_session_key)
|
||||
session_tokens = self._bind_api_server_session(
|
||||
# chat_id carries the raw session id (the
|
||||
# X-Hermes-Session-Id equivalent) exactly like
|
||||
# the other agent-entry routes bind it via
|
||||
# _run_agent(). Without it,
|
||||
# tools.async_delegation reads an empty
|
||||
# HERMES_SESSION_CHAT_ID on /v1/runs and
|
||||
# background delegations stay forced-sync
|
||||
# (no wake target).
|
||||
chat_id=session_id or "",
|
||||
session_key=approval_session_key,
|
||||
session_id=session_id or "",
|
||||
)
|
||||
register_gateway_notify(approval_session_key, _approval_notify)
|
||||
r = agent.run_conversation(
|
||||
|
|
@ -5491,19 +5501,32 @@ class APIServerAdapter(BasePlatformAdapter):
|
|||
|
||||
try:
|
||||
from hermes_cli.auth import has_usable_secret
|
||||
if not has_usable_secret(self._api_key, min_length=16):
|
||||
logger.error(
|
||||
"[%s] Refusing to start: API_SERVER_KEY is a "
|
||||
"placeholder or too short (<16 chars). This endpoint "
|
||||
"dispatches terminal-capable agent work — a guessable "
|
||||
"key is remote code execution. Generate a strong secret "
|
||||
"(e.g. `openssl rand -hex 32`) and set API_SERVER_KEY "
|
||||
"before starting the API server on %s.",
|
||||
self.name, self._host,
|
||||
)
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
# Fail CLOSED. This guard is the only thing between a guessable
|
||||
# key and a terminal-capable endpoint, so "the check could not be
|
||||
# run" must not resolve to "start anyway" — the same posture
|
||||
# tools/credential_files.py takes when its deny-list cannot be
|
||||
# consulted.
|
||||
logger.error(
|
||||
"[%s] Refusing to start: API_SERVER_KEY strength could not be "
|
||||
"verified (%s: %s), and this endpoint dispatches "
|
||||
"terminal-capable agent work. Repair the installation before "
|
||||
"starting the API server on %s.",
|
||||
self.name, type(exc).__name__, exc, self._host,
|
||||
)
|
||||
return False
|
||||
|
||||
if not has_usable_secret(self._api_key, min_length=16):
|
||||
logger.error(
|
||||
"[%s] Refusing to start: API_SERVER_KEY is a "
|
||||
"placeholder or too short (<16 chars). This endpoint "
|
||||
"dispatches terminal-capable agent work — a guessable "
|
||||
"key is remote code execution. Generate a strong secret "
|
||||
"(e.g. `openssl rand -hex 32`) and set API_SERVER_KEY "
|
||||
"before starting the API server on %s.",
|
||||
self.name, self._host,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def connect(self, *, is_reconnect: bool = False) -> bool:
|
||||
|
|
@ -5513,6 +5536,26 @@ class APIServerAdapter(BasePlatformAdapter):
|
|||
return False
|
||||
|
||||
if not self._api_key_passes_startup_guard():
|
||||
# A rejected API_SERVER_KEY is a configuration error, not a
|
||||
# transient blip — the key will not become valid on its own. A
|
||||
# bare ``return False`` makes the reconnect watcher in
|
||||
# gateway.run treat it as retryable and loop forever at the
|
||||
# backoff cap, re-instantiating the adapter (and its
|
||||
# ResponseStore sqlite connection) every retry (#38803: ~501
|
||||
# leaked connections / 1002 fds over 2.5 days until EMFILE took
|
||||
# the whole gateway down). Non-retryable drops it from the
|
||||
# reconnect queue — same treatment as the port-conflict guard
|
||||
# (api_server_port_in_use). The guard already logged the
|
||||
# specific rejection reason just above.
|
||||
self._set_fatal_error(
|
||||
"api_server_key_invalid",
|
||||
"API_SERVER_KEY was rejected by the startup guard (missing, "
|
||||
"placeholder/too short, or strength unverifiable — see the "
|
||||
"error logged above). Generate a strong secret (e.g. "
|
||||
"`openssl rand -hex 32`), set API_SERVER_KEY, then "
|
||||
"`/platform resume api_server`.",
|
||||
retryable=False,
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -105,6 +105,18 @@ def _reply_anchor_for_event(event) -> str | None:
|
|||
source = getattr(event, "source", None)
|
||||
platform = _platform_name(getattr(source, "platform", None))
|
||||
thread_id = getattr(source, "thread_id", None)
|
||||
raw_message = getattr(event, "raw_message", None)
|
||||
if (
|
||||
platform == "slack"
|
||||
and isinstance(raw_message, dict)
|
||||
and raw_message.get("_hermes_no_thread_response")
|
||||
):
|
||||
# Slack reaction handoffs into a configured target channel are meant
|
||||
# to create a new top-level message there. Returning the synthetic
|
||||
# event's message_id as reply_to would make
|
||||
# SlackAdapter._resolve_thread_ts() treat it as a thread anchor and
|
||||
# reply in a (nonexistent) thread anyway.
|
||||
return None
|
||||
if platform == "telegram" and thread_id and getattr(source, "chat_type", None) == "dm":
|
||||
# Reply to the triggering user message. Replying to Telegram's earlier
|
||||
# topic seed/anchor can render the bot response outside the active lane.
|
||||
|
|
@ -749,14 +761,14 @@ async def cache_image_from_url(url: str, ext: str = ".jpg", retries: int = 2) ->
|
|||
Raises:
|
||||
ValueError: If the URL targets a private/internal network (SSRF protection).
|
||||
"""
|
||||
from tools.url_safety import is_safe_url
|
||||
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
|
||||
if not is_safe_url(url):
|
||||
raise ValueError(f"Blocked unsafe URL (SSRF protection): {safe_url_for_log(url)}")
|
||||
|
||||
import httpx
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
async with create_ssrf_safe_async_client(
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_ssrf_redirect_guard]},
|
||||
|
|
@ -869,14 +881,14 @@ async def cache_audio_from_url(url: str, ext: str = ".ogg", retries: int = 2) ->
|
|||
Raises:
|
||||
ValueError: If the URL targets a private/internal network (SSRF protection).
|
||||
"""
|
||||
from tools.url_safety import is_safe_url
|
||||
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
|
||||
if not is_safe_url(url):
|
||||
raise ValueError(f"Blocked unsafe URL (SSRF protection): {safe_url_for_log(url)}")
|
||||
|
||||
import httpx
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
async with create_ssrf_safe_async_client(
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_ssrf_redirect_guard]},
|
||||
|
|
@ -2443,6 +2455,11 @@ class BasePlatformAdapter(ABC):
|
|||
self.config = config
|
||||
self.platform = platform
|
||||
self._message_handler: Optional[MessageHandler] = None
|
||||
# Optional gateway-supplied fan-out for platform-native emoji
|
||||
# reaction events (see ``set_reaction_handler``).
|
||||
self._reaction_handler: Optional[
|
||||
Callable[[Dict[str, Any]], Awaitable[None]]
|
||||
] = None
|
||||
# Optional hook (e.g. Telegram DM topic recovery) that rewrites
|
||||
# ``event.source.thread_id`` before session keying. Returns the
|
||||
# corrected thread_id or None to leave the source untouched.
|
||||
|
|
@ -2988,6 +3005,25 @@ class BasePlatformAdapter(ABC):
|
|||
"""Set an optional handler for messages arriving during active sessions."""
|
||||
self._busy_session_handler = handler
|
||||
|
||||
def set_reaction_handler(
|
||||
self, handler: Optional[Callable[[Dict[str, Any]], Awaitable[None]]]
|
||||
) -> None:
|
||||
"""Set the handler for emoji-reaction events on platform messages.
|
||||
|
||||
Called by adapters that subscribe to platform-native reaction events
|
||||
(currently the Slack adapter's ``reaction_added``/``reaction_removed``).
|
||||
The handler receives a normalised event dict — ``platform``,
|
||||
``event_name`` ("reaction:added"/"reaction:removed"), ``reaction``,
|
||||
``user_id``, ``item_user_id``, ``channel_id``, ``message_ts``,
|
||||
``event_ts``, ``raw_event`` — and fans out via
|
||||
``HookRegistry.emit(event_name, ...)``.
|
||||
|
||||
Adapters without reaction support simply never call the handler.
|
||||
"""
|
||||
# Assign defensively: subclasses initialized via ``object.__new__``
|
||||
# in tests never run ``BasePlatformAdapter.__init__``.
|
||||
self._reaction_handler = handler # type: ignore[attr-defined]
|
||||
|
||||
def set_authorization_check(
|
||||
self,
|
||||
callback: Optional[Callable[[str, Optional[str], Optional[str]], bool]],
|
||||
|
|
@ -5781,6 +5817,7 @@ class BasePlatformAdapter(ABC):
|
|||
user_id_alt=user_id_alt,
|
||||
chat_id_alt=chat_id_alt,
|
||||
is_bot=is_bot,
|
||||
scope_id=str(scope_id) if scope_id else None,
|
||||
guild_id=str(guild_id) if guild_id else None,
|
||||
parent_chat_id=str(parent_chat_id) if parent_chat_id else None,
|
||||
message_id=str(message_id) if message_id else None,
|
||||
|
|
@ -5895,7 +5932,26 @@ class BasePlatformAdapter(ABC):
|
|||
|
||||
# Everything remaining fits in one final chunk
|
||||
if _len(prefix) + _len(remaining) <= max_length - INDICATOR_RESERVE:
|
||||
chunks.append(prefix + remaining)
|
||||
final_chunk = prefix + remaining
|
||||
# Check fence balance: if carry_lang was set, the chunk
|
||||
# starts with an opening fence. Walk the remaining text
|
||||
# to see if the code block was closed; if not, close it.
|
||||
_final_in_code = carry_lang is not None
|
||||
_final_lang = carry_lang or ""
|
||||
if _final_in_code:
|
||||
for _line in remaining.split("\n"):
|
||||
_stripped = _line.strip()
|
||||
if _stripped.startswith("```"):
|
||||
if _final_in_code:
|
||||
_final_in_code = False
|
||||
_final_lang = ""
|
||||
else:
|
||||
_final_in_code = True
|
||||
_tag = _stripped[3:].strip()
|
||||
_final_lang = _tag.split()[0] if _tag else ""
|
||||
if _final_in_code:
|
||||
final_chunk += FENCE_CLOSE
|
||||
chunks.append(final_chunk)
|
||||
break
|
||||
|
||||
# Find a natural split point (prefer newlines, then spaces).
|
||||
|
|
|
|||
|
|
@ -312,7 +312,8 @@ class QQAdapter(BasePlatformAdapter):
|
|||
# Tighter keepalive pool so idle CLOSE_WAIT sockets drain
|
||||
# faster behind proxies like Cloudflare Warp (#18451).
|
||||
from gateway.platforms._http_client_limits import platform_httpx_limits
|
||||
self._http_client = httpx.AsyncClient(
|
||||
from tools.url_safety import create_ssrf_safe_async_client
|
||||
self._http_client = create_ssrf_safe_async_client(
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_ssrf_redirect_guard]},
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ def _create_bind_task(timeout: float = ONBOARD_API_TIMEOUT) -> Tuple[str, str]:
|
|||
if data.get("retcode") != 0:
|
||||
raise RuntimeError(data.get("msg", "create_bind_task failed"))
|
||||
|
||||
task_id = data.get("data", {}).get("task_id")
|
||||
task_id = (data.get("data") or {}).get("task_id")
|
||||
if not task_id:
|
||||
raise RuntimeError("create_bind_task: missing task_id in response")
|
||||
|
||||
|
|
|
|||
|
|
@ -2153,7 +2153,7 @@ class GroupAtGuardMiddleware(InboundMiddleware):
|
|||
for elem in msg_body:
|
||||
if elem.get("msg_type") != "TIMCustomElem":
|
||||
continue
|
||||
data_str = elem.get("msg_content", {}).get("data", "")
|
||||
data_str = (elem.get("msg_content") or {}).get("data", "")
|
||||
if not data_str:
|
||||
continue
|
||||
try:
|
||||
|
|
@ -2172,7 +2172,7 @@ class GroupAtGuardMiddleware(InboundMiddleware):
|
|||
for elem in msg_body:
|
||||
if elem.get("msg_type") != "TIMCustomElem":
|
||||
continue
|
||||
data_str = elem.get("msg_content", {}).get("data", "")
|
||||
data_str = (elem.get("msg_content") or {}).get("data", "")
|
||||
if not data_str:
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ async def download_url(
|
|||
# SSRF protection: yuanbao downloads model-supplied and inbound URLs
|
||||
# server-side. Reject private/internal targets up front, and re-validate
|
||||
# every redirect hop so a public URL can't 302 to http://169.254.169.254/.
|
||||
from tools.url_safety import is_safe_url
|
||||
from tools.url_safety import create_ssrf_safe_async_client, is_safe_url
|
||||
|
||||
if not is_safe_url(url):
|
||||
raise ValueError(f"Blocked unsafe URL (SSRF protection): {url}")
|
||||
|
|
@ -234,7 +234,7 @@ async def download_url(
|
|||
)
|
||||
|
||||
max_bytes = max_size_mb * 1024 * 1024
|
||||
async with httpx.AsyncClient(
|
||||
async with create_ssrf_safe_async_client(
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_redirect_guard]},
|
||||
|
|
|
|||
468
gateway/run.py
468
gateway/run.py
|
|
@ -529,10 +529,31 @@ async def _send_or_update_status_coro(adapter, chat_id, status_key, content, met
|
|||
return await adapter.send(chat_id, content, metadata=metadata)
|
||||
|
||||
|
||||
def _resolve_progress_thread_id(platform: Any, source_thread_id: Any, event_message_id: Any) -> Optional[str]:
|
||||
"""Return thread/root ID that progress/status bubbles should target."""
|
||||
def _resolve_progress_thread_id(
|
||||
platform: Any,
|
||||
source_thread_id: Any,
|
||||
event_message_id: Any,
|
||||
*,
|
||||
reply_in_thread: bool = True,
|
||||
) -> Optional[str]:
|
||||
"""Return thread/root ID that progress/status bubbles should target.
|
||||
|
||||
``reply_in_thread=False`` (Slack ``platforms.slack.extra.reply_in_thread``)
|
||||
disables the synthetic-thread fallback: progress messages must not create
|
||||
a thread the final flat reply would then inherit. A source.thread_id equal
|
||||
to the event's own message id is the adapter's synthetic session-keying
|
||||
thread, not a real thread — treat it as "no thread" too (#18859).
|
||||
"""
|
||||
platform_value = getattr(platform, "value", platform)
|
||||
platform_key = str(platform_value or "").lower()
|
||||
if not reply_in_thread:
|
||||
if (
|
||||
source_thread_id
|
||||
and event_message_id
|
||||
and str(source_thread_id) == str(event_message_id)
|
||||
):
|
||||
return None
|
||||
return str(source_thread_id) if source_thread_id else None
|
||||
if source_thread_id:
|
||||
return str(source_thread_id)
|
||||
if platform_key in {"slack", "mattermost"} and event_message_id:
|
||||
|
|
@ -920,6 +941,53 @@ def _uses_telegram_observed_group_context(channel_prompt: Optional[str]) -> bool
|
|||
return bool(channel_prompt and _TELEGRAM_OBSERVED_CONTEXT_PROMPT_MARKER in channel_prompt)
|
||||
|
||||
|
||||
def _csv_or_list_to_set(raw: Any) -> set[str]:
|
||||
"""Normalize a config list or comma-separated scalar into a string set."""
|
||||
if raw is None:
|
||||
return set()
|
||||
if isinstance(raw, list):
|
||||
return {str(part).strip() for part in raw if str(part).strip()}
|
||||
s = str(raw).strip()
|
||||
if not s:
|
||||
return set()
|
||||
return {part.strip() for part in s.split(",") if part.strip()}
|
||||
|
||||
|
||||
def _slack_ignored_channels_from_gateway_config(config: Any) -> set[str]:
|
||||
"""Return Slack channels that the generic gateway must never dispatch.
|
||||
|
||||
The Slack adapter has the first-line drop, but this runner-level guard is
|
||||
intentionally duplicated as a fail-safe. If a future Slack code path, test
|
||||
hook, malformed event, or stale adapter instance bypasses the Slack plugin
|
||||
adapter, ignored channels still cannot reach auth, pairing, sessions, or
|
||||
the agent/home-channel prompt pipeline.
|
||||
"""
|
||||
platform_cfg = getattr(config, "platforms", {}).get(Platform.SLACK)
|
||||
raw = None
|
||||
if platform_cfg is not None:
|
||||
raw = getattr(platform_cfg, "extra", {}).get("ignored_channels")
|
||||
if raw is None:
|
||||
# Top-level ``slack.ignored_channels`` config flows through the
|
||||
# plugin's YAML→env bridge (SLACK_IGNORED_CHANNELS) rather than
|
||||
# PlatformConfig.extra — honor it here too (#46925).
|
||||
raw = os.getenv("SLACK_IGNORED_CHANNELS") or None
|
||||
return _csv_or_list_to_set(raw)
|
||||
|
||||
|
||||
def _slack_parent_channel_id(chat_id: Any) -> str:
|
||||
"""Return the parent Slack channel from a possibly thread-scoped chat ID."""
|
||||
if not chat_id:
|
||||
return ""
|
||||
return str(chat_id).split(":", 1)[0]
|
||||
|
||||
|
||||
def _is_slack_ignored_channel(config: Any, chat_id: Any) -> bool:
|
||||
"""Check the generic Slack gateway blacklist for channel or thread IDs."""
|
||||
channel_id = _slack_parent_channel_id(chat_id)
|
||||
ignored = _slack_ignored_channels_from_gateway_config(config)
|
||||
return bool(channel_id and ("*" in ignored or channel_id in ignored))
|
||||
|
||||
|
||||
def _message_timestamps_enabled(user_config: Optional[dict]) -> bool:
|
||||
"""True when gateway.message_timestamps.enabled is opted in.
|
||||
|
||||
|
|
@ -1951,6 +2019,7 @@ from gateway.session import (
|
|||
SessionContext,
|
||||
build_session_context,
|
||||
build_session_context_prompt,
|
||||
build_channel_continuity_note,
|
||||
build_session_key,
|
||||
is_shared_multi_user_session,
|
||||
neutralize_untrusted_inline_text,
|
||||
|
|
@ -4416,6 +4485,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
except Exception:
|
||||
logger.debug("Failed to sync gateway session model metadata", exc_info=True)
|
||||
|
||||
async def _handle_reaction_event(self, ctx: Dict[str, Any]) -> None:
|
||||
"""Fan a normalised platform reaction event out to the HookRegistry.
|
||||
|
||||
Adapters call this via ``set_reaction_handler`` for every
|
||||
platform-native reaction event they surface. The adapter-supplied
|
||||
``event_name`` ("reaction:added" / "reaction:removed") becomes the
|
||||
hook event so user hooks subscribe with the same name scheme as the
|
||||
existing ``agent:*`` family. Errors never block the adapter's event
|
||||
loop — the hook contract is non-blocking.
|
||||
"""
|
||||
event_name = str(ctx.get("event_name") or "reaction:added")
|
||||
try:
|
||||
await self.hooks.emit(event_name, ctx)
|
||||
except Exception:
|
||||
logger.debug("[Gateway] reaction hook emit failed", exc_info=True)
|
||||
|
||||
async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None:
|
||||
"""React to an adapter failure after startup.
|
||||
|
||||
|
|
@ -6643,6 +6728,44 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# loop is never blocked; mirrors the /new reset path's fix (#35994).
|
||||
_CLEANUP_TIMEOUT_S = 30.0
|
||||
|
||||
def _defer_agent_cleanup_until_future_done(
|
||||
self,
|
||||
future: asyncio.Future,
|
||||
agent: Any,
|
||||
*,
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Clean up ``agent`` only after its executor future has finished.
|
||||
|
||||
A timed-out executor call keeps running in its worker thread. Closing
|
||||
the agent before that thread exits can tear down clients or providers
|
||||
it is still using. Keep a strong task reference and wait for the real
|
||||
future before invoking the normal bounded, off-loop cleanup path.
|
||||
"""
|
||||
|
||||
async def _cleanup_when_done() -> None:
|
||||
try:
|
||||
await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
# Loop shutdown can cancel this waiter while the executor still
|
||||
# runs. Never turn that cancellation into premature cleanup.
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Deferred agent worker%s finished with an error: %s",
|
||||
f" ({context})" if context else "",
|
||||
exc,
|
||||
)
|
||||
await self._cleanup_agent_resources_off_loop(agent, context=context)
|
||||
|
||||
task = asyncio.create_task(_cleanup_when_done())
|
||||
tasks = getattr(self, "_deferred_agent_cleanup_tasks", None)
|
||||
if tasks is None:
|
||||
tasks = set()
|
||||
self._deferred_agent_cleanup_tasks = tasks
|
||||
tasks.add(task)
|
||||
task.add_done_callback(tasks.discard)
|
||||
|
||||
async def _cleanup_agent_resources_off_loop(
|
||||
self, agent: Any, *, context: str = ""
|
||||
) -> None:
|
||||
|
|
@ -6903,19 +7026,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# run as a self-restart loop guard and the gateway stays stopped.
|
||||
watcher_env.pop("_HERMES_GATEWAY", None)
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
# The watcher runs sys.executable (console python) under the
|
||||
# CREATE_NO_WINDOW detach kwargs below: it owns one hidden
|
||||
# console, inherited by the `hermes gateway restart` child, so
|
||||
# nothing flashes. Do NOT swap in GUI-subsystem pythonw.exe —
|
||||
# a console-less watcher forces every console-subsystem
|
||||
# descendant to allocate a visible conhost (#54220/#56747).
|
||||
watcher_python = sys.executable
|
||||
try:
|
||||
# Prefer a real GUI-subsystem interpreter for the watcher
|
||||
# itself. With uv venvs, ``python.exe`` can re-exec the base
|
||||
# console interpreter and flash even when the Popen carries
|
||||
# CREATE_NO_WINDOW; pythonw.exe avoids console allocation.
|
||||
from hermes_cli.gateway_windows import _resolve_detached_python
|
||||
|
||||
watcher_python, _watcher_venv_dir, _watcher_site_packages = (
|
||||
_resolve_detached_python(sys.executable)
|
||||
)
|
||||
except Exception:
|
||||
watcher_python = sys.executable
|
||||
venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv")
|
||||
site_packages = venv_dir / "Lib" / "site-packages"
|
||||
if site_packages.exists():
|
||||
|
|
@ -7837,6 +7954,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
adapter.set_fatal_error_handler(self._handle_adapter_fatal_error)
|
||||
adapter.set_session_store(self.session_store)
|
||||
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
|
||||
_set_reaction = getattr(adapter, "set_reaction_handler", None)
|
||||
if callable(_set_reaction):
|
||||
_set_reaction(self._handle_reaction_event)
|
||||
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
|
||||
adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform))
|
||||
adapter._busy_text_mode = self._busy_text_mode
|
||||
|
|
@ -8815,6 +8935,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
adapter.set_fatal_error_handler(self._handle_adapter_fatal_error)
|
||||
adapter.set_session_store(self.session_store)
|
||||
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
|
||||
_set_reaction = getattr(adapter, "set_reaction_handler", None)
|
||||
if callable(_set_reaction):
|
||||
_set_reaction(self._handle_reaction_event)
|
||||
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
|
||||
adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform))
|
||||
adapter._busy_text_mode = self._busy_text_mode
|
||||
|
|
@ -9689,6 +9812,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
)
|
||||
adapter.set_session_store(self.session_store)
|
||||
adapter.set_busy_session_handler(self._handle_active_session_busy_message)
|
||||
_set_reaction = getattr(adapter, "set_reaction_handler", None)
|
||||
if callable(_set_reaction):
|
||||
_set_reaction(self._handle_reaction_event)
|
||||
adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
|
||||
adapter.set_authorization_check(
|
||||
self._make_adapter_auth_check(platform, profile_name=profile_name)
|
||||
|
|
@ -10110,6 +10236,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
return
|
||||
|
||||
config = getattr(self, "config", None)
|
||||
if (
|
||||
config
|
||||
and getattr(source, "platform", None) == Platform.SLACK
|
||||
and _is_slack_ignored_channel(config, getattr(source, "chat_id", None))
|
||||
):
|
||||
logger.info(
|
||||
"Skipping Slack platform notice for configured ignored channel %s",
|
||||
getattr(source, "chat_id", None),
|
||||
)
|
||||
return
|
||||
|
||||
notice_delivery = "public"
|
||||
if config and hasattr(config, "get_notice_delivery"):
|
||||
notice_delivery = config.get_notice_delivery(source.platform)
|
||||
|
|
@ -10316,18 +10453,37 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
except Exception:
|
||||
logger.debug("reset_session_vars failed at handler entry", exc_info=True)
|
||||
|
||||
# Internal events (e.g. background-process completion notifications)
|
||||
# are system-generated and must skip user authorization.
|
||||
is_internal = bool(getattr(event, "internal", False))
|
||||
|
||||
# Ignored-channel guard runs FIRST — before startup-restore queueing,
|
||||
# plugin hooks, auth, and session setup — so a configured ignored
|
||||
# channel can never reach pairing/auth/session state (#51899).
|
||||
# getattr: bare test runners construct GatewayRunner via
|
||||
# object.__new__ without config (see AGENTS.md pitfall on
|
||||
# object.__new__ test pattern).
|
||||
if (
|
||||
not is_internal
|
||||
and getattr(source, "platform", None) == Platform.SLACK
|
||||
and _is_slack_ignored_channel(
|
||||
getattr(self, "config", None), getattr(source, "chat_id", None)
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Dropping Slack message from configured ignored channel %s",
|
||||
getattr(source, "chat_id", None),
|
||||
)
|
||||
return None
|
||||
|
||||
if (
|
||||
getattr(self, "_startup_restore_in_progress", False)
|
||||
and not getattr(event, "internal", False)
|
||||
and not is_internal
|
||||
and not getattr(event, "_hermes_startup_restore_replay", False)
|
||||
):
|
||||
self._queue_startup_restore_event(event)
|
||||
return None
|
||||
|
||||
# Internal events (e.g. background-process completion notifications)
|
||||
# are system-generated and must skip user authorization.
|
||||
is_internal = bool(getattr(event, "internal", False))
|
||||
|
||||
# scale-to-zero (Phase 0, 0.B/F13): stamp the gateway-scoped last-inbound
|
||||
# clock for real (user-originated) inbound only. Internal/system events
|
||||
# (background-process completions, startup-restore replays) are NOT
|
||||
|
|
@ -12532,6 +12688,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
context_note = "[System note: The previous gateway session could not be recovered after a restart (API recovery timed out). This is a fresh conversation — use /resume to restore history if needed.]"
|
||||
else:
|
||||
context_note = "[System note: The user's previous session expired due to inactivity. This is a fresh conversation with no prior context.]"
|
||||
# Slack/Discord channels/threads are long-lived: point the agent at
|
||||
# the specific prior same-channel session so it recalls that context
|
||||
# via session_search instead of an unrelated recent session. Returns
|
||||
# None (appends nothing) for other platforms or when there's no prior
|
||||
# activity to recall. Deterministic — no extra API/DB calls (#36220).
|
||||
try:
|
||||
continuity_note = build_channel_continuity_note(session_entry, source)
|
||||
except Exception:
|
||||
continuity_note = None
|
||||
if continuity_note:
|
||||
context_note = context_note + "\n\n" + continuity_note
|
||||
turn_sidecar_notes.append(context_note)
|
||||
|
||||
# Send a user-facing notification explaining the reset, unless:
|
||||
|
|
@ -12692,6 +12859,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_threshold_pct = 0.85
|
||||
_hyg_compression_enabled = True
|
||||
_hyg_hard_msg_limit = 5000
|
||||
_hyg_timeout_seconds = 30.0
|
||||
_hyg_failure_cooldown_seconds = 300.0
|
||||
_hyg_config_context_length = None
|
||||
_hyg_provider = None
|
||||
_hyg_base_url = None
|
||||
|
|
@ -12737,6 +12906,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_hard_msg_limit = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
_raw_timeout = _comp_cfg.get("hygiene_timeout_seconds")
|
||||
if _raw_timeout is not None:
|
||||
try:
|
||||
_parsed = float(_raw_timeout)
|
||||
if _parsed > 0:
|
||||
_hyg_timeout_seconds = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
_raw_cooldown = _comp_cfg.get("hygiene_failure_cooldown_seconds")
|
||||
if _raw_cooldown is not None:
|
||||
try:
|
||||
_parsed = float(_raw_cooldown)
|
||||
if _parsed >= 0:
|
||||
_hyg_failure_cooldown_seconds = _parsed
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
_hyg_configured_model = _hyg_model
|
||||
_hyg_configured_provider = _hyg_provider
|
||||
|
|
@ -12847,6 +13032,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
or _msg_count >= _HARD_MSG_LIMIT
|
||||
)
|
||||
|
||||
if _needs_compress:
|
||||
_cooldowns = getattr(self, "_hygiene_compression_failure_cooldowns", None)
|
||||
if _cooldowns is None:
|
||||
_cooldowns = {}
|
||||
self._hygiene_compression_failure_cooldowns = _cooldowns
|
||||
_cooldown_key = session_entry.session_id
|
||||
_cooldown_until = float(_cooldowns.get(_cooldown_key) or 0.0)
|
||||
if _cooldown_until > time.time():
|
||||
logger.info(
|
||||
"Session hygiene: skipping compression for %s; "
|
||||
"previous failure cooldown active for %.1fs",
|
||||
_cooldown_key,
|
||||
max(0.0, _cooldown_until - time.time()),
|
||||
)
|
||||
_needs_compress = False
|
||||
|
||||
if _needs_compress:
|
||||
logger.info(
|
||||
"Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing "
|
||||
|
|
@ -12860,6 +13061,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
|
||||
|
||||
try:
|
||||
from agent.conversation_compression import CompressionCommitFence
|
||||
from run_agent import AIAgent
|
||||
|
||||
_hyg_model, _hyg_runtime = self._resolve_session_agent_runtime(
|
||||
|
|
@ -12894,6 +13096,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
session_id=session_entry.session_id,
|
||||
session_db=_hyg_session_db,
|
||||
)
|
||||
_hyg_cleanup_deferred = False
|
||||
try:
|
||||
# Gateway hygiene runs before the user turn
|
||||
# starts and already owns the session binding.
|
||||
|
|
@ -12921,13 +13124,76 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
_hyg_agent._print_fn = lambda *a, **kw: None
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
_compressed, _ = await loop.run_in_executor(
|
||||
_hyg_commit_fence = CompressionCommitFence()
|
||||
_hyg_future = loop.run_in_executor(
|
||||
None,
|
||||
lambda: _hyg_agent._compress_context(
|
||||
_hyg_msgs, "",
|
||||
approx_tokens=_approx_tokens,
|
||||
commit_fence=_hyg_commit_fence,
|
||||
),
|
||||
)
|
||||
try:
|
||||
_compressed, _ = await asyncio.wait_for(
|
||||
asyncio.shield(_hyg_future),
|
||||
timeout=_hyg_timeout_seconds,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
_cancelled = None
|
||||
while _cancelled is None:
|
||||
_cancelled = (
|
||||
_hyg_commit_fence.try_cancel_before_commit()
|
||||
)
|
||||
if _cancelled is None:
|
||||
await asyncio.sleep(0.001)
|
||||
if not _cancelled:
|
||||
# The worker crossed the commit boundary just
|
||||
# before the timeout. The fence poll waited for
|
||||
# that boundary to finish, so consume the
|
||||
# completed result instead of treating a
|
||||
# successful compaction as a timeout.
|
||||
_compressed, _ = await _hyg_future
|
||||
else:
|
||||
self._defer_agent_cleanup_until_future_done(
|
||||
_hyg_future,
|
||||
_hyg_agent,
|
||||
context="session hygiene timeout",
|
||||
)
|
||||
_hyg_cleanup_deferred = True
|
||||
if _hyg_failure_cooldown_seconds >= 0:
|
||||
self._hygiene_compression_failure_cooldowns[
|
||||
session_entry.session_id
|
||||
] = time.time() + _hyg_failure_cooldown_seconds
|
||||
logger.warning(
|
||||
"Session hygiene compression for session %s "
|
||||
"timed out after %.1fs; continuing without "
|
||||
"compression",
|
||||
session_entry.session_id,
|
||||
_hyg_timeout_seconds,
|
||||
)
|
||||
_timeout_msg = (
|
||||
"⚠️ Context compression timed out "
|
||||
f"after {_hyg_timeout_seconds:.1f}s. "
|
||||
"No messages were dropped — continuing without "
|
||||
"compression. Run /compress to retry, /reset for "
|
||||
"a clean session, or check your "
|
||||
"auxiliary.compression model configuration."
|
||||
)
|
||||
try:
|
||||
_adapter = self._adapter_for_source(source)
|
||||
if _adapter and source.chat_id:
|
||||
await _adapter.send(
|
||||
source.chat_id,
|
||||
_timeout_msg,
|
||||
metadata=_hyg_meta,
|
||||
)
|
||||
except Exception as _werr:
|
||||
logger.warning(
|
||||
"Failed to deliver compression-timeout "
|
||||
"warning to user: %s",
|
||||
_werr,
|
||||
)
|
||||
raise
|
||||
|
||||
# _compress_context ends the old session and creates
|
||||
# a new session_id. Write compressed messages into
|
||||
|
|
@ -13035,6 +13301,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# fresh.
|
||||
_comp = getattr(_hyg_agent, "context_compressor", None)
|
||||
if _comp is not None and getattr(_comp, "_last_compress_aborted", False):
|
||||
if _hyg_failure_cooldown_seconds >= 0:
|
||||
self._hygiene_compression_failure_cooldowns[
|
||||
session_entry.session_id
|
||||
] = time.time() + _hyg_failure_cooldown_seconds
|
||||
_err = getattr(_comp, "_last_summary_error", None) or "unknown error"
|
||||
# Force-redact: provider exception text
|
||||
# may contain credentials; this message
|
||||
|
|
@ -13087,9 +13357,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# rebuilds its system prompt from current
|
||||
# SOUL.md, memory, and skills.
|
||||
self._evict_cached_agent(session_key)
|
||||
await self._cleanup_agent_resources_off_loop(
|
||||
_hyg_agent, context="session hygiene"
|
||||
)
|
||||
if not _hyg_cleanup_deferred:
|
||||
await self._cleanup_agent_resources_off_loop(
|
||||
_hyg_agent, context="session hygiene"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
|
|
@ -13475,6 +13746,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
if _show_reasoning_effective and response and not _intentional_silence:
|
||||
last_reasoning = agent_result.get("last_reasoning")
|
||||
if last_reasoning:
|
||||
from gateway.stream_consumer import escape_code_fences_for_display
|
||||
# Collapse long reasoning to keep messages readable
|
||||
lines = last_reasoning.strip().splitlines()
|
||||
if len(lines) > 15:
|
||||
|
|
@ -13506,6 +13778,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
)
|
||||
response = f"> 💭 **Reasoning:**\n{_quoted}\n\n{response}"
|
||||
else:
|
||||
# Escape ``` inside reasoning so inner fences don't
|
||||
# break the outer code block used to render it.
|
||||
display_reasoning = escape_code_fences_for_display(display_reasoning)
|
||||
response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}"
|
||||
|
||||
# Runtime-metadata footer — only on the FINAL message of the turn.
|
||||
|
|
@ -14843,7 +15118,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
has_agent_tts = any(
|
||||
msg.get("role") == "assistant"
|
||||
and any(
|
||||
tc.get("function", {}).get("name") == "text_to_speech"
|
||||
(tc.get("function") or {}).get("name") == "text_to_speech"
|
||||
for tc in (msg.get("tool_calls") or [])
|
||||
)
|
||||
for msg in agent_messages
|
||||
|
|
@ -16141,6 +16416,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
metadata["direct_messages_topic_id"] = tid
|
||||
if reply_to_message_id is not None:
|
||||
metadata["telegram_reply_to_message_id"] = str(reply_to_message_id)
|
||||
if platform == Platform.SLACK and reply_to_message_id is not None:
|
||||
# Slack's reply_in_thread=false path uses message_id to distinguish
|
||||
# real existing threads from synthetic top-level session keys.
|
||||
metadata["message_id"] = str(reply_to_message_id)
|
||||
return metadata
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -17243,6 +17522,43 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
"""
|
||||
source = self._build_process_event_source(evt)
|
||||
if not source:
|
||||
# API-server-originated sessions bind a RAW session key (the
|
||||
# X-Hermes-Session-Id value — see _bind_api_server_session), not a
|
||||
# structured ``agent:main:...`` key, so _build_process_event_source
|
||||
# cannot derive routing metadata from it and returns None above.
|
||||
# Recover the raw session id and wake the real session via the API
|
||||
# server's own /v1/chat/completions entry point instead of
|
||||
# dropping the event.
|
||||
raw_sid = str(evt.get("origin_session_id") or "").strip()
|
||||
if not raw_sid:
|
||||
_sk = str(evt.get("session_key") or "").strip()
|
||||
if _sk and _parse_session_key(_sk) is None:
|
||||
raw_sid = _sk
|
||||
if raw_sid:
|
||||
adapter = self.adapters.get(Platform.API_SERVER)
|
||||
from gateway.wake import adapter_supports_push, deliver_wake
|
||||
if adapter is not None and not adapter_supports_push(adapter):
|
||||
try:
|
||||
logger.info(
|
||||
"Watch pattern notification — waking api_server "
|
||||
"session %s via self-post",
|
||||
raw_sid,
|
||||
)
|
||||
await deliver_wake(adapter, text=synth_text, session_id=raw_sid)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Watch notification self-post wake failed for "
|
||||
"session %s: %s",
|
||||
raw_sid, e,
|
||||
)
|
||||
return False
|
||||
logger.warning(
|
||||
"Dropping watch notification for raw session %s: no "
|
||||
"api_server adapter to self-post through",
|
||||
raw_sid,
|
||||
)
|
||||
return None
|
||||
logger.warning(
|
||||
"Dropping watch notification with no routing metadata for process %s",
|
||||
evt.get("session_id", "unknown"),
|
||||
|
|
@ -17256,6 +17572,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
break
|
||||
if not adapter:
|
||||
return None
|
||||
from gateway.wake import adapter_supports_push as _wake_push_ok
|
||||
if not _wake_push_ok(adapter):
|
||||
# Non-push adapter (api_server) resolved WITH routing metadata:
|
||||
# its chat_id is the raw session id (see _bind_api_server_session,
|
||||
# which binds chat_id = session_id). handle_message would run the
|
||||
# wake under a build_session_key()-derived key that never matches
|
||||
# the raw X-Hermes-Session-Id session — self-post instead.
|
||||
from gateway.wake import deliver_wake
|
||||
raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "")
|
||||
try:
|
||||
logger.info(
|
||||
"Watch pattern notification — waking api_server session "
|
||||
"%s via self-post",
|
||||
raw_sid,
|
||||
)
|
||||
await deliver_wake(adapter, text=synth_text, session_id=raw_sid)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Watch notification self-post wake failed for session "
|
||||
"%s: %s",
|
||||
raw_sid, e,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
metadata = {}
|
||||
parent_session_id = str(evt.get("parent_session_id") or "").strip()
|
||||
|
|
@ -17665,6 +18005,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
break
|
||||
|
||||
# --- Normal text-only notification ---
|
||||
# Skip when the agent already consumed this completion via
|
||||
# wait/log (#65379): process(wait) returned the exit code and
|
||||
# output inline, so the raw "[Background process ... finished
|
||||
# with exit code ...]" message would be a duplicate delivery
|
||||
# of the same completion. The agent_notify branch above
|
||||
# already honors _completion_consumed; without this check its
|
||||
# skip FALLS THROUGH to this block and re-delivers the output
|
||||
# the agent is actively summarizing. poll() is read-only and
|
||||
# intentionally does not mark consumed (#10156), so a status
|
||||
# check never suppresses this message.
|
||||
if _pr_check.is_completion_consumed(session_id):
|
||||
logger.debug(
|
||||
"Process watcher: completion for %s already consumed "
|
||||
"via wait/log — skipping raw notification (#65379)",
|
||||
session_id,
|
||||
)
|
||||
break
|
||||
# Decide whether to notify based on mode
|
||||
should_notify = (
|
||||
notify_mode in {"all", "result"}
|
||||
|
|
@ -18503,6 +18860,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
"1" if src.message_id else "0",
|
||||
)
|
||||
|
||||
# Slack renders a capability-aware platform note gated on
|
||||
# _slack_tools_loaded() — the gate state must appear in the key
|
||||
# (same parity contract as the Discord gate above) so a config /
|
||||
# MCP-registration flip re-renders once instead of serving a
|
||||
# stale pinned note for the rest of the session.
|
||||
slack_tools = ""
|
||||
if src.platform == Platform.SLACK:
|
||||
from gateway.session import _slack_tools_loaded
|
||||
|
||||
slack_tools = "1" if _slack_tools_loaded() else "0"
|
||||
|
||||
try:
|
||||
from hermes_constants import display_hermes_home
|
||||
|
||||
|
|
@ -18523,6 +18891,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
bool(context.shared_multi_user_session),
|
||||
discord_ids,
|
||||
discord_tools,
|
||||
slack_tools,
|
||||
tuple(p.value for p in context.connected_platforms),
|
||||
tuple(
|
||||
(
|
||||
|
|
@ -19898,13 +20267,38 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
# - Feishu only honors reply_in_thread when sending a reply, so topic
|
||||
# progress uses the triggering event message as the reply target
|
||||
# - Other platforms should use explicit source.thread_id only
|
||||
#
|
||||
# Slack honours platforms.slack.extra.reply_in_thread=false: if the
|
||||
# user has opted out of threaded replies, don't synthesise a thread
|
||||
# for progress messages either — the very first progress message
|
||||
# would otherwise create a thread that all subsequent replies
|
||||
# (including the final answer) would inherit (#18859).
|
||||
_progress_reply_in_thread = True
|
||||
if source.platform == Platform.SLACK:
|
||||
_slack_adapter_for_progress = self._adapter_for_source(source)
|
||||
if _slack_adapter_for_progress is not None:
|
||||
try:
|
||||
_progress_reply_in_thread = bool(
|
||||
_slack_adapter_for_progress.config.extra.get(
|
||||
"reply_in_thread", True
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
_progress_reply_in_thread = True
|
||||
_progress_thread_id = _resolve_progress_thread_id(
|
||||
source.platform, source.thread_id, event_message_id,
|
||||
reply_in_thread=_progress_reply_in_thread,
|
||||
)
|
||||
_progress_metadata = (
|
||||
self._thread_metadata_for_source(source, event_message_id)
|
||||
if _progress_thread_id == source.thread_id
|
||||
else {"thread_id": _progress_thread_id}
|
||||
else self._thread_metadata_for_target(
|
||||
source.platform,
|
||||
source.chat_id,
|
||||
_progress_thread_id,
|
||||
chat_type=getattr(source, "chat_type", None),
|
||||
reply_to_message_id=event_message_id,
|
||||
)
|
||||
) if _progress_thread_id else None
|
||||
_progress_metadata = _non_conversational_metadata(_progress_metadata, platform=source.platform)
|
||||
_progress_reply_to = (
|
||||
|
|
@ -20076,8 +20470,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
async def _roll_progress_overflow_if_needed() -> bool:
|
||||
"""Start fresh editable progress bubbles before a bubble exceeds limit.
|
||||
|
||||
Returns True when it delivered/split the current buffer and the
|
||||
caller should skip the normal send/edit path for this tick.
|
||||
Returns True when it delivered/split the current buffer, or when
|
||||
a transient edit failure left the buffer and message identity
|
||||
intact for a later retry. In either case the caller should skip
|
||||
the normal send/edit path for this tick.
|
||||
"""
|
||||
nonlocal progress_msg_id, progress_lines, can_edit
|
||||
if not progress_lines or not can_edit:
|
||||
|
|
@ -20090,6 +20486,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
if progress_msg_id is not None:
|
||||
result = await _edit_progress_message(progress_msg_id, first_text)
|
||||
if not result.success:
|
||||
if getattr(result, "retryable", False):
|
||||
logger.debug(
|
||||
"[%s] Transient overflow edit failure — keeping can_edit=True",
|
||||
adapter.name,
|
||||
)
|
||||
return True
|
||||
can_edit = False
|
||||
# Fall back to the existing non-edit behavior below.
|
||||
return False
|
||||
|
|
@ -20359,7 +20761,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
"reply_to_message_id": event_message_id,
|
||||
}
|
||||
else:
|
||||
_status_thread_metadata = self._thread_metadata_for_source(source, event_message_id) if _progress_thread_id else None
|
||||
_status_thread_metadata = (
|
||||
self._thread_metadata_for_source(source, event_message_id)
|
||||
if _progress_thread_id == source.thread_id
|
||||
else self._thread_metadata_for_target(
|
||||
source.platform,
|
||||
source.chat_id,
|
||||
_progress_thread_id,
|
||||
chat_type=getattr(source, "chat_type", None),
|
||||
reply_to_message_id=event_message_id,
|
||||
)
|
||||
) if _progress_thread_id else None
|
||||
|
||||
def _status_callback_sync(event_type: str, message: str) -> None:
|
||||
if not _status_adapter or not _run_still_current():
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import threading
|
|||
import uuid
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -342,6 +342,52 @@ that requires raw IDs). Discord is excluded because mentions use ``<@user_id>``
|
|||
and the LLM needs the real ID to tag users."""
|
||||
|
||||
|
||||
def _slack_tools_loaded() -> bool:
|
||||
"""True iff the agent will actually have Slack tools this session.
|
||||
|
||||
Two independent paths grant Slack capability:
|
||||
1. Native `slack` toolset enabled via `hermes tools` (opt-in, default
|
||||
OFF) AND `SLACK_BOT_TOKEN` set — the tool's `check_fn` gates on it
|
||||
at registry time, so config alone isn't enough.
|
||||
2. An MCP server that has ACTUALLY registered tools into the live
|
||||
registry (tools/mcp_tool.get_registered_mcp_server_names()), whose
|
||||
name suggests Slack. This is the real, availability-filtered
|
||||
signal (post-connection, post include/exclude filtering) rather
|
||||
than just what's listed in config.yaml -- a configured-but-
|
||||
unconnected or zero-tool MCP server must not claim capability.
|
||||
Named MCP servers are process-wide (one gateway connects each MCP
|
||||
server once, not per-session), so this check is intentionally NOT
|
||||
scoped further per-session -- unlike the earlier get_all_tool_names()
|
||||
approach this replaces, which conflated ALL built-in tool names
|
||||
process-wide, this only inspects the small, purpose-built MCP
|
||||
server-name map.
|
||||
|
||||
Returns False (safe default — keeps the stale-API disclaimer) on any
|
||||
error so a bad config can never silently promise tools the agent lacks.
|
||||
"""
|
||||
try:
|
||||
from tools.mcp_tool import get_registered_mcp_server_names
|
||||
if any("slack" in name.lower() for name in get_registered_mcp_server_names()):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not (os.environ.get("SLACK_BOT_TOKEN") or "").strip():
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from hermes_cli.tools_config import _get_platform_tools
|
||||
cfg = load_config()
|
||||
# include_default_mcp_servers=True (the default) so a Slack MCP
|
||||
# server that's enabled by default for this platform (not
|
||||
# explicitly listed) is also counted, in addition to the native
|
||||
# 'slack' toolset.
|
||||
enabled = _get_platform_tools(cfg, "slack")
|
||||
return "slack" in enabled
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _discord_tools_loaded() -> bool:
|
||||
"""True iff the agent will actually have Discord tools this session.
|
||||
|
||||
|
|
@ -517,15 +563,31 @@ def build_session_context_prompt(
|
|||
|
||||
# Platform-specific behavioral notes
|
||||
if context.source.platform == Platform.SLACK:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"**Platform notes:** You are running inside Slack. "
|
||||
"You do NOT have access to Slack-specific APIs — you cannot search "
|
||||
"channel history, pin/unpin messages, manage channels, or list users. "
|
||||
"Do not promise to perform these actions. The gateway may inline the "
|
||||
"current message's Slack block/attachment payload when available, but "
|
||||
"you still cannot call Slack APIs yourself."
|
||||
)
|
||||
# Inject the Slack capability note only when the agent actually has
|
||||
# Slack tools loaded this session — native `slack` toolset opt-in,
|
||||
# or a connected MCP server that has registered Slack tools.
|
||||
# Otherwise keep the stale-API disclaimer honest so we never
|
||||
# promise tools the agent lacks. Mirrors the Discord pattern below.
|
||||
if _slack_tools_loaded():
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"**Platform notes:** You are running inside Slack and have access "
|
||||
"to Slack-specific tools this session. Consult the available Slack "
|
||||
"tool schemas for the exact operations supported (e.g. channel "
|
||||
"history and thread lookups, posting, reactions) — use those tools "
|
||||
"for Slack-specific requests, and do not promise Slack actions "
|
||||
"beyond what the loaded tools actually expose."
|
||||
)
|
||||
else:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"**Platform notes:** You are running inside Slack. "
|
||||
"You do NOT have access to Slack-specific APIs — you cannot search "
|
||||
"channel history, pin/unpin messages, manage channels, or list users. "
|
||||
"Do not promise to perform these actions. The gateway may inline the "
|
||||
"current message's Slack block/attachment payload when available, but "
|
||||
"you still cannot call Slack APIs yourself."
|
||||
)
|
||||
if context.shared_multi_user_session:
|
||||
lines.append(
|
||||
"In shared Slack threads, use the current turn's sender prefix "
|
||||
|
|
@ -722,6 +784,13 @@ class SessionEntry:
|
|||
auto_reset_reason: Optional[str] = None # "idle" or "daily"
|
||||
reset_had_activity: bool = False # whether the expired session had any messages
|
||||
|
||||
# When this session was created by an auto-reset, the session_id of the
|
||||
# session it replaced. Used to give Slack/Discord channels/threads a
|
||||
# lightweight continuity hint (see build_channel_continuity_note) so the
|
||||
# agent recalls the prior same-channel session via session_search instead
|
||||
# of binding the request to an unrelated recent session.
|
||||
prev_session_id: Optional[str] = None
|
||||
|
||||
# Set by reset_session() when the user explicitly sends /new or /reset.
|
||||
# Consumed once by _handle_message_with_agent to trigger topic/channel
|
||||
# skill re-injection on the first message of the new session. We can't
|
||||
|
|
@ -794,6 +863,7 @@ class SessionEntry:
|
|||
"was_auto_reset": self.was_auto_reset,
|
||||
"auto_reset_reason": self.auto_reset_reason,
|
||||
"reset_had_activity": self.reset_had_activity,
|
||||
"prev_session_id": self.prev_session_id,
|
||||
}
|
||||
if self.model_override:
|
||||
# Defence-in-depth: strip credentials even if a caller stored an
|
||||
|
|
@ -870,10 +940,51 @@ class SessionEntry:
|
|||
was_auto_reset=data.get("was_auto_reset", False),
|
||||
auto_reset_reason=data.get("auto_reset_reason"),
|
||||
reset_had_activity=data.get("reset_had_activity", False),
|
||||
prev_session_id=data.get("prev_session_id"),
|
||||
model_override=sanitize_model_override(data.get("model_override")),
|
||||
)
|
||||
|
||||
|
||||
def build_channel_continuity_note(
|
||||
entry: "SessionEntry",
|
||||
source: SessionSource,
|
||||
) -> Optional[str]:
|
||||
"""Build a lightweight session-continuity hint for Slack/Discord channels.
|
||||
|
||||
Slack and Discord channels/threads are long-lived: when the daily/idle
|
||||
reset policy starts a fresh session, the agent loses the thread's prior
|
||||
context and can mistakenly bind a new request to an unrelated recent
|
||||
session. This deterministic one-line hint points the agent at the
|
||||
specific prior session in *this* channel/thread so it recalls that
|
||||
context via ``session_search`` before acting.
|
||||
|
||||
Returns ``None`` (and the caller adds nothing) unless **all** hold:
|
||||
- the source platform is Slack or Discord,
|
||||
- this session was created by an auto-reset that had real activity,
|
||||
- the previous session_id was recorded on the entry.
|
||||
|
||||
No LLM calls, no extra API/DB lookups — the previous session id is
|
||||
already known from :meth:`SessionStore.get_or_create_session`.
|
||||
"""
|
||||
if source.platform not in (Platform.SLACK, Platform.DISCORD):
|
||||
return None
|
||||
if not getattr(entry, "reset_had_activity", False):
|
||||
return None
|
||||
prev = getattr(entry, "prev_session_id", None)
|
||||
if not prev:
|
||||
return None
|
||||
|
||||
where = "thread" if source.thread_id else "channel"
|
||||
return (
|
||||
f"[System note: This {where} had an earlier Hermes session "
|
||||
f"(session_id: {prev}) that was auto-reset. If the user refers to "
|
||||
f"earlier work here, or the request depends on this {where}'s history, "
|
||||
f"use the session_search tool to recall that prior session before "
|
||||
f"acting — do not assume an unrelated recent session is the right "
|
||||
f"context.]"
|
||||
)
|
||||
|
||||
|
||||
def is_shared_multi_user_session(
|
||||
source: SessionSource,
|
||||
*,
|
||||
|
|
@ -931,12 +1042,16 @@ def build_session_key(
|
|||
multiplexing gateway passes a non-default profile.
|
||||
|
||||
DM rules:
|
||||
- Slack ``scope_id`` identifies the workspace before chat/user ids. Other
|
||||
platforms retain their existing key format; in particular, Discord
|
||||
guild scope is intentionally not added here as a compatibility change.
|
||||
- DMs include chat_id when present, so each private conversation is isolated.
|
||||
- thread_id further differentiates threaded DMs within the same DM chat.
|
||||
- Without chat_id, thread_id is used as a best-effort fallback.
|
||||
- Without thread_id or chat_id, DMs share a single session.
|
||||
|
||||
Group/channel rules:
|
||||
- Slack ``scope_id`` identifies the workspace before chat/thread ids.
|
||||
- chat_id identifies the parent group/channel.
|
||||
- user_id/user_id_alt isolates participants within that parent chat when available when
|
||||
``group_sessions_per_user`` is enabled.
|
||||
|
|
@ -951,15 +1066,24 @@ def build_session_key(
|
|||
"""
|
||||
ns = _session_key_namespace(profile)
|
||||
platform = source.platform.value
|
||||
slack_scope_id = (
|
||||
str(source.scope_id)
|
||||
if source.platform == Platform.SLACK and source.scope_id
|
||||
else None
|
||||
)
|
||||
if source.chat_type == "dm":
|
||||
dm_chat_id = source.chat_id
|
||||
if source.platform == Platform.WHATSAPP:
|
||||
dm_chat_id = canonical_whatsapp_identifier(source.chat_id)
|
||||
|
||||
dm_parts = [ns, platform, "dm"]
|
||||
if slack_scope_id:
|
||||
dm_parts.append(slack_scope_id)
|
||||
if dm_chat_id:
|
||||
dm_parts.append(dm_chat_id)
|
||||
if source.thread_id:
|
||||
return f"{ns}:{platform}:dm:{dm_chat_id}:{source.thread_id}"
|
||||
return f"{ns}:{platform}:dm:{dm_chat_id}"
|
||||
dm_parts.append(source.thread_id)
|
||||
return ":".join(str(part) for part in dm_parts)
|
||||
# No chat_id — fall back to the sender's own identifier before the
|
||||
# bare per-platform sink. Without this, every DM from every user that
|
||||
# arrives without a chat_id (non-standard adapters / synthetic sources)
|
||||
|
|
@ -973,12 +1097,13 @@ def build_session_key(
|
|||
or dm_participant_id
|
||||
)
|
||||
if dm_participant_id:
|
||||
dm_parts.append(str(dm_participant_id))
|
||||
if source.thread_id:
|
||||
return f"{ns}:{platform}:dm:{dm_participant_id}:{source.thread_id}"
|
||||
return f"{ns}:{platform}:dm:{dm_participant_id}"
|
||||
dm_parts.append(source.thread_id)
|
||||
return ":".join(str(part) for part in dm_parts)
|
||||
if source.thread_id:
|
||||
return f"{ns}:{platform}:dm:{source.thread_id}"
|
||||
return f"{ns}:{platform}:dm"
|
||||
dm_parts.append(source.thread_id)
|
||||
return ":".join(str(part) for part in dm_parts)
|
||||
|
||||
participant_id = source.user_id_alt or source.user_id
|
||||
if participant_id and source.platform == Platform.WHATSAPP:
|
||||
|
|
@ -988,6 +1113,8 @@ def build_session_key(
|
|||
participant_id = canonical_whatsapp_identifier(str(participant_id)) or participant_id
|
||||
key_parts = [ns, platform, source.chat_type]
|
||||
|
||||
if slack_scope_id:
|
||||
key_parts.append(slack_scope_id)
|
||||
if source.chat_id:
|
||||
key_parts.append(source.chat_id)
|
||||
if source.thread_id:
|
||||
|
|
@ -1003,7 +1130,7 @@ def build_session_key(
|
|||
if isolate_user and participant_id:
|
||||
key_parts.append(str(participant_id))
|
||||
|
||||
return ":".join(key_parts)
|
||||
return ":".join(str(part) for part in key_parts)
|
||||
|
||||
|
||||
class _SessionFlight:
|
||||
|
|
@ -1053,6 +1180,11 @@ class SessionStore:
|
|||
self._persisted_routing_generation = 0
|
||||
self._inflight_lock = threading.Lock()
|
||||
self._inflight_sessions: Dict[str, _SessionFlight] = {}
|
||||
# An unscoped pre-migration Slack key can represent at most one
|
||||
# workspace. Claim it once per process so simultaneous first messages
|
||||
# from two workspaces cannot both revive the same legacy session.
|
||||
self._legacy_slack_claim_lock = threading.Lock()
|
||||
self._claimed_legacy_slack_keys: set[str] = set()
|
||||
self._transcript_retry_lock = threading.Lock()
|
||||
self._dirty_transcripts: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self._transcript_append_failures: Dict[str, int] = {}
|
||||
|
|
@ -1436,6 +1568,74 @@ class SessionStore:
|
|||
profile=self._resolve_profile_for_key(source),
|
||||
)
|
||||
|
||||
def _legacy_slack_session_key(self, source: SessionSource) -> Optional[str]:
|
||||
"""Return the pre-workspace Slack key for an explicitly scoped source.
|
||||
|
||||
The compatibility path is deliberately Slack-only. Discord and every
|
||||
other platform keep byte-identical keys, and an unscoped Slack session
|
||||
may be claimed by only one workspace because its old key contains no
|
||||
information that could safely distinguish multiple teams.
|
||||
"""
|
||||
if source.platform != Platform.SLACK or not source.scope_id:
|
||||
return None
|
||||
legacy_source = replace(source, scope_id=None, guild_id=None)
|
||||
return build_session_key(
|
||||
legacy_source,
|
||||
group_sessions_per_user=getattr(
|
||||
self.config, "group_sessions_per_user", True
|
||||
),
|
||||
thread_sessions_per_user=getattr(
|
||||
self.config, "thread_sessions_per_user", False
|
||||
),
|
||||
profile=self._resolve_profile_for_key(source),
|
||||
)
|
||||
|
||||
def _claim_legacy_slack_key(self, legacy_key: Optional[str]) -> bool:
|
||||
"""Atomically reserve one ambiguous legacy Slack key for migration."""
|
||||
if not legacy_key:
|
||||
return False
|
||||
claim_lock = getattr(self, "_legacy_slack_claim_lock", None)
|
||||
if claim_lock is None:
|
||||
claim_lock = threading.Lock()
|
||||
self._legacy_slack_claim_lock = claim_lock
|
||||
with claim_lock:
|
||||
claimed = getattr(self, "_claimed_legacy_slack_keys", None)
|
||||
if claimed is None:
|
||||
claimed = set()
|
||||
self._claimed_legacy_slack_keys = claimed
|
||||
if legacy_key in claimed:
|
||||
return False
|
||||
claimed.add(legacy_key)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _recovered_row_matches_source_scope(
|
||||
recovered: Dict[str, Any], source: SessionSource
|
||||
) -> bool:
|
||||
"""Reject recovered rows whose recorded origin belongs to another workspace.
|
||||
|
||||
Slack group/channel rows recorded with an origin_json carry the
|
||||
workspace (scope_id) they were created under. A workspace-scoped
|
||||
lookup must not adopt a row another team recorded — even via the
|
||||
legacy-key fallback — unless the recorded origin names the same
|
||||
workspace. Rows without a parseable origin are rejected for scoped
|
||||
sources: an unattributable transcript is precisely the ambiguity
|
||||
this guard exists to avoid.
|
||||
"""
|
||||
if (
|
||||
source.platform != Platform.SLACK
|
||||
or source.chat_type == "dm"
|
||||
or not source.scope_id
|
||||
):
|
||||
return True
|
||||
try:
|
||||
origin = json.loads(recovered.get("origin_json") or "")
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if not isinstance(origin, dict):
|
||||
return False
|
||||
return origin.get("scope_id", origin.get("guild_id")) == source.scope_id
|
||||
|
||||
def _create_entry_from_recovered_row(
|
||||
self,
|
||||
*,
|
||||
|
|
@ -1460,6 +1660,45 @@ class SessionStore:
|
|||
chat_type=source.chat_type,
|
||||
)
|
||||
|
||||
def _find_gateway_session_row(
|
||||
self,
|
||||
*,
|
||||
session_key: str,
|
||||
source: SessionSource,
|
||||
allow_peer_fallback: bool,
|
||||
raise_on_lookup_error: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Query one durable gateway session row.
|
||||
|
||||
Scoped Slack lookups disable SessionDB's platform/chat/user fallback:
|
||||
that tuple does not contain a workspace id and could therefore revive
|
||||
another team's session. The caller performs one explicit exact lookup
|
||||
of the old unscoped key instead.
|
||||
"""
|
||||
if not self._db:
|
||||
return None
|
||||
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
|
||||
if not callable(finder):
|
||||
return None
|
||||
try:
|
||||
return finder(
|
||||
source=source.platform.value,
|
||||
user_id=source.user_id,
|
||||
session_key=session_key,
|
||||
chat_id=source.chat_id if allow_peer_fallback else None,
|
||||
chat_type=source.chat_type if allow_peer_fallback else None,
|
||||
thread_id=source.thread_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Gateway session DB recovery failed for %s: %s",
|
||||
session_key,
|
||||
exc,
|
||||
)
|
||||
if raise_on_lookup_error:
|
||||
raise
|
||||
return None
|
||||
|
||||
def _recover_session_from_db(
|
||||
self,
|
||||
*,
|
||||
|
|
@ -1469,27 +1708,30 @@ class SessionStore:
|
|||
raise_on_lookup_error: bool = False,
|
||||
) -> Optional[SessionEntry]:
|
||||
"""Rebuild a missing session-key mapping from durable state.db data."""
|
||||
if not self._db:
|
||||
return None
|
||||
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
|
||||
if not callable(finder):
|
||||
return None
|
||||
try:
|
||||
recovered = finder(
|
||||
source=source.platform.value,
|
||||
user_id=source.user_id,
|
||||
session_key=session_key,
|
||||
chat_id=source.chat_id,
|
||||
chat_type=source.chat_type,
|
||||
thread_id=source.thread_id,
|
||||
legacy_key = self._legacy_slack_session_key(source)
|
||||
recovered = self._find_gateway_session_row(
|
||||
session_key=session_key,
|
||||
source=source,
|
||||
allow_peer_fallback=legacy_key is None,
|
||||
raise_on_lookup_error=raise_on_lookup_error,
|
||||
)
|
||||
migrated_legacy = False
|
||||
if (
|
||||
not recovered
|
||||
and legacy_key
|
||||
and self._claim_legacy_slack_key(legacy_key)
|
||||
):
|
||||
recovered = self._find_gateway_session_row(
|
||||
session_key=legacy_key,
|
||||
source=source,
|
||||
allow_peer_fallback=False,
|
||||
raise_on_lookup_error=raise_on_lookup_error,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Gateway session DB recovery failed for %s: %s", session_key, exc)
|
||||
if raise_on_lookup_error:
|
||||
raise
|
||||
return None
|
||||
migrated_legacy = bool(recovered)
|
||||
if not recovered:
|
||||
return None
|
||||
if not self._recovered_row_matches_source_scope(recovered, source):
|
||||
return None
|
||||
if not self._recovered_row_allowed_for_active_profile(
|
||||
requested_session_key=session_key,
|
||||
recovered=recovered,
|
||||
|
|
@ -1506,38 +1748,50 @@ class SessionStore:
|
|||
self._db.reopen_session(str(recovered["id"]))
|
||||
except Exception as exc:
|
||||
logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc)
|
||||
return self._create_entry_from_recovered_row(
|
||||
entry = self._create_entry_from_recovered_row(
|
||||
row=recovered,
|
||||
session_key=session_key,
|
||||
source=source,
|
||||
now=now,
|
||||
)
|
||||
if migrated_legacy:
|
||||
self._record_gateway_session_peer(
|
||||
entry.session_id,
|
||||
session_key,
|
||||
source,
|
||||
display_name=entry.display_name,
|
||||
)
|
||||
return entry
|
||||
|
||||
def _query_recoverable_session(self, *, session_key, source, now):
|
||||
def _query_recoverable_session(
|
||||
self, *, session_key, source, now, lookup_session_key=None
|
||||
):
|
||||
"""DB-only half of _recover_session_from_db (no lock needed).
|
||||
|
||||
Returns a SessionEntry or None. Caller assigns _entries[key] under lock.
|
||||
"""
|
||||
if not self._db:
|
||||
return None
|
||||
finder = getattr(self._db, "find_latest_gateway_session_for_peer", None)
|
||||
if not callable(finder):
|
||||
return None
|
||||
try:
|
||||
recovered = finder(
|
||||
source=source.platform.value,
|
||||
user_id=source.user_id,
|
||||
session_key=session_key,
|
||||
chat_id=source.chat_id,
|
||||
chat_type=source.chat_type,
|
||||
thread_id=source.thread_id,
|
||||
legacy_key = self._legacy_slack_session_key(source)
|
||||
recovered = self._find_gateway_session_row(
|
||||
session_key=session_key,
|
||||
source=source,
|
||||
allow_peer_fallback=legacy_key is None,
|
||||
)
|
||||
migrated_legacy = False
|
||||
if (
|
||||
not recovered
|
||||
and legacy_key
|
||||
and self._claim_legacy_slack_key(legacy_key)
|
||||
):
|
||||
recovered = self._find_gateway_session_row(
|
||||
session_key=legacy_key,
|
||||
source=source,
|
||||
allow_peer_fallback=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Gateway session DB recovery failed for %s: %s",
|
||||
session_key, exc)
|
||||
return None
|
||||
migrated_legacy = bool(recovered)
|
||||
if not isinstance(recovered, dict):
|
||||
return None
|
||||
if not self._recovered_row_matches_source_scope(recovered, source):
|
||||
return None
|
||||
if not self._recovered_row_allowed_for_active_profile(
|
||||
requested_session_key=session_key,
|
||||
recovered=recovered,
|
||||
|
|
@ -1555,9 +1809,17 @@ class SessionStore:
|
|||
except Exception as exc:
|
||||
logger.debug("Gateway session DB reopen failed for %s: %s",
|
||||
session_key, exc)
|
||||
return self._create_entry_from_recovered_row(
|
||||
entry = self._create_entry_from_recovered_row(
|
||||
row=recovered, session_key=session_key, source=source, now=now,
|
||||
)
|
||||
if migrated_legacy:
|
||||
self._record_gateway_session_peer(
|
||||
entry.session_id,
|
||||
session_key,
|
||||
source,
|
||||
display_name=entry.display_name,
|
||||
)
|
||||
return entry
|
||||
def _record_gateway_session_peer(
|
||||
self,
|
||||
session_id: str,
|
||||
|
|
@ -1921,6 +2183,52 @@ class SessionStore:
|
|||
session_key = self._generate_session_key(source)
|
||||
now = _now()
|
||||
|
||||
# One-time routing-index migration for Slack sessions created before
|
||||
# workspace scope was part of the key. Move (rather than copy) the
|
||||
# legacy entry so a second workspace with identical Slack ids cannot
|
||||
# attach to the same transcript.
|
||||
#
|
||||
# Adoption policy (composed from #20583/#66398 and #68925):
|
||||
# - The legacy entry's recorded origin names a workspace → migrate
|
||||
# only when it matches the incoming workspace (precise).
|
||||
# - Scope-less origin, DM → first workspace claims it once
|
||||
# (claim-once): a 1:1 DM has a single human peer, so continuity
|
||||
# across the key-format change outweighs the ambiguity risk.
|
||||
# - Scope-less origin, channel/group → refuse: channel ids collide
|
||||
# across workspaces and a shared transcript leaking to a second
|
||||
# tenant is exactly the bug this fix removes.
|
||||
migrated_legacy_entry: Optional[SessionEntry] = None
|
||||
legacy_key = self._legacy_slack_session_key(source)
|
||||
if legacy_key and not force_new:
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
legacy_entry = self._entries.get(legacy_key)
|
||||
if session_key not in self._entries and legacy_entry is not None:
|
||||
origin_scope = (
|
||||
getattr(legacy_entry.origin, "scope_id", None)
|
||||
if legacy_entry.origin is not None
|
||||
else None
|
||||
)
|
||||
if origin_scope is not None:
|
||||
adopt = origin_scope == source.scope_id
|
||||
else:
|
||||
adopt = source.chat_type == "dm"
|
||||
if adopt and self._claim_legacy_slack_key(legacy_key):
|
||||
migrated_legacy_entry = self._entries.pop(legacy_key)
|
||||
migrated_legacy_entry.session_key = session_key
|
||||
migrated_legacy_entry.origin = source
|
||||
migrated_legacy_entry.platform = source.platform
|
||||
migrated_legacy_entry.chat_type = source.chat_type
|
||||
self._entries[session_key] = migrated_legacy_entry
|
||||
if migrated_legacy_entry is not None:
|
||||
self._save_entries()
|
||||
self._record_gateway_session_peer(
|
||||
migrated_legacy_entry.session_id,
|
||||
session_key,
|
||||
source,
|
||||
display_name=migrated_legacy_entry.display_name,
|
||||
)
|
||||
|
||||
db_end_session_id = None
|
||||
db_create_kwargs = None
|
||||
existing_session_id = None
|
||||
|
|
@ -1990,6 +2298,7 @@ class SessionStore:
|
|||
was_auto_reset = False
|
||||
auto_reset_reason = None
|
||||
reset_had_activity = False
|
||||
prev_session_id: Optional[str] = None
|
||||
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
|
|
@ -2024,6 +2333,7 @@ class SessionStore:
|
|||
auto_reset_reason = _reset_reason
|
||||
reset_had_activity = entry.last_prompt_tokens > 0
|
||||
db_end_session_id = entry.session_id
|
||||
prev_session_id = entry.session_id
|
||||
entry = None
|
||||
_needs_recover = True
|
||||
elif entry.session_id != _stale_session_id:
|
||||
|
|
@ -2038,6 +2348,7 @@ class SessionStore:
|
|||
auto_reset_reason = _reset_reason
|
||||
reset_had_activity = entry.last_prompt_tokens > 0
|
||||
db_end_session_id = entry.session_id
|
||||
prev_session_id = entry.session_id
|
||||
self._entries.pop(session_key, None)
|
||||
entry = None
|
||||
_needs_recover = True
|
||||
|
|
@ -2050,6 +2361,10 @@ class SessionStore:
|
|||
|
||||
# ---- Phase 3: no-lock I/O -- recovery + create + save + DB ops ----
|
||||
if _needs_recover and db_end_session_id is None:
|
||||
# The legacy (pre-workspace) Slack key fallback happens INSIDE
|
||||
# _query_recoverable_session (#20583/#66398 design): it performs
|
||||
# the exact-key legacy lookup, claims the key once per process,
|
||||
# and rewrites the peer row to the scoped key on success.
|
||||
recovered = self._query_recoverable_session(
|
||||
session_key=session_key, source=source, now=now,
|
||||
)
|
||||
|
|
@ -2078,6 +2393,7 @@ class SessionStore:
|
|||
was_auto_reset=was_auto_reset,
|
||||
auto_reset_reason=auto_reset_reason,
|
||||
reset_had_activity=reset_had_activity,
|
||||
prev_session_id=prev_session_id,
|
||||
)
|
||||
with self._lock:
|
||||
current = self._entries.get(session_key)
|
||||
|
|
|
|||
|
|
@ -96,12 +96,11 @@ _SESSION_PROFILE: ContextVar = ContextVar("HERMES_SESSION_PROFILE", default=_UNS
|
|||
# Whether the current session's delivery channel can route an ASYNC completion
|
||||
# back to the agent AFTER the current turn ends (i.e. wake a fresh turn).
|
||||
#
|
||||
# True — CLI (in-process completion_queue drain) and the real gateway
|
||||
# platforms (Telegram/Discord/Slack/...), which hold a persistent
|
||||
# outbound channel and run the watcher/drain loops.
|
||||
# False — stateless request/response adapters (the API server: every route,
|
||||
# spec and proprietary, tears down its channel when the turn ends, so
|
||||
# a background completion that finishes later has nowhere to go).
|
||||
# True — long-lived CLI sessions (in-process completion_queue drain) and the
|
||||
# real gateway platforms (Telegram/Discord/Slack/...), which hold a
|
||||
# persistent outbound channel and run the watcher/drain loops.
|
||||
# False — finite runtimes that can end before a detached completion returns:
|
||||
# stateless API-server requests and dispatcher-spawned Kanban workers.
|
||||
#
|
||||
# Tools that promise async delivery (terminal notify_on_complete /
|
||||
# watch_patterns, delegate_task background=True) read this via
|
||||
|
|
@ -355,18 +354,29 @@ def declare_stateless_channel() -> None:
|
|||
def async_delivery_supported() -> bool:
|
||||
"""Whether the current session can deliver a background completion later.
|
||||
|
||||
Returns ``False`` when the active session was bound by a stateless channel:
|
||||
an adapter that cannot route a notification back after the turn ends (the
|
||||
API server), or a one-shot runner that exits after its final response
|
||||
(``hermes -z``, cron — see :func:`declare_stateless_channel`). The real
|
||||
gateway platforms, the interactive CLI, and any path that never bound the
|
||||
contextvar return ``True``.
|
||||
Returns ``False`` for finite runtimes that can end before a detached result
|
||||
is delivered: sessions explicitly bound by a stateless channel — an adapter
|
||||
that cannot route a notification back after the turn ends (the API server),
|
||||
or a one-shot runner that exits after its final response (``hermes -z``,
|
||||
cron — see :func:`declare_stateless_channel`) — and dispatcher-spawned
|
||||
Kanban workers (identified by ``HERMES_KANBAN_TASK``), which are one-shot
|
||||
``chat -q`` subprocesses. The real gateway platforms, the interactive CLI,
|
||||
and any other path that never bound the contextvar return ``True``.
|
||||
|
||||
Tools that promise async delivery (``terminal`` notify_on_complete /
|
||||
watch_patterns, ``delegate_task`` background=True) consult this before
|
||||
registering a watcher / dispatching a detached child, so they can refuse a
|
||||
promise the channel can't keep instead of silently no-op'ing.
|
||||
"""
|
||||
import os
|
||||
|
||||
# A Kanban worker is a one-shot subprocess. Its parent session and process
|
||||
# disappear after the quiet turn returns, so a completion queued later has
|
||||
# no durable consumer even though an ordinary CLI session can drain that
|
||||
# queue. Force tools onto their existing synchronous/polling fallbacks.
|
||||
if os.environ.get("HERMES_KANBAN_TASK"):
|
||||
return False
|
||||
|
||||
value = _SESSION_ASYNC_DELIVERY.get()
|
||||
if value is _UNSET:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -3562,6 +3562,22 @@ class GatewaySlashCommandsMixin:
|
|||
defer_context_engine_notification=True,
|
||||
)
|
||||
)
|
||||
|
||||
# If _compress_context returned unchanged because a
|
||||
# concurrent compression lock is held, tell the user
|
||||
# clearly instead of showing the misleading
|
||||
# "No changes from compression" no-op text. The wording
|
||||
# distinguishes a confirmed holder from an unconfirmed
|
||||
# acquisition failure (describe_compression_lock_skip).
|
||||
# The deferred context-engine notification is discarded by
|
||||
# the finally block below (finalize committed=False).
|
||||
_lock_skipped = getattr(tmp_agent, "_compression_skipped_due_to_lock", None)
|
||||
if _lock_skipped is True or isinstance(_lock_skipped, str):
|
||||
from agent.manual_compression_feedback import (
|
||||
describe_compression_lock_skip,
|
||||
)
|
||||
return describe_compression_lock_skip(_lock_skipped)
|
||||
|
||||
if partial and tail:
|
||||
compressed = rejoin_compressed_head_and_tail(compressed, tail)
|
||||
|
||||
|
|
|
|||
|
|
@ -42,13 +42,7 @@ logger = logging.getLogger("gateway.stream_consumer")
|
|||
|
||||
# Sentinel to signal the stream is complete
|
||||
_DONE = object()
|
||||
|
||||
# Sentinel to signal a tool boundary — finalize current message and start a
|
||||
# new one so that subsequent text appears below tool progress messages.
|
||||
_NEW_SEGMENT = object()
|
||||
|
||||
# Queue marker for a completed assistant commentary message emitted between
|
||||
# API/tool iterations (for example: "I'll inspect the repo first.").
|
||||
_COMMENTARY = object()
|
||||
|
||||
# Queue marker for a synchronous flush barrier. Enqueued as
|
||||
|
|
@ -61,6 +55,76 @@ _COMMENTARY = object()
|
|||
_FLUSH = object()
|
||||
|
||||
|
||||
def escape_code_fences_for_display(text: str) -> str:
|
||||
"""Escape triple-backtick markers so text can be safely wrapped
|
||||
inside an outer ``` code block without breaking the fence.
|
||||
|
||||
When reasoning content contains ``` (e.g. the model quotes code
|
||||
in its thinking), wrapping it in an outer ``` for display causes
|
||||
the inner fence to break the outer block. Solution: replace each
|
||||
`` ``` `` with `` \\`\\`\\` `` before wrapping.
|
||||
|
||||
Returns:
|
||||
The input text with each `` ``` `` replaced by `` \\`\\`\\` ``,
|
||||
or the input unchanged if no triple-backticks are present.
|
||||
"""
|
||||
if not isinstance(text, str) or "```" not in text:
|
||||
return text
|
||||
return text.replace("```", "\\`\\`\\`")
|
||||
|
||||
|
||||
def ensure_closed_code_fences(text: str) -> str:
|
||||
"""Append a closing `` ``` `` fence and/or `` ` `` if the text has
|
||||
orphaned code-block or inline-code markers.
|
||||
|
||||
When model output is truncated mid-code-block (e.g. by token limits
|
||||
or a finish_reason="length"), the resulting message has an unclosed
|
||||
code fence. On Discord, Slack, and other platforms this causes
|
||||
everything after the orphaned fence to render as a single code block.
|
||||
The same problem applies to inline-code spans closed by a single
|
||||
backtick: an orphaned `` ` `` makes the remainder of the message
|
||||
render as inline code.
|
||||
|
||||
Triple-backtick: count `` ``` `` occurrences. If odd, append a
|
||||
closing fence on its own line. This is safe because nested
|
||||
triple-backtick fences (e.g. a literal `` ``` `` inside a code block)
|
||||
are exceedingly rare in model output and, when they do appear, the
|
||||
extra closing fence just creates a brief empty code block at the end
|
||||
of the message — far less harmful than the entire message being one
|
||||
giant code block.
|
||||
|
||||
Single backtick: after balancing triple-backtick fences, strip all
|
||||
complete `` ```…``` `` regions and count remaining standalone `` ` ``.
|
||||
If odd, append a closing inline-code backtick. Same trade-off: a
|
||||
stray closing backtick may produce a brief empty inline-code span,
|
||||
which is far less harmful than the rest of the message being rendered
|
||||
as inline code.
|
||||
|
||||
Returns:
|
||||
The input text with closing markers appended if needed, or the
|
||||
input text unchanged.
|
||||
"""
|
||||
if not isinstance(text, str) or not text:
|
||||
return text
|
||||
|
||||
# Step 1: fix triple-backtick code-block fences (existing logic)
|
||||
if text.count("```") % 2 == 1:
|
||||
text = text.rstrip("\n") + "\n```"
|
||||
|
||||
# Step 2: fix single-backtick inline-code spans
|
||||
# Remove complete ```…``` regions so their internal backticks don't
|
||||
# pollute the standalone count. Also remove any trailing unclosed
|
||||
# ``` that leaks through (defence in depth).
|
||||
import re
|
||||
without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
|
||||
without_fences = re.sub(r"```[^`]*$", "", without_fences)
|
||||
|
||||
if without_fences.count("`") % 2 == 1:
|
||||
text = text + "`"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamConsumerConfig:
|
||||
"""Runtime config for a single stream consumer instance."""
|
||||
|
|
@ -744,39 +808,76 @@ class GatewayStreamConsumer:
|
|||
and self._message_id is None
|
||||
):
|
||||
# No existing message to edit (first message or after a
|
||||
# segment break). Use truncate_message — the same
|
||||
# helper the non-streaming path uses — to split with
|
||||
# proper word/code-fence boundaries and chunk
|
||||
# indicators like "(1/2)".
|
||||
chunks = self.adapter.truncate_message(
|
||||
self._accumulated, _safe_limit, len_fn=_len_fn,
|
||||
# segment break). Seal only the overflowing head chunks
|
||||
# as fixed messages, then keep the trailing chunk in
|
||||
# _accumulated so the normal send/edit path below makes
|
||||
# it the active preview. That lets chunk 2, 3, ... keep
|
||||
# updating in-place as later streamed deltas arrive
|
||||
# instead of posting every split as an immutable message.
|
||||
chunks = self._truncate_for_stream(
|
||||
self._accumulated, _safe_limit, _len_fn,
|
||||
)
|
||||
if len(chunks) <= 1:
|
||||
# A malformed/legacy adapter result must not leave
|
||||
# this overflow branch with an unsplittable payload.
|
||||
chunks = self._split_text_chunks(
|
||||
self._accumulated, _safe_limit, _len_fn,
|
||||
)
|
||||
chunks_delivered = False
|
||||
reply_to = self._message_id or self._initial_reply_to_id
|
||||
for chunk in chunks:
|
||||
reply_to = self._initial_reply_to_id
|
||||
all_heads_delivered = len(chunks) > 1
|
||||
for chunk in chunks[:-1]:
|
||||
new_id = await self._send_new_chunk(
|
||||
chunk,
|
||||
reply_to,
|
||||
final=got_done,
|
||||
)
|
||||
if new_id is not None and new_id != reply_to:
|
||||
chunks_delivered = True
|
||||
self._accumulated = ""
|
||||
self._last_sent_text = ""
|
||||
if new_id is None or new_id == reply_to:
|
||||
# Failed to deliver a sealed head; keep the
|
||||
# full accumulated text intact so the gateway's
|
||||
# fallback path can still deliver it completely.
|
||||
all_heads_delivered = False
|
||||
chunks_delivered = False
|
||||
break
|
||||
chunks_delivered = True
|
||||
reply_to = new_id
|
||||
|
||||
if all_heads_delivered:
|
||||
self._accumulated = chunks[-1]
|
||||
# The head chunks are sealed. Clear the edit target
|
||||
# so the remaining tail is sent as a fresh active
|
||||
# chunk, then edited by subsequent deltas.
|
||||
self._message_id = None
|
||||
self._message_created_ts = None
|
||||
self._last_sent_text = ""
|
||||
else:
|
||||
# A prior head may have landed before a later head
|
||||
# failed. Do not edit that sealed message with the
|
||||
# unsplit full payload; let the fallback path retry.
|
||||
self._message_id = None
|
||||
self._message_created_ts = None
|
||||
self._last_sent_text = ""
|
||||
|
||||
self._last_edit_time = time.monotonic()
|
||||
if got_done:
|
||||
# Only claim final delivery if THESE chunks actually
|
||||
# landed. ``_already_sent`` may be True from prior
|
||||
# tool-progress edits or fallback-mode promotion (#10748)
|
||||
# — that doesn't mean the final answer reached the user.
|
||||
self._final_response_sent = chunks_delivered
|
||||
if chunks_delivered:
|
||||
tail_delivered = True
|
||||
if self._accumulated:
|
||||
tail_delivered = await self._send_or_edit(
|
||||
self._accumulated, finalize=True,
|
||||
)
|
||||
# Only claim final delivery if the sealed chunks and
|
||||
# final tail actually landed. ``_already_sent`` may
|
||||
# be True from prior progress/fallback state (#10748).
|
||||
self._final_response_sent = chunks_delivered and tail_delivered
|
||||
if self._final_response_sent:
|
||||
self._final_content_delivered = True
|
||||
return
|
||||
if got_segment_break:
|
||||
self._message_id = None
|
||||
self._fallback_final_send = False
|
||||
self._fallback_prefix = ""
|
||||
if not self._accumulated:
|
||||
continue
|
||||
|
||||
# This iteration consumed a _FLUSH barrier and delivered
|
||||
# the buffered prose via the chunk loop above, then takes
|
||||
|
|
@ -797,8 +898,8 @@ class GatewayStreamConsumer:
|
|||
self._accumulated, _safe_limit, _len_fn,
|
||||
)
|
||||
split_at = self._accumulated.rfind("\n", 0, _cp_budget)
|
||||
if split_at < _safe_limit // 2:
|
||||
split_at = _safe_limit
|
||||
if split_at < _cp_budget // 2:
|
||||
split_at = _cp_budget
|
||||
chunk = self._accumulated[:split_at]
|
||||
# finalize=True so the adapter applies platform-specific
|
||||
# rich-text markup (e.g. Telegram MarkdownV2). This
|
||||
|
|
@ -1074,26 +1175,103 @@ class GatewayStreamConsumer:
|
|||
return final_text[len(prefix):].lstrip()
|
||||
return final_text
|
||||
|
||||
@staticmethod
|
||||
def _balance_fences_across_chunks(chunks: "list[str]") -> "list[str]":
|
||||
"""Close orphaned ``` fences at each chunk boundary and reopen on the next.
|
||||
|
||||
When a split lands inside a triple-backtick code block, the head chunk
|
||||
would render everything after the orphaned fence as code, and the tail
|
||||
chunk's content would lose its code formatting. Mirror
|
||||
``BasePlatformAdapter.truncate_message``'s contract: close the fence at
|
||||
the end of the chunk and reopen it (with the original language tag) at
|
||||
the start of the next one, so EVERY delivered chunk is fence-balanced
|
||||
on its own.
|
||||
"""
|
||||
if len(chunks) <= 1:
|
||||
return chunks
|
||||
out: "list[str]" = []
|
||||
carry_lang: "Optional[str]" = None
|
||||
for chunk in chunks:
|
||||
prefix = f"```{carry_lang}\n" if carry_lang is not None else ""
|
||||
in_code = carry_lang is not None
|
||||
lang = carry_lang or ""
|
||||
for line in chunk.split("\n"):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
if in_code:
|
||||
in_code = False
|
||||
lang = ""
|
||||
else:
|
||||
in_code = True
|
||||
tag = stripped[3:].strip()
|
||||
lang = tag.split()[0] if tag else ""
|
||||
body = prefix + chunk
|
||||
if in_code:
|
||||
body += "\n```"
|
||||
carry_lang = lang
|
||||
else:
|
||||
carry_lang = None
|
||||
out.append(body)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _split_text_chunks(
|
||||
text: str, limit: int,
|
||||
text: str,
|
||||
limit: int,
|
||||
len_fn: "Callable[[str], int]" = len,
|
||||
) -> list[str]:
|
||||
"""Split text into reasonably sized chunks for fallback sends."""
|
||||
"""Split text into reasonably sized chunks for fallback sends.
|
||||
|
||||
Chunks are fence-balanced: a split inside a ``` code block closes the
|
||||
fence on the head chunk and reopens it on the tail, so no chunk leaves
|
||||
the rest of a message rendering as one giant code block.
|
||||
"""
|
||||
if len_fn(text) <= limit:
|
||||
return [text]
|
||||
# Reserve headroom for the close/reopen fence markers the balancing
|
||||
# pass may add, so balanced chunks stay within the platform limit.
|
||||
split_limit = limit
|
||||
if "```" in text:
|
||||
split_limit = max(limit - 16, limit // 2, 1)
|
||||
chunks: list[str] = []
|
||||
remaining = text
|
||||
while len_fn(remaining) > limit:
|
||||
_cp_budget = _custom_unit_to_cp(remaining, limit, len_fn)
|
||||
while len_fn(remaining) > split_limit:
|
||||
_cp_budget = _custom_unit_to_cp(remaining, split_limit, len_fn)
|
||||
split_at = remaining.rfind("\n", 0, _cp_budget)
|
||||
if split_at < limit // 2:
|
||||
split_at = limit
|
||||
if split_at < _cp_budget // 2:
|
||||
split_at = _cp_budget
|
||||
chunks.append(remaining[:split_at])
|
||||
remaining = remaining[split_at:].lstrip("\n")
|
||||
if remaining:
|
||||
chunks.append(remaining)
|
||||
return chunks
|
||||
return GatewayStreamConsumer._balance_fences_across_chunks(chunks)
|
||||
|
||||
def _truncate_for_stream(
|
||||
self,
|
||||
text: str,
|
||||
limit: int,
|
||||
len_fn: "Callable[[str], int]",
|
||||
) -> list[str]:
|
||||
"""Use the adapter's canonical splitter for streaming overflow.
|
||||
|
||||
Platform adapters may add word-boundary, code-fence, table, or
|
||||
platform-specific formatting rules. The consumer must not replace
|
||||
those rules with newline-only slicing. Non-base test doubles and
|
||||
legacy adapters retain the historical two-argument call shape.
|
||||
"""
|
||||
truncate = getattr(self.adapter, "truncate_message", None)
|
||||
if not callable(truncate):
|
||||
return self._split_text_chunks(text, limit, len_fn)
|
||||
|
||||
if isinstance(self.adapter, _BasePlatformAdapter):
|
||||
chunks = truncate(text, limit, len_fn=len_fn)
|
||||
else:
|
||||
chunks = truncate(text, limit)
|
||||
if not isinstance(chunks, (list, tuple)) or not all(
|
||||
isinstance(chunk, str) for chunk in chunks
|
||||
):
|
||||
return self._split_text_chunks(text, limit, len_fn)
|
||||
return list(chunks)
|
||||
|
||||
async def _send_fallback_final(self, text: str) -> None:
|
||||
"""Send the final continuation after streaming edits stop working.
|
||||
|
|
@ -1101,6 +1279,10 @@ class GatewayStreamConsumer:
|
|||
Retries each chunk once on flood-control failures with a short delay.
|
||||
"""
|
||||
final_text = self._clean_for_display(text)
|
||||
# Ensure balanced code fences before computing continuation,
|
||||
# so the closing fence reaches the user even when the fallback
|
||||
# only delivers the tail after mid-stream edits failed.
|
||||
final_text = ensure_closed_code_fences(final_text)
|
||||
continuation = self._continuation_text(final_text)
|
||||
self._fallback_final_send = False
|
||||
if not continuation.strip():
|
||||
|
|
@ -1758,6 +1940,12 @@ class GatewayStreamConsumer:
|
|||
# Media files are delivered as native attachments after the stream
|
||||
# finishes (via _deliver_media_from_response in gateway/run.py).
|
||||
text = self._clean_for_display(text)
|
||||
# Ensure code fences are balanced before send/edit. Model output
|
||||
# truncated mid-code-block (e.g. finish_reason="length") leaves an
|
||||
# orphaned ``` which, on Discord/Slack/Matrix, causes the entire
|
||||
# remaining output to render as a single code block. This covers
|
||||
# the streaming edit path (G2) and first-send path alike.
|
||||
text = ensure_closed_code_fences(text)
|
||||
# A bare streaming cursor is not meaningful user-visible content and
|
||||
# can render as a stray tofu/white-box message on some clients.
|
||||
visible_without_cursor = text
|
||||
|
|
|
|||
184
gateway/wake.py
Normal file
184
gateway/wake.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""Wake an existing agent session from a background completion event.
|
||||
|
||||
Two delivery strategies, selected by the target adapter's
|
||||
``supports_async_delivery`` capability flag:
|
||||
|
||||
* Push-capable adapters (telegram, discord, plugin platforms, ...): inject a
|
||||
synthetic ``MessageEvent(internal=True)`` through ``adapter.handle_message``
|
||||
— the pre-existing wake path, preserved exactly.
|
||||
|
||||
* Stateless request/response adapters (the API server,
|
||||
``supports_async_delivery = False``): ``handle_message`` would run the wake
|
||||
turn under a ``build_session_key()``-derived key
|
||||
(``agent:main:api_server:group:<sid>``) that NEVER matches the raw
|
||||
``X-Hermes-Session-Id`` key real gateway/HQ turns run under
|
||||
(``_bind_api_server_session``), so the wake lands in a parallel, invisible
|
||||
session. Instead we self-POST ``/v1/chat/completions`` on the in-pod API
|
||||
server with the raw session id in the ``X-Hermes-Session-Id`` header — the
|
||||
exact entry point real turns use — so the wake turn resumes the REAL
|
||||
session, with full history, and its result is visible the next time the
|
||||
client polls/reopens the conversation.
|
||||
|
||||
Failures RAISE (after bounded retries on transient errors) so callers can
|
||||
rewind cursors / retry instead of silently losing the event.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A wake self-post runs the entire agent turn synchronously (stream=false);
|
||||
# generous ceiling so long tool-using turns aren't killed mid-flight.
|
||||
WAKE_TURN_TIMEOUT_SECONDS = 600.0
|
||||
|
||||
# Backoff delays between retries on transient failures (429 concurrency cap,
|
||||
# connection errors). The API server has no per-session lock — concurrent
|
||||
# turns on one session are last-writer-wins — but it DOES enforce a global
|
||||
# max_concurrent_runs cap via HTTP 429, which is worth waiting out.
|
||||
_RETRY_DELAYS_SECONDS = (2.0, 5.0, 10.0)
|
||||
|
||||
|
||||
def adapter_supports_push(adapter: Any) -> bool:
|
||||
"""Whether this adapter can push a message to the user after a turn ends.
|
||||
|
||||
Mirrors ``gateway.session_context.async_delivery_supported`` but reads the
|
||||
capability off the adapter class (``supports_async_delivery``) instead of
|
||||
the request-scoped contextvar — background watchers run outside any bound
|
||||
session context. Adapters that don't declare the flag are push-capable.
|
||||
"""
|
||||
return bool(getattr(adapter, "supports_async_delivery", True))
|
||||
|
||||
|
||||
async def deliver_wake(
|
||||
adapter: Any,
|
||||
*,
|
||||
text: str,
|
||||
session_id: str = "",
|
||||
source: Any = None,
|
||||
) -> None:
|
||||
"""Deliver a wake turn to the session behind ``adapter``.
|
||||
|
||||
``session_id`` is the RAW session id (the ``X-Hermes-Session-Id`` value /
|
||||
``state.db`` key) — required for non-push adapters. ``source`` is the
|
||||
``SessionSource`` used to build the synthetic event — required for
|
||||
push-capable adapters.
|
||||
|
||||
Raises on failure (bad arguments, exhausted retries, HTTP error) so the
|
||||
caller can rewind/retry instead of treating the wake as delivered.
|
||||
"""
|
||||
if adapter_supports_push(adapter):
|
||||
if source is None:
|
||||
raise ValueError(
|
||||
"deliver_wake: push-capable adapter requires a SessionSource"
|
||||
)
|
||||
from gateway.platforms.base import MessageEvent, MessageType
|
||||
|
||||
synth_event = MessageEvent(
|
||||
text=text,
|
||||
message_type=MessageType.TEXT,
|
||||
source=source,
|
||||
internal=True,
|
||||
)
|
||||
await adapter.handle_message(synth_event)
|
||||
return
|
||||
|
||||
if not session_id:
|
||||
raise ValueError(
|
||||
"deliver_wake: non-push adapter (supports_async_delivery=False) "
|
||||
"requires the raw session id to self-post the wake turn"
|
||||
)
|
||||
await _self_post_chat_completion(adapter, text=text, session_id=session_id)
|
||||
|
||||
|
||||
async def _self_post_chat_completion(
|
||||
adapter: Any, *, text: str, session_id: str
|
||||
) -> None:
|
||||
"""POST the wake text to the in-pod API server as a normal session turn.
|
||||
|
||||
Uses the adapter's own bind host/port/key (``ApiServerAdapter.__init__``).
|
||||
Session continuation via ``X-Hermes-Session-Id`` is 403-gated on
|
||||
``API_SERVER_KEY`` being configured, so a missing key is a hard error —
|
||||
raise loudly rather than run the wake in a fresh fingerprint-derived
|
||||
session nobody is looking at.
|
||||
"""
|
||||
import aiohttp
|
||||
|
||||
host = str(getattr(adapter, "_host", "") or "127.0.0.1")
|
||||
if host in ("0.0.0.0", "::", "*"):
|
||||
# Wildcard bind address — connect over loopback.
|
||||
host = "127.0.0.1"
|
||||
port = int(getattr(adapter, "_port", 0) or 8642)
|
||||
api_key = str(getattr(adapter, "_api_key", "") or "")
|
||||
if not api_key:
|
||||
raise RuntimeError(
|
||||
"wake self-post requires API_SERVER_KEY: session continuation via "
|
||||
"X-Hermes-Session-Id is rejected (403) on an unauthenticated API "
|
||||
"server, so the wake cannot reach the target session"
|
||||
)
|
||||
|
||||
if ":" in host and not host.startswith("["):
|
||||
host = f"[{host}]" # bare IPv6 literal
|
||||
url = f"http://{host}:{port}/v1/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"X-Hermes-Session-Id": session_id,
|
||||
}
|
||||
payload = {
|
||||
"model": str(getattr(adapter, "_model_name", "") or "hermes-agent"),
|
||||
"messages": [{"role": "user", "content": text}],
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
last_err: Optional[BaseException] = None
|
||||
attempts = 1 + len(_RETRY_DELAYS_SECONDS)
|
||||
for attempt in range(attempts):
|
||||
if attempt:
|
||||
await asyncio.sleep(_RETRY_DELAYS_SECONDS[attempt - 1])
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=WAKE_TURN_TIMEOUT_SECONDS)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as http:
|
||||
async with http.post(url, json=payload, headers=headers) as resp:
|
||||
if resp.status == 429:
|
||||
# Global concurrency cap (max_concurrent_runs) —
|
||||
# transient; back off and retry.
|
||||
last_err = RuntimeError(
|
||||
f"wake self-post got HTTP 429 (concurrency cap) "
|
||||
f"for session {session_id}"
|
||||
)
|
||||
logger.warning(
|
||||
"%s; attempt %d/%d", last_err, attempt + 1, attempts
|
||||
)
|
||||
continue
|
||||
if resp.status >= 400:
|
||||
body = (await resp.text())[:300]
|
||||
# Non-transient (auth/validation) — fail immediately.
|
||||
raise RuntimeError(
|
||||
f"wake self-post failed for session {session_id}: "
|
||||
f"HTTP {resp.status}: {body}"
|
||||
)
|
||||
await resp.read()
|
||||
logger.info(
|
||||
"wake self-post delivered for session %s (attempt %d)",
|
||||
session_id,
|
||||
attempt + 1,
|
||||
)
|
||||
return
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as exc:
|
||||
last_err = exc
|
||||
logger.warning(
|
||||
"wake self-post transient failure for session %s "
|
||||
"(attempt %d/%d): %s",
|
||||
session_id,
|
||||
attempt + 1,
|
||||
attempts,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
raise RuntimeError(
|
||||
f"wake self-post gave up for session {session_id} after "
|
||||
f"{attempts} attempts: {last_err}"
|
||||
) from last_err
|
||||
226
hermes_cli/_early_recovery.py
Normal file
226
hermes_cli/_early_recovery.py
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
"""Dependency-light venv recovery that runs BEFORE hermes_cli.main's imports.
|
||||
|
||||
The ``hermes`` console entry point is ``hermes_cli.main:main``. Importing
|
||||
``hermes_cli.main`` pulls in third-party packages at module level (``dotenv``
|
||||
via ``hermes_cli.env_loader``, ``yaml`` via ``hermes_cli.config``, ...). In
|
||||
the exact failure state the update-recovery markers exist for — a failed lazy
|
||||
backend refresh or interrupted core install that wiped a core package's
|
||||
import files (#57828) — a normal launch crashes *while importing main.py*,
|
||||
before ``_recover_from_interrupted_install()`` can run. The marker system is
|
||||
unreachable precisely when it is needed most.
|
||||
|
||||
This module is deliberately **stdlib-only** so importing it can never fail on
|
||||
a corrupted venv. ``hermes_cli.main`` imports and calls
|
||||
:func:`recover_if_needed` at the very top of its module body, before any
|
||||
third-party import.
|
||||
|
||||
Scope: this early pass only repairs enough for ``hermes_cli.main`` to become
|
||||
importable again (force-reinstall of the known-fragile core packages, using
|
||||
the pins from pyproject.toml). It NEVER clears the recovery markers — the
|
||||
full, confirmed marker lifecycle stays with ``_recover_from_interrupted_install()``
|
||||
in main.py, which runs right after import succeeds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Core packages a failed lazy ``uv pip install`` is known to leave with intact
|
||||
# distribution metadata but wiped import files (#57828). ``module`` is what we
|
||||
# probe via a real import; ``attr`` guards against an empty/stub module.
|
||||
# main.py's marker-recovery path reuses these tables — keep them here (the
|
||||
# dependency-light module) so both layers probe and repair the same set.
|
||||
LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = (
|
||||
("yaml", "SafeDumper"),
|
||||
("dotenv", "load_dotenv"),
|
||||
("click", "Command"),
|
||||
("certifi", "contents"),
|
||||
("rich", "print"),
|
||||
("cryptography", "__version__"),
|
||||
("jwt", "encode"),
|
||||
)
|
||||
|
||||
LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = {
|
||||
"yaml": "PyYAML",
|
||||
"dotenv": "python-dotenv",
|
||||
"click": "click",
|
||||
"certifi": "certifi",
|
||||
"rich": "rich",
|
||||
"cryptography": "cryptography",
|
||||
"jwt": "PyJWT",
|
||||
}
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _pinned_specs(packages: list[str], project_root: Path) -> list[str]:
|
||||
"""Map bare package names to their pinned specs from pyproject.toml.
|
||||
|
||||
Stdlib-only (tomllib + naive requirement-head parsing — ``packaging`` may
|
||||
itself be broken in the failure state this module exists for). Unknown
|
||||
packages fall back to their bare name.
|
||||
"""
|
||||
pyproject = project_root / "pyproject.toml"
|
||||
if not pyproject.is_file():
|
||||
return packages
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
with open(pyproject, "rb") as f:
|
||||
raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or []
|
||||
except Exception:
|
||||
return packages
|
||||
|
||||
name_to_spec: dict[str, str] = {}
|
||||
for spec in raw_deps:
|
||||
head = spec.split(";", 1)[0].strip()
|
||||
bare = head
|
||||
for op in ("==", ">=", "<=", "~=", ">", "<", "!="):
|
||||
if op in bare:
|
||||
bare = bare.split(op, 1)[0]
|
||||
break
|
||||
key = bare.strip().split("[", 1)[0].strip().lower()
|
||||
if key:
|
||||
name_to_spec[key] = head
|
||||
return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages]
|
||||
|
||||
|
||||
def _probe_broken_packages() -> list[str]:
|
||||
"""Import-probe the fragile core packages in THIS process.
|
||||
|
||||
Returns repair package names (deduped, probe order) for modules that fail
|
||||
to import or lack their sentinel attribute. Failed imports leave nothing
|
||||
in ``sys.modules``, so a post-repair retry in the same process works.
|
||||
"""
|
||||
broken: list[str] = []
|
||||
for mod_name, attr in LAZY_REFRESH_IMPORT_PROBES:
|
||||
try:
|
||||
mod = importlib.import_module(mod_name)
|
||||
if not hasattr(mod, attr):
|
||||
raise ImportError(f"{mod_name} missing {attr}")
|
||||
except Exception:
|
||||
pkg = LAZY_REFRESH_REPAIR_PACKAGES.get(mod_name)
|
||||
if pkg and pkg not in broken:
|
||||
broken.append(pkg)
|
||||
return broken
|
||||
|
||||
|
||||
def _run_repair_install(specs: list[str], project_root: Path) -> bool:
|
||||
"""ensurepip + ``pip install --force-reinstall`` the given specs.
|
||||
|
||||
Streams nothing to stdout (``hermes acp`` speaks JSON-RPC on stdout);
|
||||
output is captured and replayed to stderr only on failure. Never raises.
|
||||
"""
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "--force-reinstall", *specs],
|
||||
cwd=project_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f" ✗ Early venv repair could not run pip: {exc}", file=sys.stderr)
|
||||
return False
|
||||
if result.returncode != 0:
|
||||
tail = (result.stderr or result.stdout or "")[-2000:]
|
||||
if tail:
|
||||
print(tail, file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def recover_if_needed(
|
||||
project_root: Path | None = None,
|
||||
argv: list[str] | None = None,
|
||||
) -> None:
|
||||
"""Repair wiped core packages so ``hermes_cli.main`` can import at all.
|
||||
|
||||
Fast path (no marker present) is two ``lstat`` calls. Only acts when a
|
||||
recovery marker from a prior ``hermes update`` exists AND an import probe
|
||||
confirms a core package is actually broken. Markers are intentionally
|
||||
NOT cleared here — ``_recover_from_interrupted_install()`` in main.py owns
|
||||
the confirmed marker lifecycle and runs immediately after import succeeds.
|
||||
|
||||
Never raises: on any failure the import of main.py proceeds and surfaces
|
||||
the real error.
|
||||
"""
|
||||
try:
|
||||
args = sys.argv[1:] if argv is None else argv
|
||||
# Same deliberately-loose match as main(): the real update flow writes
|
||||
# and clears its own markers — a recovery install must not race it.
|
||||
if "update" in args:
|
||||
return
|
||||
root = _project_root() if project_root is None else project_root
|
||||
core_marker = root / ".update-incomplete"
|
||||
lazy_marker = root / ".lazy-refresh-incomplete"
|
||||
if not core_marker.exists() and not lazy_marker.exists():
|
||||
return
|
||||
# Managed/Docker/PyPI installs have no source tree here — the marker
|
||||
# is not ours to act on; main.py's recovery clears it.
|
||||
if not (root / "pyproject.toml").is_file():
|
||||
return
|
||||
|
||||
broken = _probe_broken_packages()
|
||||
if not broken:
|
||||
# Imports are fine — main.py will load and run full recovery.
|
||||
return
|
||||
|
||||
# Single-flight: share main.py's recovery lock so an early repair
|
||||
# never races a concurrent full recovery into the same shared venv.
|
||||
lock_path = root / ".update-incomplete.lock"
|
||||
try:
|
||||
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
os.write(fd, f"{os.getpid()}\n".encode())
|
||||
os.close(fd)
|
||||
except FileExistsError:
|
||||
try:
|
||||
if time.time() - lock_path.stat().st_mtime > 3600:
|
||||
lock_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
except OSError:
|
||||
pass # read-only fs / perms — proceed unlocked, install surfaces it
|
||||
|
||||
try:
|
||||
specs = _pinned_specs(broken, root)
|
||||
print(
|
||||
"⚠ Core package(s) broken by an interrupted update — "
|
||||
f"repairing before launch: {', '.join(broken)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if _run_repair_install(specs, root) and not _probe_broken_packages():
|
||||
print(" ✓ Core packages repaired.", file=sys.stderr)
|
||||
else:
|
||||
print(
|
||||
" ✗ Automatic repair incomplete. Recover manually with:",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f" {sys.executable} -m pip install --force-reinstall "
|
||||
+ " ".join(specs),
|
||||
file=sys.stderr,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
lock_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
# Never block launch — the import of main.py will surface the truth.
|
||||
pass
|
||||
|
|
@ -10,8 +10,9 @@ Several common subprocess patterns break silently-or-loudly on Windows:
|
|||
|
||||
* ``start_new_session=True`` — on POSIX, this maps to ``os.setsid()`` and
|
||||
actually detaches the child. On Windows it's silently ignored; the
|
||||
Windows equivalent is ``CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS``
|
||||
creationflags, which Python only applies when you pass them explicitly.
|
||||
Windows equivalent is the ``CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW``
|
||||
creationflags bundle, which Python only applies when you pass it
|
||||
explicitly.
|
||||
|
||||
* Console-window flashes — every ``subprocess.Popen`` of a ``.exe`` on
|
||||
Windows spawns a cmd window briefly unless ``CREATE_NO_WINDOW`` is
|
||||
|
|
@ -98,7 +99,23 @@ def resolve_node_command(name: str, argv: Sequence[str]) -> list[str]:
|
|||
# because CREATE_NO_WINDOW and DETACHED_PROCESS aren't guaranteed to be
|
||||
# present on stdlib subprocess on older Pythons or non-Windows builds.
|
||||
_CREATE_NEW_PROCESS_GROUP = 0x00000200
|
||||
_DETACHED_PROCESS = 0x00000008
|
||||
# DETACHED_PROCESS is intentionally NOT part of any flag bundle here — do not
|
||||
# re-add it. Two reasons (the recurring console-flash bug #54220 / #56747):
|
||||
#
|
||||
# 1. MSDN (Process Creation Flags): CREATE_NO_WINDOW "is ignored if used with
|
||||
# either CREATE_NEW_CONSOLE or DETACHED_PROCESS". Combining them means
|
||||
# DETACHED_PROCESS governs and the no-window bit is dead.
|
||||
# 2. A DETACHED_PROCESS child has NO console at all, so every console-subsystem
|
||||
# descendant it ever spawns (git, gh, cmd, node, wmic, powershell, …) must
|
||||
# allocate its OWN console — a visible flash per spawn, including spawns
|
||||
# inside third-party libraries that no per-call-site CREATE_NO_WINDOW sweep
|
||||
# can reach. A CREATE_NO_WINDOW child instead OWNS a hidden console that
|
||||
# all descendants inherit, making "no flashing windows" a property of the
|
||||
# one daemon launch. Root cause isolated + A/B verified on Windows 11 by
|
||||
# the desktop backend fix (commit aa2ae36c3f): with per-site hide flags
|
||||
# neutered, naive git/gh/cmd spawns don't flash under a hidden-console
|
||||
# parent and do flash under a console-less one.
|
||||
_DETACHED_PROCESS = 0x00000008 # kept for reference; must stay out of bundles
|
||||
_CREATE_NO_WINDOW = 0x08000000
|
||||
# Escape any Win32 job object the parent process belongs to. Without this,
|
||||
# a detached child still inherits its parent's job object membership, and
|
||||
|
|
@ -114,7 +131,8 @@ _CREATE_BREAKAWAY_FROM_JOB = 0x01000000
|
|||
|
||||
def windows_detach_flags() -> int:
|
||||
"""Return Win32 creationflags that detach a child from the parent
|
||||
console and process group. 0 on non-Windows.
|
||||
console and process group without leaving it console-less. 0 on
|
||||
non-Windows.
|
||||
|
||||
Pair with ``start_new_session=False`` (default) when calling
|
||||
subprocess.Popen — on POSIX use ``start_new_session=True`` instead,
|
||||
|
|
@ -123,19 +141,23 @@ def windows_detach_flags() -> int:
|
|||
Rationale:
|
||||
- ``CREATE_NEW_PROCESS_GROUP`` — child has its own process group so
|
||||
Ctrl+C in the parent console doesn't propagate.
|
||||
- ``DETACHED_PROCESS`` — child has no console at all. Necessary for
|
||||
background daemons (gateway watchers, update respawners) because
|
||||
without it, closing the console kills the child.
|
||||
- ``CREATE_NO_WINDOW`` — suppress the brief cmd flash that would
|
||||
otherwise appear when launching a console app. Redundant with
|
||||
DETACHED_PROCESS but explicit for clarity.
|
||||
- ``CREATE_NO_WINDOW`` — the child gets its own fresh console that is
|
||||
never shown. This both detaches it from the parent's console
|
||||
lifetime (closing the launching terminal doesn't CTRL_CLOSE it) AND
|
||||
gives every console-subsystem descendant (git, gh, cmd, node, …) a
|
||||
console to inherit, so they don't allocate visible flashing ones.
|
||||
This deliberately replaces the old ``DETACHED_PROCESS`` approach:
|
||||
MSDN specifies CREATE_NO_WINDOW is *ignored* when combined with
|
||||
DETACHED_PROCESS, and a truly console-less daemon re-creates the
|
||||
per-descendant console-flash bug (#54220/#56747) at every spawn —
|
||||
see the note on ``_DETACHED_PROCESS`` above.
|
||||
- ``CREATE_BREAKAWAY_FROM_JOB`` — escape any job object the parent is
|
||||
in. Electron (Desktop app) and Tauri (bootstrap installer) wrap
|
||||
their children in job objects; without breakaway, those children
|
||||
die when the parent process exits even if they were spawned with
|
||||
DETACHED_PROCESS. This was the missing flag that made the
|
||||
post-update gateway respawn watcher silently die alongside the
|
||||
Tauri updater after the Electron Desktop's update flow finished.
|
||||
die when the parent process exits even though they have their own
|
||||
console. This was the missing flag that made the post-update
|
||||
gateway respawn watcher silently die alongside the Tauri updater
|
||||
after the Electron Desktop's update flow finished.
|
||||
|
||||
If a process is in a job that disallows breakaway (rare —
|
||||
JOB_OBJECT_LIMIT_BREAKAWAY_OK isn't set), CreateProcess returns
|
||||
|
|
@ -149,7 +171,6 @@ def windows_detach_flags() -> int:
|
|||
return 0
|
||||
return (
|
||||
_CREATE_NEW_PROCESS_GROUP
|
||||
| _DETACHED_PROCESS
|
||||
| _CREATE_NO_WINDOW
|
||||
| _CREATE_BREAKAWAY_FROM_JOB
|
||||
)
|
||||
|
|
@ -182,7 +203,7 @@ def windows_detach_flags_without_breakaway() -> int:
|
|||
"""
|
||||
if not IS_WINDOWS:
|
||||
return 0
|
||||
return _CREATE_NEW_PROCESS_GROUP | _DETACHED_PROCESS | _CREATE_NO_WINDOW
|
||||
return _CREATE_NEW_PROCESS_GROUP | _CREATE_NO_WINDOW
|
||||
|
||||
|
||||
def windows_hide_flags() -> int:
|
||||
|
|
@ -193,10 +214,12 @@ def windows_hide_flags() -> int:
|
|||
operation (``taskkill``, ``where``, version probes) where we want no
|
||||
flash but also want to collect stdout/exit code synchronously.
|
||||
|
||||
The key difference from :func:`windows_detach_flags`: NO
|
||||
``DETACHED_PROCESS`` — the child still inherits stdio handles so
|
||||
``capture_output=True`` works. ``DETACHED_PROCESS`` would sever
|
||||
stdio and break stdout capture.
|
||||
The difference from :func:`windows_detach_flags`: no
|
||||
``CREATE_NEW_PROCESS_GROUP`` / ``CREATE_BREAKAWAY_FROM_JOB`` — the
|
||||
child stays in the parent's process group and job so Ctrl+C and job
|
||||
teardown propagate normally, as a short-lived helper wants. Stdio
|
||||
handles are inherited either way, so ``capture_output=True`` works
|
||||
with both bundles.
|
||||
"""
|
||||
if not IS_WINDOWS:
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -1447,6 +1447,73 @@ def read_credential_pool(provider_id: Optional[str] = None) -> Dict[str, Any]:
|
|||
return list(global_entries) if isinstance(global_entries, list) else []
|
||||
|
||||
|
||||
_POOL_STATUS_FIELDS = (
|
||||
"last_status",
|
||||
"last_status_at",
|
||||
"last_error_code",
|
||||
"last_error_reason",
|
||||
"last_error_message",
|
||||
"last_error_reset_at",
|
||||
)
|
||||
|
||||
|
||||
def _merge_disk_cooldown_state(
|
||||
entry: Dict[str, Any],
|
||||
disk_entry: Optional[Dict[str, Any]],
|
||||
provider_id: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Keep a newer on-disk cooldown/quarantine over a stale in-memory one.
|
||||
|
||||
``write_credential_pool`` callers persist an in-memory snapshot that may
|
||||
predate another process marking the same credential exhausted or dead
|
||||
(last-writer-wins lost update). Without this merge, process B's later
|
||||
rewrite resurrects a rate-limited key as healthy and both processes
|
||||
resume hammering it. Adopt the on-disk status fields only when they are
|
||||
strictly more recent (by ``last_status_at``) AND still binding — a DEAD
|
||||
marker, or an EXHAUSTED cooldown that has not yet expired. Expired
|
||||
cooldowns are not resurrected, so the pool's own expiry-clear (which
|
||||
resets ``last_status_at`` to None) is never overridden.
|
||||
"""
|
||||
if not isinstance(disk_entry, dict):
|
||||
return entry
|
||||
try:
|
||||
from agent.credential_pool import (
|
||||
PooledCredential,
|
||||
STATUS_DEAD,
|
||||
STATUS_EXHAUSTED,
|
||||
_exhausted_until,
|
||||
_parse_absolute_timestamp,
|
||||
)
|
||||
|
||||
disk_status = disk_entry.get("last_status")
|
||||
if disk_status not in (STATUS_DEAD, STATUS_EXHAUSTED):
|
||||
return entry
|
||||
# A token change means the caller re-authed/refreshed this entry and
|
||||
# intentionally cleared its status (e.g. _sync_codex_entry_from_
|
||||
# auth_store after a fresh device-code login) — never resurrect the
|
||||
# old cooldown onto fresh credentials.
|
||||
mem_access = entry.get("access_token") or ""
|
||||
disk_access = disk_entry.get("access_token") or ""
|
||||
if mem_access and disk_access and mem_access != disk_access:
|
||||
return entry
|
||||
disk_ts = _parse_absolute_timestamp(disk_entry.get("last_status_at")) or 0.0
|
||||
mem_ts = _parse_absolute_timestamp(entry.get("last_status_at")) or 0.0
|
||||
if disk_ts <= mem_ts:
|
||||
return entry
|
||||
if disk_status == STATUS_EXHAUSTED:
|
||||
until = _exhausted_until(
|
||||
PooledCredential.from_dict(provider_id, disk_entry)
|
||||
)
|
||||
if until is None or until <= time.time():
|
||||
return entry
|
||||
merged_entry = dict(entry)
|
||||
for status_field in _POOL_STATUS_FIELDS:
|
||||
merged_entry[status_field] = disk_entry.get(status_field)
|
||||
return merged_entry
|
||||
except Exception: # pragma: no cover - best-effort merge
|
||||
return entry
|
||||
|
||||
|
||||
def write_credential_pool(
|
||||
provider_id: str,
|
||||
entries: List[Dict[str, Any]],
|
||||
|
|
@ -1464,6 +1531,10 @@ def write_credential_pool(
|
|||
the caller loaded its in-memory snapshot; without this merge a later
|
||||
rotation/exhaustion rewrite drops the concurrent credential.
|
||||
|
||||
For entries present on BOTH sides, status fields are merged by
|
||||
``last_status_at`` recency via ``_merge_disk_cooldown_state`` so a stale
|
||||
snapshot cannot erase a cooldown/quarantine another process just wrote.
|
||||
|
||||
Pass ``removed_ids`` for entries the caller intentionally removed, so the
|
||||
merge does not resurrect them from the on-disk copy.
|
||||
"""
|
||||
|
|
@ -1481,12 +1552,24 @@ def write_credential_pool(
|
|||
]
|
||||
existing = pool.get(provider_id)
|
||||
existing_list = existing if isinstance(existing, list) else []
|
||||
existing_by_id = {
|
||||
entry.get("id"): entry
|
||||
for entry in existing_list
|
||||
if isinstance(entry, dict) and entry.get("id")
|
||||
}
|
||||
new_ids = {
|
||||
entry.get("id")
|
||||
for entry in sanitized_entries
|
||||
if isinstance(entry, dict) and entry.get("id")
|
||||
}
|
||||
merged: List[Dict[str, Any]] = list(sanitized_entries)
|
||||
merged: List[Dict[str, Any]] = [
|
||||
_merge_disk_cooldown_state(
|
||||
entry, existing_by_id.get(entry.get("id")), provider_id
|
||||
)
|
||||
if isinstance(entry, dict)
|
||||
else entry
|
||||
for entry in sanitized_entries
|
||||
]
|
||||
for disk_entry in existing_list:
|
||||
if not isinstance(disk_entry, dict):
|
||||
continue
|
||||
|
|
@ -1588,7 +1671,7 @@ def is_provider_explicitly_configured(provider_id: str) -> bool:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Check config.yaml model.provider
|
||||
# 2. Check config.yaml model.provider and other explicit provider slots.
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config()
|
||||
|
|
@ -1597,6 +1680,37 @@ def is_provider_explicitly_configured(provider_id: str) -> bool:
|
|||
cfg_provider = (model_cfg.get("provider") or "").strip().lower()
|
||||
if cfg_provider == normalized:
|
||||
return True
|
||||
|
||||
# MoA presets are explicit model selections too. A user who configured
|
||||
# ``provider: anthropic`` as a MoA advisor/aggregator has opted Hermes
|
||||
# into using Anthropic credentials for that slot even when the main
|
||||
# session model is another provider. Without this, Claude Code OAuth
|
||||
# entries are pruned/ignored by credential_pool.load_pool("anthropic"),
|
||||
# so MoA Anthropic advisors fail with "no ANTHROPIC_API_KEY" while the
|
||||
# normal model picker says Anthropic is logged in.
|
||||
def _slot_matches_provider(slot):
|
||||
return (
|
||||
isinstance(slot, dict)
|
||||
and (slot.get("provider") or "").strip().lower() == normalized
|
||||
)
|
||||
|
||||
moa_cfg = cfg.get("moa")
|
||||
if isinstance(moa_cfg, dict):
|
||||
for slot in moa_cfg.get("reference_models") or []:
|
||||
if _slot_matches_provider(slot):
|
||||
return True
|
||||
if _slot_matches_provider(moa_cfg.get("aggregator")):
|
||||
return True
|
||||
presets = moa_cfg.get("presets")
|
||||
if isinstance(presets, dict):
|
||||
for preset in presets.values():
|
||||
if not isinstance(preset, dict):
|
||||
continue
|
||||
for slot in preset.get("reference_models") or []:
|
||||
if _slot_matches_provider(slot):
|
||||
return True
|
||||
if _slot_matches_provider(preset.get("aggregator")):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -503,13 +503,21 @@ class CLIAgentSetupMixin:
|
|||
if resolved_meta:
|
||||
session_meta = resolved_meta
|
||||
|
||||
restored = self._session_db.get_messages_as_conversation(
|
||||
self.session_id, repair_alternation=True
|
||||
)
|
||||
model_history, display_history = self._session_db.get_resume_conversations(self.session_id)
|
||||
restored = model_history
|
||||
if restored:
|
||||
restored = [m for m in restored if m.get("role") != "session_meta"]
|
||||
self.conversation_history = restored
|
||||
msg_count = len([m for m in restored if m.get("role") == "user"])
|
||||
self._resume_display_history = [
|
||||
m for m in display_history if m.get("role") != "session_meta"
|
||||
]
|
||||
msg_count = len(
|
||||
[
|
||||
m
|
||||
for m in self._resume_display_history
|
||||
if m.get("role") == "user" and not m.get("display_kind")
|
||||
]
|
||||
)
|
||||
title_part = ""
|
||||
if session_meta.get("title"):
|
||||
title_part = f' "{session_meta["title"]}"'
|
||||
|
|
@ -552,7 +560,8 @@ class CLIAgentSetupMixin:
|
|||
"""
|
||||
from cli import CLI_CONFIG, _record_output_history_entry, _strip_reasoning_tags, _suspend_output_history
|
||||
from tools.ansi_strip import sanitize_display_text as _sanitize_display_text
|
||||
if not self.conversation_history:
|
||||
display_history = getattr(self, "_resume_display_history", self.conversation_history)
|
||||
if not display_history:
|
||||
return
|
||||
|
||||
# Check config: resume_display setting
|
||||
|
|
@ -571,11 +580,21 @@ class CLIAgentSetupMixin:
|
|||
entries = [] # list of (role, display_text)
|
||||
_last_asst_idx = None # index of last assistant entry
|
||||
_last_asst_full = None # un-truncated display text for last assistant
|
||||
for msg in self.conversation_history:
|
||||
for msg in display_history:
|
||||
role = msg.get("role", "")
|
||||
display_kind = msg.get("display_kind")
|
||||
content = msg.get("content")
|
||||
tool_calls = msg.get("tool_calls") or []
|
||||
|
||||
if display_kind == "hidden":
|
||||
continue
|
||||
if display_kind == "model_switch":
|
||||
entries.append(("event", "model changed"))
|
||||
continue
|
||||
if display_kind == "async_delegation_complete":
|
||||
entries.append(("event", "background delegation completed"))
|
||||
continue
|
||||
|
||||
if role == "system":
|
||||
continue
|
||||
if role == "tool":
|
||||
|
|
@ -682,7 +701,9 @@ class CLIAgentSetupMixin:
|
|||
)
|
||||
|
||||
for i, (role, text) in enumerate(entries):
|
||||
if role == "user":
|
||||
if role == "event":
|
||||
lines.append(f" ◈ {text}\n", style="dim italic")
|
||||
elif role == "user":
|
||||
lines.append(" ● You: ", style=f"dim bold {_session_label_c}")
|
||||
# Show first line inline, indent rest
|
||||
msg_lines = text.splitlines()
|
||||
|
|
|
|||
|
|
@ -780,11 +780,20 @@ class CLICommandsMixin:
|
|||
# becomes ``self.conversation_history`` for subsequent turns. Heal a
|
||||
# durable ``user;user`` violation once here instead of re-firing the
|
||||
# pre-request repair on every request for the rest of the session.
|
||||
restored = self._session_db.get_messages_as_conversation(
|
||||
target_id, repair_alternation=True
|
||||
#
|
||||
# Both projections come from one lineage SELECT: model_history is
|
||||
# alternation-repaired for live replay; display_history is the full
|
||||
# lineage verbatim, used by _display_resumed_history() so timeline
|
||||
# events and ancestor rows render correctly (matching the startup
|
||||
# --resume path in _preload_resumed_session).
|
||||
model_history, display_history = self._session_db.get_resume_conversations(
|
||||
target_id
|
||||
)
|
||||
restored = [m for m in (restored or []) if m.get("role") != "session_meta"]
|
||||
restored = [m for m in (model_history or []) if m.get("role") != "session_meta"]
|
||||
self.conversation_history = restored
|
||||
self._resume_display_history = [
|
||||
m for m in (display_history or []) if m.get("role") != "session_meta"
|
||||
]
|
||||
|
||||
# Re-open the target session so it's not marked as ended
|
||||
try:
|
||||
|
|
@ -824,7 +833,7 @@ class CLICommandsMixin:
|
|||
pass
|
||||
|
||||
title_part = f" \"{session_meta['title']}\"" if session_meta.get("title") else ""
|
||||
msg_count = len([m for m in self.conversation_history if m.get("role") == "user"])
|
||||
msg_count = len([m for m in self._resume_display_history if m.get("role") == "user" and not m.get("display_kind")])
|
||||
if self.conversation_history:
|
||||
_cprint(
|
||||
f" ↻ Resumed session {target_id}{title_part}"
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue