From 885d1242a265e1a373311dbc1899c569dac56646 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 17:41:09 -0700 Subject: [PATCH 001/142] refactor(run_agent): extract message sanitization to agent/message_sanitization.py Pull the 10 pure sanitization/repair helpers (\_sanitize_surrogates, \_sanitize_structure_surrogates, \_sanitize_messages_surrogates, \_escape_invalid_chars_in_json_strings, \_repair_tool_call_arguments, \_strip_non_ascii, \_sanitize_messages_non_ascii, \_sanitize_tools_non_ascii, \_strip_images_from_messages, \_sanitize_structure_non_ascii) and the \_SURROGATE_RE constant out of run_agent.py into a new module. These are stateless byte-walking helpers with no AIAgent dependency. Backward compatibility: run_agent re-exports every name via a single import block, so existing 'from run_agent import _sanitize_surrogates' imports in tests and cli.py keep working unchanged. Same pattern the file already uses for _summarize_user_message_for_log (codex_responses_adapter). run_agent.py: 16077 -> 15682 lines (-395). --- agent/message_sanitization.py | 444 ++++++++++++++++++++++++++++++++++ run_agent.py | 427 +------------------------------- 2 files changed, 457 insertions(+), 414 deletions(-) create mode 100644 agent/message_sanitization.py diff --git a/agent/message_sanitization.py b/agent/message_sanitization.py new file mode 100644 index 00000000000..ff53d247a84 --- /dev/null +++ b/agent/message_sanitization.py @@ -0,0 +1,444 @@ +"""Message and tool-payload sanitization helpers. + +Pure functions extracted from ``run_agent.py`` so the AIAgent module can +stay focused on the conversation loop. These walk OpenAI-format message +lists and structured payloads, repairing or stripping problematic +characters that would otherwise crash ``json.dumps`` inside the OpenAI +SDK or be rejected by upstream APIs. + +All helpers are stateless and side-effect-free except for in-place +mutation of their input (where documented). Backward-compatible +re-exports from ``run_agent`` remain in place so existing imports +``from run_agent import _sanitize_surrogates`` keep working. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any + +logger = logging.getLogger(__name__) + +# Lone surrogate code points are invalid in UTF-8 and crash json.dumps +# inside the OpenAI SDK. Used by every surrogate-sanitization helper +# below as well as by run_agent and the CLI for paste-from-clipboard +# scrubbing. +_SURROGATE_RE = re.compile(r'[\ud800-\udfff]') + + +def _sanitize_surrogates(text: str) -> str: + """Replace lone surrogate code points with U+FFFD (replacement character). + + Surrogates are invalid in UTF-8 and will crash ``json.dumps()`` inside the + OpenAI SDK. This is a fast no-op when the text contains no surrogates. + """ + if _SURROGATE_RE.search(text): + return _SURROGATE_RE.sub('\ufffd', text) + return text + + +def _sanitize_structure_surrogates(payload: Any) -> bool: + """Replace surrogate code points in nested dict/list payloads in-place. + + Mirror of ``_sanitize_structure_non_ascii`` but for surrogate recovery. + Used to scrub nested structured fields (e.g. ``reasoning_details`` — an + array of dicts with ``summary``/``text`` strings) that flat per-field + checks don't reach. Returns True if any surrogates were replaced. + """ + found = False + + def _walk(node): + nonlocal found + if isinstance(node, dict): + for key, value in node.items(): + if isinstance(value, str): + if _SURROGATE_RE.search(value): + node[key] = _SURROGATE_RE.sub('\ufffd', value) + found = True + elif isinstance(value, (dict, list)): + _walk(value) + elif isinstance(node, list): + for idx, value in enumerate(node): + if isinstance(value, str): + if _SURROGATE_RE.search(value): + node[idx] = _SURROGATE_RE.sub('\ufffd', value) + found = True + elif isinstance(value, (dict, list)): + _walk(value) + + _walk(payload) + return found + + +def _sanitize_messages_surrogates(messages: list) -> bool: + """Sanitize surrogate characters from all string content in a messages list. + + Walks message dicts in-place. Returns True if any surrogates were found + and replaced, False otherwise. Covers content/text, name, tool call + metadata/arguments, AND any additional string or nested structured fields + (``reasoning``, ``reasoning_content``, ``reasoning_details``, etc.) so + retries don't fail on a non-content field. Byte-level reasoning models + (xiaomi/mimo, kimi, glm) can emit lone surrogates in reasoning output + that flow through to ``api_messages["reasoning_content"]`` on the next + turn and crash json.dumps inside the OpenAI SDK. + """ + found = False + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("content") + if isinstance(content, str) and _SURROGATE_RE.search(content): + msg["content"] = _SURROGATE_RE.sub('\ufffd', content) + found = True + elif isinstance(content, list): + for part in content: + if isinstance(part, dict): + text = part.get("text") + if isinstance(text, str) and _SURROGATE_RE.search(text): + part["text"] = _SURROGATE_RE.sub('\ufffd', text) + found = True + name = msg.get("name") + if isinstance(name, str) and _SURROGATE_RE.search(name): + msg["name"] = _SURROGATE_RE.sub('\ufffd', name) + found = True + tool_calls = msg.get("tool_calls") + if isinstance(tool_calls, list): + for tc in tool_calls: + if not isinstance(tc, dict): + continue + tc_id = tc.get("id") + if isinstance(tc_id, str) and _SURROGATE_RE.search(tc_id): + tc["id"] = _SURROGATE_RE.sub('\ufffd', tc_id) + found = True + fn = tc.get("function") + if isinstance(fn, dict): + fn_name = fn.get("name") + if isinstance(fn_name, str) and _SURROGATE_RE.search(fn_name): + fn["name"] = _SURROGATE_RE.sub('\ufffd', fn_name) + found = True + fn_args = fn.get("arguments") + if isinstance(fn_args, str) and _SURROGATE_RE.search(fn_args): + fn["arguments"] = _SURROGATE_RE.sub('\ufffd', fn_args) + found = True + # Walk any additional string / nested fields (reasoning, + # reasoning_content, reasoning_details, etc.) — surrogates from + # byte-level reasoning models (xiaomi/mimo, kimi, glm) can lurk + # in these fields and aren't covered by the per-field checks above. + # Matches _sanitize_messages_non_ascii's coverage (PR #10537). + for key, value in msg.items(): + if key in {"content", "name", "tool_calls", "role"}: + continue + if isinstance(value, str): + if _SURROGATE_RE.search(value): + msg[key] = _SURROGATE_RE.sub('\ufffd', value) + found = True + elif isinstance(value, (dict, list)): + if _sanitize_structure_surrogates(value): + found = True + return found + + +def _escape_invalid_chars_in_json_strings(raw: str) -> str: + """Escape unescaped control chars inside JSON string values. + + Walks the raw JSON character-by-character, tracking whether we are + inside a double-quoted string. Inside strings, replaces literal + control characters (0x00-0x1F) that aren't already part of an escape + sequence with their ``\\uXXXX`` equivalents. Pass-through for everything + else. + + Ported from #12093 — complements the other repair passes in + ``_repair_tool_call_arguments`` when ``json.loads(strict=False)`` is + not enough (e.g. llama.cpp backends that emit literal apostrophes or + tabs alongside other malformations). + """ + out: list[str] = [] + in_string = False + i = 0 + n = len(raw) + while i < n: + ch = raw[i] + if in_string: + if ch == "\\" and i + 1 < n: + # Already-escaped char — pass through as-is + out.append(ch) + out.append(raw[i + 1]) + i += 2 + continue + if ch == '"': + in_string = False + out.append(ch) + elif ord(ch) < 0x20: + out.append(f"\\u{ord(ch):04x}") + else: + out.append(ch) + else: + if ch == '"': + in_string = True + out.append(ch) + i += 1 + return "".join(out) + + +def _repair_tool_call_arguments(raw_args: str, tool_name: str = "?") -> str: + """Attempt to repair malformed tool_call argument JSON. + + Models like GLM-5.1 via Ollama can produce truncated JSON, trailing + commas, Python ``None``, etc. The API proxy rejects these with HTTP 400 + "invalid tool call arguments". This function applies common repairs; + if all fail it returns ``"{}"`` so the request succeeds (better than + crashing the session). All repairs are logged at WARNING level. + """ + raw_stripped = raw_args.strip() if isinstance(raw_args, str) else "" + + # Fast-path: empty / whitespace-only -> empty object + if not raw_stripped: + logger.warning("Sanitized empty tool_call arguments for %s", tool_name) + return "{}" + + # Python-literal None -> normalise to {} + if raw_stripped == "None": + logger.warning("Sanitized Python-None tool_call arguments for %s", tool_name) + return "{}" + + # Repair pass 0: llama.cpp backends sometimes emit literal control + # characters (tabs, newlines) inside JSON string values. json.loads + # with strict=False accepts these and lets us re-serialise the + # result into wire-valid JSON without any string surgery. This is + # the most common local-model repair case (#12068). + try: + parsed = json.loads(raw_stripped, strict=False) + reserialised = json.dumps(parsed, separators=(",", ":")) + if reserialised != raw_stripped: + logger.warning( + "Repaired unescaped control chars in tool_call arguments for %s", + tool_name, + ) + return reserialised + except (json.JSONDecodeError, TypeError, ValueError): + pass + + # Attempt common JSON repairs + fixed = raw_stripped + # 1. Strip trailing commas before } or ] + fixed = re.sub(r',\s*([}\]])', r'\1', fixed) + # 2. Close unclosed structures + open_curly = fixed.count('{') - fixed.count('}') + open_bracket = fixed.count('[') - fixed.count(']') + if open_curly > 0: + fixed += '}' * open_curly + if open_bracket > 0: + fixed += ']' * open_bracket + # 3. Remove excess closing braces/brackets (bounded to 50 iterations) + for _ in range(50): + try: + json.loads(fixed) + break + except json.JSONDecodeError: + if fixed.endswith('}') and fixed.count('}') > fixed.count('{'): + fixed = fixed[:-1] + elif fixed.endswith(']') and fixed.count(']') > fixed.count('['): + fixed = fixed[:-1] + else: + break + + try: + json.loads(fixed) + logger.warning( + "Repaired malformed tool_call arguments for %s: %s → %s", + tool_name, raw_stripped[:80], fixed[:80], + ) + return fixed + except json.JSONDecodeError: + pass + + # Repair pass 4: escape unescaped control chars inside JSON strings, + # then retry. Catches cases where strict=False alone fails because + # other malformations are present too. + try: + escaped = _escape_invalid_chars_in_json_strings(fixed) + if escaped != fixed: + json.loads(escaped) + logger.warning( + "Repaired control-char-laced tool_call arguments for %s: %s → %s", + tool_name, raw_stripped[:80], escaped[:80], + ) + return escaped + except (json.JSONDecodeError, TypeError, ValueError): + pass + + # Last resort: replace with empty object so the API request doesn't + # crash the entire session. + logger.warning( + "Unrepairable tool_call arguments for %s — " + "replaced with empty object (was: %s)", + tool_name, raw_stripped[:80], + ) + return "{}" + + +def _strip_non_ascii(text: str) -> str: + """Remove non-ASCII characters, replacing with closest ASCII equivalent or removing. + + Used as a last resort when the system encoding is ASCII and can't handle + any non-ASCII characters (e.g. LANG=C on Chromebooks). + """ + return text.encode('ascii', errors='ignore').decode('ascii') + + +def _sanitize_messages_non_ascii(messages: list) -> bool: + """Strip non-ASCII characters from all string content in a messages list. + + This is a last-resort recovery for systems with ASCII-only encoding + (LANG=C, Chromebooks, minimal containers). Returns True if any + non-ASCII content was found and sanitized. + """ + found = False + for msg in messages: + if not isinstance(msg, dict): + continue + # Sanitize content (string) + content = msg.get("content") + if isinstance(content, str): + sanitized = _strip_non_ascii(content) + if sanitized != content: + msg["content"] = sanitized + found = True + elif isinstance(content, list): + for part in content: + if isinstance(part, dict): + text = part.get("text") + if isinstance(text, str): + sanitized = _strip_non_ascii(text) + if sanitized != text: + part["text"] = sanitized + found = True + # Sanitize name field (can contain non-ASCII in tool results) + name = msg.get("name") + if isinstance(name, str): + sanitized = _strip_non_ascii(name) + if sanitized != name: + msg["name"] = sanitized + found = True + # Sanitize tool_calls + tool_calls = msg.get("tool_calls") + if isinstance(tool_calls, list): + for tc in tool_calls: + if isinstance(tc, dict): + fn = tc.get("function", {}) + if isinstance(fn, dict): + fn_args = fn.get("arguments") + if isinstance(fn_args, str): + sanitized = _strip_non_ascii(fn_args) + if sanitized != fn_args: + fn["arguments"] = sanitized + found = True + # Sanitize any additional top-level string fields (e.g. reasoning_content) + for key, value in msg.items(): + if key in {"content", "name", "tool_calls", "role"}: + continue + if isinstance(value, str): + sanitized = _strip_non_ascii(value) + if sanitized != value: + msg[key] = sanitized + found = True + return found + + +def _sanitize_tools_non_ascii(tools: list) -> bool: + """Strip non-ASCII characters from tool payloads in-place.""" + return _sanitize_structure_non_ascii(tools) + + +def _strip_images_from_messages(messages: list) -> bool: + """Remove image_url content parts from all messages in-place. + + Called when a server signals it does not support images (e.g. + "Only 'text' content type is supported."). Mutates messages so the + next API call sends text only. + + Preserves message alternation invariants: + * ``tool``-role messages whose content was entirely images are replaced + with a plaintext placeholder, NOT deleted — deleting them would leave + the paired ``tool_call_id`` on the prior assistant message unmatched, + which providers reject with HTTP 400. + * Non-tool messages whose content becomes empty are dropped. In + practice this only hits synthetic image-only user messages appended + for attachment delivery; real user turns always include text. + + Returns True if any image parts were removed. + """ + found = False + to_delete = [] + for i, msg in enumerate(messages): + if not isinstance(msg, dict): + continue + content = msg.get("content") + if not isinstance(content, list): + continue + new_parts = [] + for part in content: + if isinstance(part, dict) and part.get("type") in {"image_url", "image", "input_image"}: + found = True + else: + new_parts.append(part) + if len(new_parts) < len(content): + if new_parts: + msg["content"] = new_parts + elif msg.get("role") == "tool": + # Preserve tool_call_id linkage — providers require every + # assistant tool_call to have a matching tool response. + msg["content"] = "[image content removed — server does not support images]" + else: + # Synthetic image-only user/assistant message with no text; + # safe to drop. + to_delete.append(i) + for i in reversed(to_delete): + del messages[i] + return found + + +def _sanitize_structure_non_ascii(payload: Any) -> bool: + """Strip non-ASCII characters from nested dict/list payloads in-place.""" + found = False + + def _walk(node): + nonlocal found + if isinstance(node, dict): + for key, value in node.items(): + if isinstance(value, str): + sanitized = _strip_non_ascii(value) + if sanitized != value: + node[key] = sanitized + found = True + elif isinstance(value, (dict, list)): + _walk(value) + elif isinstance(node, list): + for idx, value in enumerate(node): + if isinstance(value, str): + sanitized = _strip_non_ascii(value) + if sanitized != value: + node[idx] = sanitized + found = True + elif isinstance(value, (dict, list)): + _walk(value) + + _walk(payload) + return found + + +__all__ = [ + "_SURROGATE_RE", + "_sanitize_surrogates", + "_sanitize_structure_surrogates", + "_sanitize_messages_surrogates", + "_escape_invalid_chars_in_json_strings", + "_repair_tool_call_arguments", + "_strip_non_ascii", + "_sanitize_messages_non_ascii", + "_sanitize_tools_non_ascii", + "_strip_images_from_messages", + "_sanitize_structure_non_ascii", +] diff --git a/run_agent.py b/run_agent.py index 325e1e13ef3..b4d88f59eff 100644 --- a/run_agent.py +++ b/run_agent.py @@ -189,6 +189,19 @@ from agent.trajectory import ( convert_scratchpad_to_think, has_incomplete_scratchpad, save_trajectory as _save_trajectory_to_file, ) +from agent.message_sanitization import ( + _SURROGATE_RE, + _sanitize_surrogates, + _sanitize_structure_surrogates, + _sanitize_messages_surrogates, + _escape_invalid_chars_in_json_strings, + _repair_tool_call_arguments, + _strip_non_ascii, + _sanitize_messages_non_ascii, + _sanitize_tools_non_ascii, + _strip_images_from_messages, + _sanitize_structure_non_ascii, +) from utils import atomic_json_write, base_url_host_matches, base_url_hostname, env_var_enabled, normalize_proxy_url from hermes_cli.config import cfg_get @@ -465,12 +478,6 @@ def _paths_overlap(left: Path, right: Path) -> bool: return left_parts[:common_len] == right_parts[:common_len] - -_SURROGATE_RE = re.compile(r'[\ud800-\udfff]') - - - - def _is_multimodal_tool_result(value: Any) -> bool: """True if the value is a multimodal tool result envelope. @@ -617,414 +624,6 @@ def _trajectory_normalize_msg(msg: Dict[str, Any]) -> Dict[str, Any]: return msg -def _sanitize_surrogates(text: str) -> str: - """Replace lone surrogate code points with U+FFFD (replacement character). - - Surrogates are invalid in UTF-8 and will crash ``json.dumps()`` inside the - OpenAI SDK. This is a fast no-op when the text contains no surrogates. - """ - if _SURROGATE_RE.search(text): - return _SURROGATE_RE.sub('\ufffd', text) - return text - - -# _summarize_user_message_for_log is imported from agent.codex_responses_adapter -# (see import block above). Remains importable from run_agent for backward compat. - - -def _sanitize_structure_surrogates(payload: Any) -> bool: - """Replace surrogate code points in nested dict/list payloads in-place. - - Mirror of ``_sanitize_structure_non_ascii`` but for surrogate recovery. - Used to scrub nested structured fields (e.g. ``reasoning_details`` — an - array of dicts with ``summary``/``text`` strings) that flat per-field - checks don't reach. Returns True if any surrogates were replaced. - """ - found = False - - def _walk(node): - nonlocal found - if isinstance(node, dict): - for key, value in node.items(): - if isinstance(value, str): - if _SURROGATE_RE.search(value): - node[key] = _SURROGATE_RE.sub('\ufffd', value) - found = True - elif isinstance(value, (dict, list)): - _walk(value) - elif isinstance(node, list): - for idx, value in enumerate(node): - if isinstance(value, str): - if _SURROGATE_RE.search(value): - node[idx] = _SURROGATE_RE.sub('\ufffd', value) - found = True - elif isinstance(value, (dict, list)): - _walk(value) - - _walk(payload) - return found - - -def _sanitize_messages_surrogates(messages: list) -> bool: - """Sanitize surrogate characters from all string content in a messages list. - - Walks message dicts in-place. Returns True if any surrogates were found - and replaced, False otherwise. Covers content/text, name, tool call - metadata/arguments, AND any additional string or nested structured fields - (``reasoning``, ``reasoning_content``, ``reasoning_details``, etc.) so - retries don't fail on a non-content field. Byte-level reasoning models - (xiaomi/mimo, kimi, glm) can emit lone surrogates in reasoning output - that flow through to ``api_messages["reasoning_content"]`` on the next - turn and crash json.dumps inside the OpenAI SDK. - """ - found = False - for msg in messages: - if not isinstance(msg, dict): - continue - content = msg.get("content") - if isinstance(content, str) and _SURROGATE_RE.search(content): - msg["content"] = _SURROGATE_RE.sub('\ufffd', content) - found = True - elif isinstance(content, list): - for part in content: - if isinstance(part, dict): - text = part.get("text") - if isinstance(text, str) and _SURROGATE_RE.search(text): - part["text"] = _SURROGATE_RE.sub('\ufffd', text) - found = True - name = msg.get("name") - if isinstance(name, str) and _SURROGATE_RE.search(name): - msg["name"] = _SURROGATE_RE.sub('\ufffd', name) - found = True - tool_calls = msg.get("tool_calls") - if isinstance(tool_calls, list): - for tc in tool_calls: - if not isinstance(tc, dict): - continue - tc_id = tc.get("id") - if isinstance(tc_id, str) and _SURROGATE_RE.search(tc_id): - tc["id"] = _SURROGATE_RE.sub('\ufffd', tc_id) - found = True - fn = tc.get("function") - if isinstance(fn, dict): - fn_name = fn.get("name") - if isinstance(fn_name, str) and _SURROGATE_RE.search(fn_name): - fn["name"] = _SURROGATE_RE.sub('\ufffd', fn_name) - found = True - fn_args = fn.get("arguments") - if isinstance(fn_args, str) and _SURROGATE_RE.search(fn_args): - fn["arguments"] = _SURROGATE_RE.sub('\ufffd', fn_args) - found = True - # Walk any additional string / nested fields (reasoning, - # reasoning_content, reasoning_details, etc.) — surrogates from - # byte-level reasoning models (xiaomi/mimo, kimi, glm) can lurk - # in these fields and aren't covered by the per-field checks above. - # Matches _sanitize_messages_non_ascii's coverage (PR #10537). - for key, value in msg.items(): - if key in {"content", "name", "tool_calls", "role"}: - continue - if isinstance(value, str): - if _SURROGATE_RE.search(value): - msg[key] = _SURROGATE_RE.sub('\ufffd', value) - found = True - elif isinstance(value, (dict, list)): - if _sanitize_structure_surrogates(value): - found = True - return found - - -def _escape_invalid_chars_in_json_strings(raw: str) -> str: - """Escape unescaped control chars inside JSON string values. - - Walks the raw JSON character-by-character, tracking whether we are - inside a double-quoted string. Inside strings, replaces literal - control characters (0x00-0x1F) that aren't already part of an escape - sequence with their ``\\uXXXX`` equivalents. Pass-through for everything - else. - - Ported from #12093 — complements the other repair passes in - ``_repair_tool_call_arguments`` when ``json.loads(strict=False)`` is - not enough (e.g. llama.cpp backends that emit literal apostrophes or - tabs alongside other malformations). - """ - out: list[str] = [] - in_string = False - i = 0 - n = len(raw) - while i < n: - ch = raw[i] - if in_string: - if ch == "\\" and i + 1 < n: - # Already-escaped char — pass through as-is - out.append(ch) - out.append(raw[i + 1]) - i += 2 - continue - if ch == '"': - in_string = False - out.append(ch) - elif ord(ch) < 0x20: - out.append(f"\\u{ord(ch):04x}") - else: - out.append(ch) - else: - if ch == '"': - in_string = True - out.append(ch) - i += 1 - return "".join(out) - - -def _repair_tool_call_arguments(raw_args: str, tool_name: str = "?") -> str: - """Attempt to repair malformed tool_call argument JSON. - - Models like GLM-5.1 via Ollama can produce truncated JSON, trailing - commas, Python ``None``, etc. The API proxy rejects these with HTTP 400 - "invalid tool call arguments". This function applies common repairs; - if all fail it returns ``"{}"`` so the request succeeds (better than - crashing the session). All repairs are logged at WARNING level. - """ - raw_stripped = raw_args.strip() if isinstance(raw_args, str) else "" - - # Fast-path: empty / whitespace-only -> empty object - if not raw_stripped: - logger.warning("Sanitized empty tool_call arguments for %s", tool_name) - return "{}" - - # Python-literal None -> normalise to {} - if raw_stripped == "None": - logger.warning("Sanitized Python-None tool_call arguments for %s", tool_name) - return "{}" - - # Repair pass 0: llama.cpp backends sometimes emit literal control - # characters (tabs, newlines) inside JSON string values. json.loads - # with strict=False accepts these and lets us re-serialise the - # result into wire-valid JSON without any string surgery. This is - # the most common local-model repair case (#12068). - try: - parsed = json.loads(raw_stripped, strict=False) - reserialised = json.dumps(parsed, separators=(",", ":")) - if reserialised != raw_stripped: - logger.warning( - "Repaired unescaped control chars in tool_call arguments for %s", - tool_name, - ) - return reserialised - except (json.JSONDecodeError, TypeError, ValueError): - pass - - # Attempt common JSON repairs - fixed = raw_stripped - # 1. Strip trailing commas before } or ] - fixed = re.sub(r',\s*([}\]])', r'\1', fixed) - # 2. Close unclosed structures - open_curly = fixed.count('{') - fixed.count('}') - open_bracket = fixed.count('[') - fixed.count(']') - if open_curly > 0: - fixed += '}' * open_curly - if open_bracket > 0: - fixed += ']' * open_bracket - # 3. Remove excess closing braces/brackets (bounded to 50 iterations) - for _ in range(50): - try: - json.loads(fixed) - break - except json.JSONDecodeError: - if fixed.endswith('}') and fixed.count('}') > fixed.count('{'): - fixed = fixed[:-1] - elif fixed.endswith(']') and fixed.count(']') > fixed.count('['): - fixed = fixed[:-1] - else: - break - - try: - json.loads(fixed) - logger.warning( - "Repaired malformed tool_call arguments for %s: %s → %s", - tool_name, raw_stripped[:80], fixed[:80], - ) - return fixed - except json.JSONDecodeError: - pass - - # Repair pass 4: escape unescaped control chars inside JSON strings, - # then retry. Catches cases where strict=False alone fails because - # other malformations are present too. - try: - escaped = _escape_invalid_chars_in_json_strings(fixed) - if escaped != fixed: - json.loads(escaped) - logger.warning( - "Repaired control-char-laced tool_call arguments for %s: %s → %s", - tool_name, raw_stripped[:80], escaped[:80], - ) - return escaped - except (json.JSONDecodeError, TypeError, ValueError): - pass - - # Last resort: replace with empty object so the API request doesn't - # crash the entire session. - logger.warning( - "Unrepairable tool_call arguments for %s — " - "replaced with empty object (was: %s)", - tool_name, raw_stripped[:80], - ) - return "{}" - - -def _strip_non_ascii(text: str) -> str: - """Remove non-ASCII characters, replacing with closest ASCII equivalent or removing. - - Used as a last resort when the system encoding is ASCII and can't handle - any non-ASCII characters (e.g. LANG=C on Chromebooks). - """ - return text.encode('ascii', errors='ignore').decode('ascii') - - -def _sanitize_messages_non_ascii(messages: list) -> bool: - """Strip non-ASCII characters from all string content in a messages list. - - This is a last-resort recovery for systems with ASCII-only encoding - (LANG=C, Chromebooks, minimal containers). Returns True if any - non-ASCII content was found and sanitized. - """ - found = False - for msg in messages: - if not isinstance(msg, dict): - continue - # Sanitize content (string) - content = msg.get("content") - if isinstance(content, str): - sanitized = _strip_non_ascii(content) - if sanitized != content: - msg["content"] = sanitized - found = True - elif isinstance(content, list): - for part in content: - if isinstance(part, dict): - text = part.get("text") - if isinstance(text, str): - sanitized = _strip_non_ascii(text) - if sanitized != text: - part["text"] = sanitized - found = True - # Sanitize name field (can contain non-ASCII in tool results) - name = msg.get("name") - if isinstance(name, str): - sanitized = _strip_non_ascii(name) - if sanitized != name: - msg["name"] = sanitized - found = True - # Sanitize tool_calls - tool_calls = msg.get("tool_calls") - if isinstance(tool_calls, list): - for tc in tool_calls: - if isinstance(tc, dict): - fn = tc.get("function", {}) - if isinstance(fn, dict): - fn_args = fn.get("arguments") - if isinstance(fn_args, str): - sanitized = _strip_non_ascii(fn_args) - if sanitized != fn_args: - fn["arguments"] = sanitized - found = True - # Sanitize any additional top-level string fields (e.g. reasoning_content) - for key, value in msg.items(): - if key in {"content", "name", "tool_calls", "role"}: - continue - if isinstance(value, str): - sanitized = _strip_non_ascii(value) - if sanitized != value: - msg[key] = sanitized - found = True - return found - - -def _sanitize_tools_non_ascii(tools: list) -> bool: - """Strip non-ASCII characters from tool payloads in-place.""" - return _sanitize_structure_non_ascii(tools) - - -def _strip_images_from_messages(messages: list) -> bool: - """Remove image_url content parts from all messages in-place. - - Called when a server signals it does not support images (e.g. - "Only 'text' content type is supported."). Mutates messages so the - next API call sends text only. - - Preserves message alternation invariants: - * ``tool``-role messages whose content was entirely images are replaced - with a plaintext placeholder, NOT deleted — deleting them would leave - the paired ``tool_call_id`` on the prior assistant message unmatched, - which providers reject with HTTP 400. - * Non-tool messages whose content becomes empty are dropped. In - practice this only hits synthetic image-only user messages appended - for attachment delivery; real user turns always include text. - - Returns True if any image parts were removed. - """ - found = False - to_delete = [] - for i, msg in enumerate(messages): - if not isinstance(msg, dict): - continue - content = msg.get("content") - if not isinstance(content, list): - continue - new_parts = [] - for part in content: - if isinstance(part, dict) and part.get("type") in {"image_url", "image", "input_image"}: - found = True - else: - new_parts.append(part) - if len(new_parts) < len(content): - if new_parts: - msg["content"] = new_parts - elif msg.get("role") == "tool": - # Preserve tool_call_id linkage — providers require every - # assistant tool_call to have a matching tool response. - msg["content"] = "[image content removed — server does not support images]" - else: - # Synthetic image-only user/assistant message with no text; - # safe to drop. - to_delete.append(i) - for i in reversed(to_delete): - del messages[i] - return found - - -def _sanitize_structure_non_ascii(payload: Any) -> bool: - """Strip non-ASCII characters from nested dict/list payloads in-place.""" - found = False - - def _walk(node): - nonlocal found - if isinstance(node, dict): - for key, value in node.items(): - if isinstance(value, str): - sanitized = _strip_non_ascii(value) - if sanitized != value: - node[key] = sanitized - found = True - elif isinstance(value, (dict, list)): - _walk(value) - elif isinstance(node, list): - for idx, value in enumerate(node): - if isinstance(value, str): - sanitized = _strip_non_ascii(value) - if sanitized != value: - node[idx] = sanitized - found = True - elif isinstance(value, (dict, list)): - _walk(value) - - _walk(payload) - return found - - - - - # ========================================================================= # Large tool result handler — save oversized output to temp file # ========================================================================= From 59f1c0f0b668db8311b8ea15bdfc2d1fe373227d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 17:54:26 -0700 Subject: [PATCH 002/142] refactor(run_agent): extract tool-dispatch helpers to agent/tool_dispatch_helpers.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pull module-level helpers used by the tool-execution path out of run_agent.py: * parallelism gating — _NEVER_PARALLEL_TOOLS, _PARALLEL_SAFE_TOOLS, _PATH_SCOPED_TOOLS, _DESTRUCTIVE_PATTERNS, _REDIRECT_OVERWRITE, _is_destructive_command, _should_parallelize_tool_batch, _extract_parallel_scope_path, _paths_overlap * multimodal envelopes — _is_multimodal_tool_result, _multimodal_text_summary, _append_subdir_hint_to_multimodal * file-mutation verifier inputs — _extract_file_mutation_targets, _extract_error_preview * trajectory normalization — _trajectory_normalize_msg All pure functions. run_agent re-exports every name so existing 'from run_agent import _is_multimodal_tool_result' callers in tests/tools/, tests/run_agent/, and tools/file_state.py keep working. tests/run_agent/: 1341 passed, 3 skipped. run_agent.py: 15682 -> 15427 lines (-255). --- agent/tool_dispatch_helpers.py | 321 +++++++++++++++++++++++++++++++++ run_agent.py | 289 ++--------------------------- 2 files changed, 338 insertions(+), 272 deletions(-) create mode 100644 agent/tool_dispatch_helpers.py diff --git a/agent/tool_dispatch_helpers.py b/agent/tool_dispatch_helpers.py new file mode 100644 index 00000000000..289e10fb027 --- /dev/null +++ b/agent/tool_dispatch_helpers.py @@ -0,0 +1,321 @@ +"""Tool-dispatch helpers — parallelism gating, multimodal envelopes, mutation tracking. + +Pure module-level utilities extracted from ``run_agent.py``: + +* ``_is_destructive_command`` — terminal-command heuristic used to gate + parallel batch dispatch. +* ``_should_parallelize_tool_batch`` / ``_extract_parallel_scope_path`` / + ``_paths_overlap`` — the rules engine deciding when a multi-tool batch + can run concurrently. +* ``_is_multimodal_tool_result`` / ``_multimodal_text_summary`` / + ``_append_subdir_hint_to_multimodal`` — envelope helpers for the + ``{"_multimodal": True, "content": [...], "text_summary": ...}`` dict + shape returned by tools like ``computer_use``. +* ``_extract_file_mutation_targets`` / ``_extract_error_preview`` — + per-turn file-mutation verifier inputs. +* ``_trajectory_normalize_msg`` — strip image blobs from a message for + trajectory saving. + +All helpers are stateless. ``run_agent`` re-exports each name so existing +``from run_agent import ...`` imports in tests and other modules keep +working unchanged. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +from agent.tool_result_classification import ( + FILE_MUTATING_TOOL_NAMES as _FILE_MUTATING_TOOLS, +) + +logger = logging.getLogger(__name__) + +# Tools that must never run concurrently (interactive / user-facing). +# When any of these appear in a batch, we fall back to sequential execution. +_NEVER_PARALLEL_TOOLS = frozenset({"clarify"}) + +# Read-only tools with no shared mutable session state. +_PARALLEL_SAFE_TOOLS = frozenset({ + "ha_get_state", + "ha_list_entities", + "ha_list_services", + "read_file", + "search_files", + "session_search", + "skill_view", + "skills_list", + "vision_analyze", + "web_extract", + "web_search", +}) + +# File tools can run concurrently when they target independent paths. +_PATH_SCOPED_TOOLS = frozenset({"read_file", "write_file", "patch"}) + +# Patterns that indicate a terminal command may modify/delete files. +_DESTRUCTIVE_PATTERNS = re.compile( + r"""(?:^|\s|&&|\|\||;|`)(?: + rm\s|rmdir\s| + cp\s|install\s| + mv\s| + sed\s+-i| + truncate\s| + dd\s| + shred\s| + git\s+(?:reset|clean|checkout)\s + )""", + re.VERBOSE, +) +# Output redirects that overwrite files (> but not >>) +_REDIRECT_OVERWRITE = re.compile(r'[^>]>[^>]|^>[^>]') + + +def _is_destructive_command(cmd: str) -> bool: + """Heuristic: does this terminal command look like it modifies/deletes files?""" + if not cmd: + return False + if _DESTRUCTIVE_PATTERNS.search(cmd): + return True + if _REDIRECT_OVERWRITE.search(cmd): + return True + return False + + +def _should_parallelize_tool_batch(tool_calls) -> bool: + """Return True when a tool-call batch is safe to run concurrently.""" + if len(tool_calls) <= 1: + return False + + tool_names = [tc.function.name for tc in tool_calls] + if any(name in _NEVER_PARALLEL_TOOLS for name in tool_names): + return False + + reserved_paths: list[Path] = [] + for tool_call in tool_calls: + tool_name = tool_call.function.name + try: + function_args = json.loads(tool_call.function.arguments) + except Exception: + logging.debug( + "Could not parse args for %s — defaulting to sequential; raw=%s", + tool_name, + tool_call.function.arguments[:200], + ) + return False + if not isinstance(function_args, dict): + logging.debug( + "Non-dict args for %s (%s) — defaulting to sequential", + tool_name, + type(function_args).__name__, + ) + return False + + if tool_name in _PATH_SCOPED_TOOLS: + scoped_path = _extract_parallel_scope_path(tool_name, function_args) + if scoped_path is None: + return False + if any(_paths_overlap(scoped_path, existing) for existing in reserved_paths): + return False + reserved_paths.append(scoped_path) + continue + + if tool_name not in _PARALLEL_SAFE_TOOLS: + return False + + return True + + +def _extract_parallel_scope_path(tool_name: str, function_args: dict) -> Optional[Path]: + """Return the normalized file target for path-scoped tools.""" + if tool_name not in _PATH_SCOPED_TOOLS: + return None + + raw_path = function_args.get("path") + if not isinstance(raw_path, str) or not raw_path.strip(): + return None + + expanded = Path(raw_path).expanduser() + if expanded.is_absolute(): + return Path(os.path.abspath(str(expanded))) + + # Avoid resolve(); the file may not exist yet. + return Path(os.path.abspath(str(Path.cwd() / expanded))) + + +def _paths_overlap(left: Path, right: Path) -> bool: + """Return True when two paths may refer to the same subtree.""" + left_parts = left.parts + right_parts = right.parts + if not left_parts or not right_parts: + # Empty paths shouldn't reach here (guarded upstream), but be safe. + return bool(left_parts) == bool(right_parts) and bool(left_parts) + common_len = min(len(left_parts), len(right_parts)) + return left_parts[:common_len] == right_parts[:common_len] + + +def _is_multimodal_tool_result(value: Any) -> bool: + """True if the value is a multimodal tool result envelope. + + Multimodal handlers (e.g. tools/computer_use) return a dict with + `_multimodal=True`, a `content` key holding OpenAI-style content + parts, and an optional `text_summary` for string-only fallbacks. + """ + return ( + isinstance(value, dict) + and value.get("_multimodal") is True + and isinstance(value.get("content"), list) + ) + + +def _multimodal_text_summary(value: Any) -> str: + """Extract a plain text view of a multimodal tool result. + + Used wherever downstream code needs a string — logging, previews, + persistence size heuristics, fall-back content for providers that + don't support multipart tool messages. + """ + if _is_multimodal_tool_result(value): + if value.get("text_summary"): + return str(value["text_summary"]) + parts = [] + for p in value.get("content") or []: + if isinstance(p, dict) and p.get("type") == "text": + parts.append(str(p.get("text", ""))) + if parts: + return "\n".join(parts) + return "[multimodal tool result]" + if isinstance(value, str): + return value + try: + return json.dumps(value, default=str) + except Exception: + return str(value) + + +def _append_subdir_hint_to_multimodal(value: Dict[str, Any], hint: str) -> None: + """Mutate a multimodal tool-result envelope to append a subdir hint. + + The hint is added to the first text part so the model sees it; image + parts are left untouched. `text_summary` is also updated for + string-fallback callers. + """ + if not _is_multimodal_tool_result(value): + return + parts = value.get("content") or [] + for p in parts: + if isinstance(p, dict) and p.get("type") == "text": + p["text"] = str(p.get("text", "")) + hint + break + else: + parts.insert(0, {"type": "text", "text": hint}) + value["content"] = parts + if isinstance(value.get("text_summary"), str): + value["text_summary"] = value["text_summary"] + hint + + +def _extract_file_mutation_targets(tool_name: str, args: Dict[str, Any]) -> List[str]: + """Return the file paths a ``write_file`` or ``patch`` call is targeting. + + For ``write_file`` and ``patch`` in replace mode this is just ``args["path"]``. + For ``patch`` in V4A patch mode we parse the patch content for + ``*** Update File:`` / ``*** Add File:`` / ``*** Delete File:`` headers so + the verifier can track each file in a multi-file patch separately. + """ + if tool_name not in _FILE_MUTATING_TOOLS: + return [] + if tool_name == "write_file": + p = args.get("path") + return [str(p)] if p else [] + # tool_name == "patch" + mode = args.get("mode") or "replace" + if mode == "replace": + p = args.get("path") + return [str(p)] if p else [] + if mode == "patch": + body = args.get("patch") or "" + if not isinstance(body, str) or not body: + return [] + paths: List[str] = [] + for _m in re.finditer( + r'^\*\*\*\s+(?:Update|Add|Delete)\s+File:\s*(.+)$', + body, + re.MULTILINE, + ): + p = _m.group(1).strip() + if p: + paths.append(p) + return paths + return [] + + +def _extract_error_preview(result: Any, max_len: int = 180) -> str: + """Pull a one-line error summary out of a tool result for footer display.""" + text = _multimodal_text_summary(result) if result is not None else "" + if not isinstance(text, str): + try: + text = str(text) + except Exception: + return "" + # Try to parse JSON and pull the ``error`` field — tool handlers return + # ``{"success": false, "error": "..."}``; raw string wins if parse fails. + stripped = text.strip() + if stripped.startswith("{"): + try: + data = json.loads(stripped) + if isinstance(data, dict) and isinstance(data.get("error"), str): + text = data["error"] + except Exception: + pass + # Collapse whitespace, trim to max_len. + text = " ".join(text.split()) + if len(text) > max_len: + text = text[: max_len - 1] + "…" + return text + + +def _trajectory_normalize_msg(msg: Dict[str, Any]) -> Dict[str, Any]: + """Strip image blobs from a message for trajectory saving. + + Returns a shallow copy with multimodal tool results replaced by their + text_summary, and image parts in content lists replaced by + `[screenshot]` placeholders. Keeps the message schema otherwise intact. + """ + if not isinstance(msg, dict): + return msg + content = msg.get("content") + if _is_multimodal_tool_result(content): + return {**msg, "content": _multimodal_text_summary(content)} + if isinstance(content, list): + cleaned = [] + for p in content: + if isinstance(p, dict) and p.get("type") in {"image", "image_url", "input_image"}: + cleaned.append({"type": "text", "text": "[screenshot]"}) + else: + cleaned.append(p) + return {**msg, "content": cleaned} + return msg + + +__all__ = [ + "_NEVER_PARALLEL_TOOLS", + "_PARALLEL_SAFE_TOOLS", + "_PATH_SCOPED_TOOLS", + "_DESTRUCTIVE_PATTERNS", + "_REDIRECT_OVERWRITE", + "_is_destructive_command", + "_should_parallelize_tool_batch", + "_extract_parallel_scope_path", + "_paths_overlap", + "_is_multimodal_tool_result", + "_multimodal_text_summary", + "_append_subdir_hint_to_multimodal", + "_extract_file_mutation_targets", + "_extract_error_preview", + "_trajectory_normalize_msg", +] diff --git a/run_agent.py b/run_agent.py index b4d88f59eff..eed7550c468 100644 --- a/run_agent.py +++ b/run_agent.py @@ -202,6 +202,23 @@ from agent.message_sanitization import ( _strip_images_from_messages, _sanitize_structure_non_ascii, ) +from agent.tool_dispatch_helpers import ( + _NEVER_PARALLEL_TOOLS, + _PARALLEL_SAFE_TOOLS, + _PATH_SCOPED_TOOLS, + _DESTRUCTIVE_PATTERNS, + _REDIRECT_OVERWRITE, + _is_destructive_command, + _should_parallelize_tool_batch, + _extract_parallel_scope_path, + _paths_overlap, + _is_multimodal_tool_result, + _multimodal_text_summary, + _append_subdir_hint_to_multimodal, + _extract_file_mutation_targets, + _extract_error_preview, + _trajectory_normalize_msg, +) from utils import atomic_json_write, base_url_host_matches, base_url_hostname, env_var_enabled, normalize_proxy_url from hermes_cli.config import cfg_get @@ -342,31 +359,6 @@ class IterationBudget: return max(0, self.max_total - self._used) -# Tools that must never run concurrently (interactive / user-facing). -# When any of these appear in a batch, we fall back to sequential execution. -_NEVER_PARALLEL_TOOLS = frozenset({"clarify"}) - -# Read-only tools with no shared mutable session state. -_PARALLEL_SAFE_TOOLS = frozenset({ - "ha_get_state", - "ha_list_entities", - "ha_list_services", - "read_file", - "search_files", - "session_search", - "skill_view", - "skills_list", - "vision_analyze", - "web_extract", - "web_search", -}) - -# File tools can run concurrently when they target independent paths. -_PATH_SCOPED_TOOLS = frozenset({"read_file", "write_file", "patch"}) - -# Tools that mutate files on disk. Used by the per-turn verifier that -# surfaces silently-failed file edits so the model can't over-claim success. -# Imported above as `_FILE_MUTATING_TOOLS` from `agent.tool_result_classification`. # Maximum number of concurrent worker threads for parallel tool execution. _MAX_TOOL_WORKERS = 8 @@ -377,253 +369,6 @@ _MAX_TOOL_WORKERS = 8 # exhaust the system thread limit (RuntimeError: can't start new thread). _openrouter_prewarm_done = threading.Event() -# Patterns that indicate a terminal command may modify/delete files. -_DESTRUCTIVE_PATTERNS = re.compile( - r"""(?:^|\s|&&|\|\||;|`)(?: - rm\s|rmdir\s| - cp\s|install\s| - mv\s| - sed\s+-i| - truncate\s| - dd\s| - shred\s| - git\s+(?:reset|clean|checkout)\s - )""", - re.VERBOSE, -) -# Output redirects that overwrite files (> but not >>) -_REDIRECT_OVERWRITE = re.compile(r'[^>]>[^>]|^>[^>]') - - -def _is_destructive_command(cmd: str) -> bool: - """Heuristic: does this terminal command look like it modifies/deletes files?""" - if not cmd: - return False - if _DESTRUCTIVE_PATTERNS.search(cmd): - return True - if _REDIRECT_OVERWRITE.search(cmd): - return True - return False - - -def _should_parallelize_tool_batch(tool_calls) -> bool: - """Return True when a tool-call batch is safe to run concurrently.""" - if len(tool_calls) <= 1: - return False - - tool_names = [tc.function.name for tc in tool_calls] - if any(name in _NEVER_PARALLEL_TOOLS for name in tool_names): - return False - - reserved_paths: list[Path] = [] - for tool_call in tool_calls: - tool_name = tool_call.function.name - try: - function_args = json.loads(tool_call.function.arguments) - except Exception: - logging.debug( - "Could not parse args for %s — defaulting to sequential; raw=%s", - tool_name, - tool_call.function.arguments[:200], - ) - return False - if not isinstance(function_args, dict): - logging.debug( - "Non-dict args for %s (%s) — defaulting to sequential", - tool_name, - type(function_args).__name__, - ) - return False - - if tool_name in _PATH_SCOPED_TOOLS: - scoped_path = _extract_parallel_scope_path(tool_name, function_args) - if scoped_path is None: - return False - if any(_paths_overlap(scoped_path, existing) for existing in reserved_paths): - return False - reserved_paths.append(scoped_path) - continue - - if tool_name not in _PARALLEL_SAFE_TOOLS: - return False - - return True - - -def _extract_parallel_scope_path(tool_name: str, function_args: dict) -> Path | None: - """Return the normalized file target for path-scoped tools.""" - if tool_name not in _PATH_SCOPED_TOOLS: - return None - - raw_path = function_args.get("path") - if not isinstance(raw_path, str) or not raw_path.strip(): - return None - - expanded = Path(raw_path).expanduser() - if expanded.is_absolute(): - return Path(os.path.abspath(str(expanded))) - - # Avoid resolve(); the file may not exist yet. - return Path(os.path.abspath(str(Path.cwd() / expanded))) - - -def _paths_overlap(left: Path, right: Path) -> bool: - """Return True when two paths may refer to the same subtree.""" - left_parts = left.parts - right_parts = right.parts - if not left_parts or not right_parts: - # Empty paths shouldn't reach here (guarded upstream), but be safe. - return bool(left_parts) == bool(right_parts) and bool(left_parts) - common_len = min(len(left_parts), len(right_parts)) - return left_parts[:common_len] == right_parts[:common_len] - - -def _is_multimodal_tool_result(value: Any) -> bool: - """True if the value is a multimodal tool result envelope. - - Multimodal handlers (e.g. tools/computer_use) return a dict with - `_multimodal=True`, a `content` key holding OpenAI-style content - parts, and an optional `text_summary` for string-only fallbacks. - """ - return ( - isinstance(value, dict) - and value.get("_multimodal") is True - and isinstance(value.get("content"), list) - ) - - -def _multimodal_text_summary(value: Any) -> str: - """Extract a plain text view of a multimodal tool result. - - Used wherever downstream code needs a string — logging, previews, - persistence size heuristics, fall-back content for providers that - don't support multipart tool messages. - """ - if _is_multimodal_tool_result(value): - if value.get("text_summary"): - return str(value["text_summary"]) - parts = [] - for p in value.get("content") or []: - if isinstance(p, dict) and p.get("type") == "text": - parts.append(str(p.get("text", ""))) - if parts: - return "\n".join(parts) - return "[multimodal tool result]" - if isinstance(value, str): - return value - try: - import json as _json - return _json.dumps(value, default=str) - except Exception: - return str(value) - - -def _append_subdir_hint_to_multimodal(value: Dict[str, Any], hint: str) -> None: - """Mutate a multimodal tool-result envelope to append a subdir hint. - - The hint is added to the first text part so the model sees it; image - parts are left untouched. `text_summary` is also updated for - string-fallback callers. - """ - if not _is_multimodal_tool_result(value): - return - parts = value.get("content") or [] - for p in parts: - if isinstance(p, dict) and p.get("type") == "text": - p["text"] = str(p.get("text", "")) + hint - break - else: - parts.insert(0, {"type": "text", "text": hint}) - value["content"] = parts - if isinstance(value.get("text_summary"), str): - value["text_summary"] = value["text_summary"] + hint - - -def _extract_file_mutation_targets(tool_name: str, args: Dict[str, Any]) -> List[str]: - """Return the file paths a ``write_file`` or ``patch`` call is targeting. - - For ``write_file`` and ``patch`` in replace mode this is just ``args["path"]``. - For ``patch`` in V4A patch mode we parse the patch content for - ``*** Update File:`` / ``*** Add File:`` / ``*** Delete File:`` headers so - the verifier can track each file in a multi-file patch separately. - """ - if tool_name not in _FILE_MUTATING_TOOLS: - return [] - if tool_name == "write_file": - p = args.get("path") - return [str(p)] if p else [] - # tool_name == "patch" - mode = args.get("mode") or "replace" - if mode == "replace": - p = args.get("path") - return [str(p)] if p else [] - if mode == "patch": - body = args.get("patch") or "" - if not isinstance(body, str) or not body: - return [] - import re as _re - paths: List[str] = [] - for _m in _re.finditer( - r'^\*\*\*\s+(?:Update|Add|Delete)\s+File:\s*(.+)$', - body, - _re.MULTILINE, - ): - p = _m.group(1).strip() - if p: - paths.append(p) - return paths - return [] - - -def _extract_error_preview(result: Any, max_len: int = 180) -> str: - """Pull a one-line error summary out of a tool result for footer display.""" - text = _multimodal_text_summary(result) if result is not None else "" - if not isinstance(text, str): - try: - text = str(text) - except Exception: - return "" - # Try to parse JSON and pull the ``error`` field — tool handlers return - # ``{"success": false, "error": "..."}``; raw string wins if parse fails. - stripped = text.strip() - if stripped.startswith("{"): - try: - import json as _json - data = _json.loads(stripped) - if isinstance(data, dict) and isinstance(data.get("error"), str): - text = data["error"] - except Exception: - pass - # Collapse whitespace, trim to max_len. - text = " ".join(text.split()) - if len(text) > max_len: - text = text[: max_len - 1] + "…" - return text - - -def _trajectory_normalize_msg(msg: Dict[str, Any]) -> Dict[str, Any]: - """Strip image blobs from a message for trajectory saving. - - Returns a shallow copy with multimodal tool results replaced by their - text_summary, and image parts in content lists replaced by - `[screenshot]` placeholders. Keeps the message schema otherwise intact. - """ - if not isinstance(msg, dict): - return msg - content = msg.get("content") - if _is_multimodal_tool_result(content): - return {**msg, "content": _multimodal_text_summary(content)} - if isinstance(content, list): - cleaned = [] - for p in content: - if isinstance(p, dict) and p.get("type") in {"image", "image_url", "input_image"}: - cleaned.append({"type": "text", "text": "[screenshot]"}) - else: - cleaned.append(p) - return {**msg, "content": cleaned} - return msg - - # ========================================================================= # Large tool result handler — save oversized output to temp file # ========================================================================= From 5f309ae685d08a2d23eaa992a5bca1f70f52486a Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 17:59:32 -0700 Subject: [PATCH 003/142] refactor(run_agent): extract OpenAI proxy, safe stdio, IterationBudget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small extractions into focused modules: * agent/process_bootstrap.py — \_OpenAIProxy (lazy openai.OpenAI import), \_SafeWriter (broken-pipe-resistant stdio wrapper), \_install_safe_stdio, \_get_proxy_from_env, \_get_proxy_for_base_url. All process / IO bootstrap. * agent/iteration_budget.py — IterationBudget class (thread-safe consume/ refund counter shared by parent agent and subagents). run_agent re-exports every name so existing test patches like patch('run_agent.OpenAI', ...) and 'from run_agent import IterationBudget' keep working unchanged. Verified the patch-rebinding contract for OpenAI explicitly. tests/run_agent/ + tests/agent/test_gemini_fast_fallback.py: 1347 passed, 3 skipped. run_agent.py: 15427 -> 15261 lines (-166). --- agent/iteration_budget.py | 62 +++++++++++++ agent/process_bootstrap.py | 167 ++++++++++++++++++++++++++++++++++ run_agent.py | 179 +++---------------------------------- 3 files changed, 241 insertions(+), 167 deletions(-) create mode 100644 agent/iteration_budget.py create mode 100644 agent/process_bootstrap.py diff --git a/agent/iteration_budget.py b/agent/iteration_budget.py new file mode 100644 index 00000000000..213b97c0226 --- /dev/null +++ b/agent/iteration_budget.py @@ -0,0 +1,62 @@ +"""Per-agent iteration budget — thread-safe consume/refund counter. + +Extracted from ``run_agent.py``. Each ``AIAgent`` instance (parent or +subagent) holds an :class:`IterationBudget`; the parent's cap comes from +``max_iterations`` (default 90), each subagent's cap comes from +``delegation.max_iterations`` (default 50). + +``run_agent`` re-exports ``IterationBudget`` so existing +``from run_agent import IterationBudget`` imports keep working unchanged. +""" + +from __future__ import annotations + +import threading + + +class IterationBudget: + """Thread-safe iteration counter for an agent. + + Each agent (parent or subagent) gets its own ``IterationBudget``. + The parent's budget is capped at ``max_iterations`` (default 90). + Each subagent gets an independent budget capped at + ``delegation.max_iterations`` (default 50) — this means total + iterations across parent + subagents can exceed the parent's cap. + Users control the per-subagent limit via ``delegation.max_iterations`` + in config.yaml. + + ``execute_code`` (programmatic tool calling) iterations are refunded via + :meth:`refund` so they don't eat into the budget. + """ + + def __init__(self, max_total: int): + self.max_total = max_total + self._used = 0 + self._lock = threading.Lock() + + def consume(self) -> bool: + """Try to consume one iteration. Returns True if allowed.""" + with self._lock: + if self._used >= self.max_total: + return False + self._used += 1 + return True + + def refund(self) -> None: + """Give back one iteration (e.g. for execute_code turns).""" + with self._lock: + if self._used > 0: + self._used -= 1 + + @property + def used(self) -> int: + with self._lock: + return self._used + + @property + def remaining(self) -> int: + with self._lock: + return max(0, self.max_total - self._used) + + +__all__ = ["IterationBudget"] diff --git a/agent/process_bootstrap.py b/agent/process_bootstrap.py new file mode 100644 index 00000000000..fdd9053f5d8 --- /dev/null +++ b/agent/process_bootstrap.py @@ -0,0 +1,167 @@ +"""Process-level bootstrap helpers for ``run_agent``. + +Three concerns, all tied to ``AIAgent`` boot-time / runtime IO setup: + +1. **Lazy OpenAI SDK import** — ``_load_openai_cls`` + ``_OpenAIProxy`` + defer the 240ms-ish ``from openai import OpenAI`` cost until first use, + while preserving ``isinstance(client, OpenAI)`` checks and + ``patch("run_agent.OpenAI", ...)`` test patterns. + +2. **Crash-resistant stdio** — ``_SafeWriter`` wraps stdout/stderr so + ``OSError: Input/output error`` from broken pipes (systemd, Docker, + thread teardown races) cannot crash the agent. ``_install_safe_stdio`` + applies the wrapper. + +3. **HTTP proxy resolution** — ``_get_proxy_from_env`` reads + ``HTTPS_PROXY`` / ``HTTP_PROXY`` / ``ALL_PROXY``; + ``_get_proxy_for_base_url`` respects ``NO_PROXY`` for the given base URL. + +``run_agent`` re-exports every name so existing +``from run_agent import _get_proxy_from_env`` imports keep working +unchanged. +""" + +from __future__ import annotations + +import os +import sys +import urllib.request +from typing import Optional + +from utils import base_url_hostname, normalize_proxy_url + + +# Cached at module level so we only pay the OpenAI SDK import cost once +# per process (after the first lazy load). +_OPENAI_CLS_CACHE = None + + +def _load_openai_cls() -> type: + """Import and cache ``openai.OpenAI``.""" + global _OPENAI_CLS_CACHE + if _OPENAI_CLS_CACHE is None: + from openai import OpenAI as _cls + _OPENAI_CLS_CACHE = _cls + return _OPENAI_CLS_CACHE + + +class _OpenAIProxy: + """Module-level proxy that looks like ``openai.OpenAI`` but imports lazily.""" + + __slots__ = () + + def __call__(self, *args, **kwargs): + return _load_openai_cls()(*args, **kwargs) + + def __instancecheck__(self, obj): + return isinstance(obj, _load_openai_cls()) + + def __repr__(self): + return "" + + +class _SafeWriter: + """Transparent stdio wrapper that catches OSError/ValueError from broken pipes. + + When hermes-agent runs as a systemd service, Docker container, or headless + daemon, the stdout/stderr pipe can become unavailable (idle timeout, buffer + exhaustion, socket reset). Any print() call then raises + ``OSError: [Errno 5] Input/output error``, which can crash agent setup or + run_conversation() — especially via double-fault when an except handler + also tries to print. + + Additionally, when subagents run in ThreadPoolExecutor threads, the shared + stdout handle can close between thread teardown and cleanup, raising + ``ValueError: I/O operation on closed file`` instead of OSError. + + This wrapper delegates all writes to the underlying stream and silently + catches both OSError and ValueError. It is transparent when the wrapped + stream is healthy. + """ + + __slots__ = ("_inner",) + + def __init__(self, inner): + object.__setattr__(self, "_inner", inner) + + def write(self, data): + try: + return self._inner.write(data) + except (OSError, ValueError): + return len(data) if isinstance(data, str) else 0 + + def flush(self): + try: + self._inner.flush() + except (OSError, ValueError): + pass + + def fileno(self): + return self._inner.fileno() + + def isatty(self): + try: + return self._inner.isatty() + except (OSError, ValueError): + return False + + def __getattr__(self, name): + return getattr(self._inner, name) + + +def _get_proxy_from_env() -> Optional[str]: + """Read proxy URL from environment variables. + + Checks HTTPS_PROXY, HTTP_PROXY, ALL_PROXY (and lowercase variants) in order. + Returns the first valid proxy URL found, or None if no proxy is configured. + """ + for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", + "https_proxy", "http_proxy", "all_proxy"): + value = os.environ.get(key, "").strip() + if value: + return normalize_proxy_url(value) + return None + + +def _get_proxy_for_base_url(base_url: Optional[str]) -> Optional[str]: + """Return an env-configured proxy unless NO_PROXY excludes this base URL.""" + proxy = _get_proxy_from_env() + if not proxy or not base_url: + return proxy + + host = base_url_hostname(base_url) + if not host: + return proxy + + try: + if urllib.request.proxy_bypass_environment(host): + return None + except Exception: + pass + + return proxy + + +def _install_safe_stdio() -> None: + """Wrap stdout/stderr so best-effort console output cannot crash the agent.""" + for stream_name in ("stdout", "stderr"): + stream = getattr(sys, stream_name, None) + if stream is not None and not isinstance(stream, _SafeWriter): + setattr(sys, stream_name, _SafeWriter(stream)) + + +# Module-level proxy instance — drops in for ``openai.OpenAI``. Imported as +# ``from agent.process_bootstrap import OpenAI`` (or re-exported via +# ``run_agent`` for legacy tests). +OpenAI = _OpenAIProxy() + + +__all__ = [ + "OpenAI", + "_OpenAIProxy", + "_load_openai_cls", + "_SafeWriter", + "_install_safe_stdio", + "_get_proxy_from_env", + "_get_proxy_for_base_url", +] diff --git a/run_agent.py b/run_agent.py index eed7550c468..22848b2f20e 100644 --- a/run_agent.py +++ b/run_agent.py @@ -70,38 +70,20 @@ from pathlib import Path from hermes_constants import get_hermes_home - -_OPENAI_CLS_CACHE: Optional[type] = None +# OpenAI lazy proxy + safe stdio + proxy URL helpers — see agent/process_bootstrap.py. +# `OpenAI` is re-exported here so `patch("run_agent.OpenAI", ...)` in tests works. +from agent.process_bootstrap import ( + OpenAI, + _OpenAIProxy, + _load_openai_cls, + _SafeWriter, + _install_safe_stdio, + _get_proxy_from_env, + _get_proxy_for_base_url, +) +from agent.iteration_budget import IterationBudget -def _load_openai_cls() -> type: - """Import and cache ``openai.OpenAI``.""" - global _OPENAI_CLS_CACHE - if _OPENAI_CLS_CACHE is None: - from openai import OpenAI as _cls - _OPENAI_CLS_CACHE = _cls - return _OPENAI_CLS_CACHE - - -class _OpenAIProxy: - """Module-level proxy that looks like ``openai.OpenAI`` but imports lazily.""" - - __slots__ = () - - def __call__(self, *args, **kwargs): - return _load_openai_cls()(*args, **kwargs) - - def __instancecheck__(self, obj): - return isinstance(obj, _load_openai_cls()) - - def __repr__(self): - return "" - - -OpenAI = _OpenAIProxy() - -# Load .env from ~/.hermes/.env first, then project root as dev fallback. -# User-managed env files should override stale shell exports on restart. from hermes_cli.env_loader import load_hermes_dotenv from hermes_cli.timeouts import ( get_provider_request_timeout, @@ -224,143 +206,6 @@ from hermes_cli.config import cfg_get -class _SafeWriter: - """Transparent stdio wrapper that catches OSError/ValueError from broken pipes. - - When hermes-agent runs as a systemd service, Docker container, or headless - daemon, the stdout/stderr pipe can become unavailable (idle timeout, buffer - exhaustion, socket reset). Any print() call then raises - ``OSError: [Errno 5] Input/output error``, which can crash agent setup or - run_conversation() — especially via double-fault when an except handler - also tries to print. - - Additionally, when subagents run in ThreadPoolExecutor threads, the shared - stdout handle can close between thread teardown and cleanup, raising - ``ValueError: I/O operation on closed file`` instead of OSError. - - This wrapper delegates all writes to the underlying stream and silently - catches both OSError and ValueError. It is transparent when the wrapped - stream is healthy. - """ - - __slots__ = ("_inner",) - - def __init__(self, inner): - object.__setattr__(self, "_inner", inner) - - def write(self, data): - try: - return self._inner.write(data) - except (OSError, ValueError): - return len(data) if isinstance(data, str) else 0 - - def flush(self): - try: - self._inner.flush() - except (OSError, ValueError): - pass - - def fileno(self): - return self._inner.fileno() - - def isatty(self): - try: - return self._inner.isatty() - except (OSError, ValueError): - return False - - def __getattr__(self, name): - return getattr(self._inner, name) - - -def _get_proxy_from_env() -> Optional[str]: - """Read proxy URL from environment variables. - - Checks HTTPS_PROXY, HTTP_PROXY, ALL_PROXY (and lowercase variants) in order. - Returns the first valid proxy URL found, or None if no proxy is configured. - """ - for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", - "https_proxy", "http_proxy", "all_proxy"): - value = os.environ.get(key, "").strip() - if value: - return normalize_proxy_url(value) - return None - - -def _get_proxy_for_base_url(base_url: Optional[str]) -> Optional[str]: - """Return an env-configured proxy unless NO_PROXY excludes this base URL.""" - proxy = _get_proxy_from_env() - if not proxy or not base_url: - return proxy - - host = base_url_hostname(base_url) - if not host: - return proxy - - try: - if urllib.request.proxy_bypass_environment(host): - return None - except Exception: - pass - - return proxy - - -def _install_safe_stdio() -> None: - """Wrap stdout/stderr so best-effort console output cannot crash the agent.""" - for stream_name in ("stdout", "stderr"): - stream = getattr(sys, stream_name, None) - if stream is not None and not isinstance(stream, _SafeWriter): - setattr(sys, stream_name, _SafeWriter(stream)) - - -class IterationBudget: - """Thread-safe iteration counter for an agent. - - Each agent (parent or subagent) gets its own ``IterationBudget``. - The parent's budget is capped at ``max_iterations`` (default 90). - Each subagent gets an independent budget capped at - ``delegation.max_iterations`` (default 50) — this means total - iterations across parent + subagents can exceed the parent's cap. - Users control the per-subagent limit via ``delegation.max_iterations`` - in config.yaml. - - ``execute_code`` (programmatic tool calling) iterations are refunded via - :meth:`refund` so they don't eat into the budget. - """ - - def __init__(self, max_total: int): - self.max_total = max_total - self._used = 0 - self._lock = threading.Lock() - - def consume(self) -> bool: - """Try to consume one iteration. Returns True if allowed.""" - with self._lock: - if self._used >= self.max_total: - return False - self._used += 1 - return True - - def refund(self) -> None: - """Give back one iteration (e.g. for execute_code turns).""" - with self._lock: - if self._used > 0: - self._used -= 1 - - @property - def used(self) -> int: - with self._lock: - return self._used - - @property - def remaining(self) -> int: - with self._lock: - return max(0, self.max_total - self._used) - - - -# Maximum number of concurrent worker threads for parallel tool execution. _MAX_TOOL_WORKERS = 8 # Guard so the OpenRouter metadata pre-warm thread is only spawned once per From 1f6eb1738c206e95c3e0641c3d8000a4d0be841b Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:05:01 -0700 Subject: [PATCH 004/142] refactor(run_agent): extract background memory/skill review to agent/background_review.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the background-review subsystem (the self-improvement loop — see the README) out of run_agent.py into a dedicated module. * summarize_background_review_actions — was the @staticmethod that builds the user-facing action summary * spawn_background_review_thread — builds the thread target + prompt; the actual review loop body (forked AIAgent, runtime inheritance, tool whitelist, suppression, teardown) lives in _run_review_in_thread * build_memory_write_metadata — provenance for external memory mirrors AIAgent keeps thin wrappers for backward compatibility AND because tests patch run_agent.threading.Thread to assert lifecycle behavior — the threading.Thread construction stays in AIAgent._spawn_background_review, the inner work moves out. tests/run_agent/ + tests/agent/: 4313 passed, 1 pre-existing failure (test_auxiliary_client.py::test_custom_endpoint... — confirmed failing on main before this change). 3 skipped. run_agent.py: 15272 -> 14972 lines (-300). --- agent/background_review.py | 360 +++++++++++++++++++++++++++++++++++++ run_agent.py | 311 +++----------------------------- 2 files changed, 386 insertions(+), 285 deletions(-) create mode 100644 agent/background_review.py diff --git a/agent/background_review.py b/agent/background_review.py new file mode 100644 index 00000000000..351ab1d43dc --- /dev/null +++ b/agent/background_review.py @@ -0,0 +1,360 @@ +"""Background memory/skill review — fork the agent to evaluate the turn. + +After every turn, ``AIAgent.run_conversation`` may call +:func:`spawn_background_review` to fire off a daemon thread that replays +the conversation snapshot in a forked :class:`AIAgent` and asks itself +"should any skill/memory be saved or updated?". Writes go straight to +the memory + skill stores. Main conversation and prompt cache are never +touched. + +The fork inherits the parent's live runtime (provider, model, base_url, +credentials, cached system prompt) so it hits the same prefix cache and +uses the same auth. It runs with a tool whitelist limited to memory and +skill management tools; everything else is denied at runtime. + +See the ``hermes-agent-dev`` skill (``references/self-improvement-loop.md``) +for invariants and PR review criteria. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +def summarize_background_review_actions( + review_messages: List[Dict], + prior_snapshot: List[Dict], +) -> List[str]: + """Build the human-facing action summary for a background review pass. + + Walks the review agent's session messages and collects "successful tool + action" descriptions to surface to the user (e.g. "Memory updated"). + Tool messages already present in ``prior_snapshot`` are skipped so we + don't re-surface stale results from the prior conversation that the + review agent inherited via ``conversation_history`` (issue #14944). + + Matching is by ``tool_call_id`` when available, with a content-equality + fallback for tool messages that lack one. + """ + existing_tool_call_ids = set() + existing_tool_contents = set() + for prior in prior_snapshot or []: + if not isinstance(prior, dict) or prior.get("role") != "tool": + continue + tcid = prior.get("tool_call_id") + if tcid: + existing_tool_call_ids.add(tcid) + else: + content = prior.get("content") + if isinstance(content, str): + existing_tool_contents.add(content) + + actions: List[str] = [] + for msg in review_messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + tcid = msg.get("tool_call_id") + if tcid and tcid in existing_tool_call_ids: + continue + if not tcid: + content_str = msg.get("content") + if isinstance(content_str, str) and content_str in existing_tool_contents: + continue + try: + data = json.loads(msg.get("content", "{}")) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(data, dict) or not data.get("success"): + continue + message = data.get("message", "") + target = data.get("target", "") + if "created" in message.lower(): + actions.append(message) + elif "updated" in message.lower(): + actions.append(message) + elif "added" in message.lower() or (target and "add" in message.lower()): + label = "Memory" if target == "memory" else "User profile" if target == "user" else target + actions.append(f"{label} updated") + elif "Entry added" in message: + label = "Memory" if target == "memory" else "User profile" if target == "user" else target + actions.append(f"{label} updated") + elif "removed" in message.lower() or "replaced" in message.lower(): + label = "Memory" if target == "memory" else "User profile" if target == "user" else target + actions.append(f"{label} updated") + return actions + + +def build_memory_write_metadata( + agent: Any, + *, + write_origin: Optional[str] = None, + execution_context: Optional[str] = None, + task_id: Optional[str] = None, + tool_call_id: Optional[str] = None, +) -> Dict[str, Any]: + """Build provenance metadata for external memory-provider mirrors.""" + metadata: Dict[str, Any] = { + "write_origin": write_origin or getattr(agent, "_memory_write_origin", "assistant_tool"), + "execution_context": ( + execution_context + or getattr(agent, "_memory_write_context", "foreground") + ), + "session_id": agent.session_id or "", + "parent_session_id": agent._parent_session_id or "", + "platform": agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), + "tool_name": "memory", + } + if task_id: + metadata["task_id"] = task_id + if tool_call_id: + metadata["tool_call_id"] = tool_call_id + return {k: v for k, v in metadata.items() if v not in {None, ""}} + + +def _run_review_in_thread( + agent: Any, + messages_snapshot: List[Dict], + prompt: str, +) -> None: + """Worker function executed in the background-review daemon thread. + + Spawns a forked ``AIAgent`` inheriting the parent's runtime, runs the + review prompt, and surfaces a compact action summary back to the user + via ``agent._safe_print`` and ``agent.background_review_callback``. + """ + # Local import to avoid a hard circular dep at module load. + from run_agent import AIAgent + from tools.terminal_tool import set_approval_callback as _set_approval_callback + + # Install a non-interactive approval callback on this worker + # thread so any dangerous-command guard the review agent trips + # resolves to "deny" instead of falling back to input() -- which + # deadlocks against the parent's prompt_toolkit TUI (#15216). + # Same pattern as _subagent_auto_deny in tools/delegate_tool.py. + def _bg_review_auto_deny(command, description, **kwargs): + logger.warning( + "Background review auto-denied dangerous command: %s (%s)", + command, description, + ) + return "deny" + try: + _set_approval_callback(_bg_review_auto_deny) + except Exception: + pass + + review_agent = None + review_messages: List[Dict] = [] + try: + with open(os.devnull, "w", encoding="utf-8") as _devnull, \ + contextlib.redirect_stdout(_devnull), \ + contextlib.redirect_stderr(_devnull): + # Inherit the parent agent's live runtime (provider, model, + # base_url, api_key, api_mode) so the fork uses the exact + # same credentials the main turn is using. Without this, + # AIAgent.__init__ re-runs auto-resolution from env vars, + # which fails for OAuth-only providers, session-scoped + # creds, or credential-pool setups where the resolver can't + # reconstruct auth from scratch -- producing the spurious + # "No LLM provider configured" warning at end of turn. + _parent_runtime = agent._current_main_runtime() + _parent_api_mode = _parent_runtime.get("api_mode") or None + # The review fork needs to call agent-loop tools (memory, + # skill_manage). Those tools require Hermes' own dispatch, + # which the codex_app_server runtime bypasses entirely + # (it runs the turn inside codex's subprocess). So when + # the parent is on codex_app_server, downgrade the review + # fork to codex_responses — same auth/credentials, but + # talks to the OpenAI Responses API directly so Hermes + # owns the loop and the agent-loop tools dispatch. + if _parent_api_mode == "codex_app_server": + _parent_api_mode = "codex_responses" + review_agent = AIAgent( + model=agent.model, + max_iterations=16, + quiet_mode=True, + platform=agent.platform, + provider=agent.provider, + api_mode=_parent_api_mode, + base_url=_parent_runtime.get("base_url") or None, + api_key=_parent_runtime.get("api_key") or None, + credential_pool=getattr(agent, "_credential_pool", None), + parent_session_id=agent.session_id, + ) + review_agent._memory_write_origin = "background_review" + review_agent._memory_write_context = "background_review" + review_agent._memory_store = agent._memory_store + review_agent._memory_enabled = agent._memory_enabled + review_agent._user_profile_enabled = agent._user_profile_enabled + review_agent._memory_nudge_interval = 0 + review_agent._skill_nudge_interval = 0 + # Suppress all status/warning emits from the fork so the + # user only sees the final successful-action summary. + # Without this, mid-review "Iteration budget exhausted", + # rate-limit retries, compression warnings, and other + # lifecycle messages bubble up through _emit_status -> + # _vprint and leak past the stdout redirect (they go via + # _print_fn/status_callback, which bypass sys.stdout). + review_agent.suppress_status_output = True + # Inherit the parent's cached system prompt verbatim so + # the review fork's outbound HTTP request hits the same + # Anthropic/OpenRouter prefix cache the parent warmed. + # Without this, the fork rebuilds the system prompt from + # scratch (fresh _hermes_now() timestamp, fresh + # session_id, narrower toolset → different skills_prompt) + # and the byte-exact prefix-cache key misses. See + # issue #25322 and PR #17276 for the full analysis + + # measured impact (~26% end-to-end cost reduction on + # Sonnet 4.5). + review_agent._cached_system_prompt = agent._cached_system_prompt + # Defensive: pin session_start + session_id to the + # parent's so any code path that re-renders parts of + # the system prompt (compression, plugin hooks) still + # produces byte-identical output. The cached-prompt + # assignment above already short-circuits the normal + # rebuild path, but these pins guarantee parity even + # if a future code path bypasses the cache. + review_agent.session_start = agent.session_start + review_agent.session_id = agent.session_id + + from model_tools import get_tool_definitions + from hermes_cli.plugins import ( + set_thread_tool_whitelist, + clear_thread_tool_whitelist, + ) + + review_whitelist = { + t["function"]["name"] + for t in get_tool_definitions( + enabled_toolsets=["memory", "skills"], + quiet_mode=True, + ) + } + set_thread_tool_whitelist( + review_whitelist, + deny_msg_fmt=( + "Background review denied non-whitelisted tool: " + "{tool_name}. Only memory/skill tools are allowed." + ), + ) + try: + review_agent.run_conversation( + user_message=( + prompt + + "\n\nYou can only call memory and skill " + "management tools. Other tools will be denied " + "at runtime — do not attempt them." + ), + conversation_history=messages_snapshot, + ) + finally: + clear_thread_tool_whitelist() + + # Tear down memory providers while stdout is still + # redirected so background thread teardown (Honcho flush, + # Hindsight sync, etc.) stays silent. The finally block + # below is a safety net for the exception path. + try: + review_agent.shutdown_memory_provider() + except Exception: + pass + try: + review_agent.close() + except Exception: + pass + review_messages = list(getattr(review_agent, "_session_messages", [])) + review_agent = None + + # Scan the review agent's messages for successful tool actions + # and surface a compact summary to the user. Tool messages + # already present in messages_snapshot must be skipped, since + # the review agent inherits that history and would otherwise + # re-surface stale "created"/"updated" messages from the prior + # conversation as if they just happened (issue #14944). + actions = summarize_background_review_actions( + review_messages, + messages_snapshot, + ) + + if actions: + summary = " · ".join(dict.fromkeys(actions)) + agent._safe_print( + f" 💾 Self-improvement review: {summary}" + ) + _bg_cb = agent.background_review_callback + if _bg_cb: + try: + _bg_cb( + f"💾 Self-improvement review: {summary}" + ) + except Exception: + pass + + except Exception as e: + logger.warning("Background memory/skill review failed: %s", e) + agent._emit_auxiliary_failure("background review", e) + finally: + # Safety-net cleanup for the exception path. Normal + # completion already shut down inside redirect_stdout above. + # Re-open devnull here so any teardown output (Honcho flush, + # Hindsight sync, background thread joins) stays silent even + # on the exception path where redirect_stdout already exited. + if review_agent is not None: + try: + with open(os.devnull, "w", encoding="utf-8") as _fn, \ + contextlib.redirect_stdout(_fn), \ + contextlib.redirect_stderr(_fn): + try: + review_agent.shutdown_memory_provider() + except Exception: + pass + try: + review_agent.close() + except Exception: + pass + except Exception: + pass + # Clear the approval callback on this bg-review thread so a + # recycled thread-id doesn't inherit a stale reference. + try: + _set_approval_callback(None) + except Exception: + pass + + +def spawn_background_review_thread( + agent: Any, + messages_snapshot: List[Dict], + review_memory: bool = False, + review_skills: bool = False, +): + """Build the review thread target and prompt for a background review. + + Returns a ``(target, prompt)`` tuple. The caller (``AIAgent._spawn_background_review``) + owns the actual ``threading.Thread`` construction so test-level patches + of ``run_agent.threading.Thread`` keep working. + """ + # Pick the right prompt based on which triggers fired + if review_memory and review_skills: + prompt = agent._COMBINED_REVIEW_PROMPT + elif review_memory: + prompt = agent._MEMORY_REVIEW_PROMPT + else: + prompt = agent._SKILL_REVIEW_PROMPT + + def _target() -> None: + _run_review_in_thread(agent, messages_snapshot, prompt) + + return _target, prompt + + +__all__ = [ + "spawn_background_review_thread", + "summarize_background_review_actions", + "build_memory_write_metadata", +] diff --git a/run_agent.py b/run_agent.py index 22848b2f20e..28171724dd6 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3358,63 +3358,9 @@ class AIAgent: review_messages: List[Dict], prior_snapshot: List[Dict], ) -> List[str]: - """Build the human-facing action summary for a background review pass. - - Walks the review agent's session messages and collects "successful tool - action" descriptions to surface to the user (e.g. "Memory updated"). - Tool messages already present in ``prior_snapshot`` are skipped so we - don't re-surface stale results from the prior conversation that the - review agent inherited via ``conversation_history`` (issue #14944). - - Matching is by ``tool_call_id`` when available, with a content-equality - fallback for tool messages that lack one. - """ - existing_tool_call_ids = set() - existing_tool_contents = set() - for prior in prior_snapshot or []: - if not isinstance(prior, dict) or prior.get("role") != "tool": - continue - tcid = prior.get("tool_call_id") - if tcid: - existing_tool_call_ids.add(tcid) - else: - content = prior.get("content") - if isinstance(content, str): - existing_tool_contents.add(content) - - actions: List[str] = [] - for msg in review_messages or []: - if not isinstance(msg, dict) or msg.get("role") != "tool": - continue - tcid = msg.get("tool_call_id") - if tcid and tcid in existing_tool_call_ids: - continue - if not tcid: - content_str = msg.get("content") - if isinstance(content_str, str) and content_str in existing_tool_contents: - continue - try: - data = json.loads(msg.get("content", "{}")) - except (json.JSONDecodeError, TypeError): - continue - if not isinstance(data, dict) or not data.get("success"): - continue - message = data.get("message", "") - target = data.get("target", "") - if "created" in message.lower(): - actions.append(message) - elif "updated" in message.lower(): - actions.append(message) - elif "added" in message.lower() or (target and "add" in message.lower()): - label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") - elif "Entry added" in message: - label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") - elif "removed" in message.lower() or "replaced" in message.lower(): - label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") - return actions + """Forwarder — see ``agent.background_review.summarize_background_review_actions``.""" + from agent.background_review import summarize_background_review_actions + return summarize_background_review_actions(review_messages, prior_snapshot) def _spawn_background_review( self, @@ -3422,219 +3368,22 @@ class AIAgent: review_memory: bool = False, review_skills: bool = False, ) -> None: - """Spawn a background thread to review the conversation for memory/skill saves. + """Spawn the background memory/skill review thread. - Creates a full AIAgent fork with the same model, tools, and context as the - main session. The review prompt is appended as the next user turn in the - forked conversation. Writes directly to the shared memory/skill stores. - Never modifies the main conversation history or produces user-visible output. + Thin wrapper — the heavy lifting lives in + ``agent.background_review.spawn_background_review_thread`` which + returns the thread target. ``threading.Thread`` is constructed + here so existing tests that patch ``run_agent.threading.Thread`` + keep working. """ - import threading - - # Pick the right prompt based on which triggers fired - if review_memory and review_skills: - prompt = self._COMBINED_REVIEW_PROMPT - elif review_memory: - prompt = self._MEMORY_REVIEW_PROMPT - else: - prompt = self._SKILL_REVIEW_PROMPT - - def _run_review(): - import contextlib - # Install a non-interactive approval callback on this worker - # thread so any dangerous-command guard the review agent trips - # resolves to "deny" instead of falling back to input() -- which - # deadlocks against the parent's prompt_toolkit TUI (#15216). - # Same pattern as _subagent_auto_deny in tools/delegate_tool.py. - def _bg_review_auto_deny(command, description, **kwargs): - logger.warning( - "Background review auto-denied dangerous command: %s (%s)", - command, description, - ) - return "deny" - try: - _set_approval_callback(_bg_review_auto_deny) - except Exception: - pass - review_agent = None - review_messages = [] - try: - with open(os.devnull, "w", encoding="utf-8") as _devnull, \ - contextlib.redirect_stdout(_devnull), \ - contextlib.redirect_stderr(_devnull): - # Inherit the parent agent's live runtime (provider, model, - # base_url, api_key, api_mode) so the fork uses the exact - # same credentials the main turn is using. Without this, - # AIAgent.__init__ re-runs auto-resolution from env vars, - # which fails for OAuth-only providers, session-scoped - # creds, or credential-pool setups where the resolver can't - # reconstruct auth from scratch -- producing the spurious - # "No LLM provider configured" warning at end of turn. - _parent_runtime = self._current_main_runtime() - _parent_api_mode = _parent_runtime.get("api_mode") or None - # The review fork needs to call agent-loop tools (memory, - # skill_manage). Those tools require Hermes' own dispatch, - # which the codex_app_server runtime bypasses entirely - # (it runs the turn inside codex's subprocess). So when - # the parent is on codex_app_server, downgrade the review - # fork to codex_responses — same auth/credentials, but - # talks to the OpenAI Responses API directly so Hermes - # owns the loop and the agent-loop tools dispatch. - if _parent_api_mode == "codex_app_server": - _parent_api_mode = "codex_responses" - review_agent = AIAgent( - model=self.model, - max_iterations=16, - quiet_mode=True, - platform=self.platform, - provider=self.provider, - api_mode=_parent_api_mode, - base_url=_parent_runtime.get("base_url") or None, - api_key=_parent_runtime.get("api_key") or None, - credential_pool=getattr(self, "_credential_pool", None), - parent_session_id=self.session_id, - ) - review_agent._memory_write_origin = "background_review" - review_agent._memory_write_context = "background_review" - review_agent._memory_store = self._memory_store - review_agent._memory_enabled = self._memory_enabled - review_agent._user_profile_enabled = self._user_profile_enabled - review_agent._memory_nudge_interval = 0 - review_agent._skill_nudge_interval = 0 - # Suppress all status/warning emits from the fork so the - # user only sees the final successful-action summary. - # Without this, mid-review "Iteration budget exhausted", - # rate-limit retries, compression warnings, and other - # lifecycle messages bubble up through _emit_status -> - # _vprint and leak past the stdout redirect (they go via - # _print_fn/status_callback, which bypass sys.stdout). - review_agent.suppress_status_output = True - # Inherit the parent's cached system prompt verbatim so - # the review fork's outbound HTTP request hits the same - # Anthropic/OpenRouter prefix cache the parent warmed. - # Without this, the fork rebuilds the system prompt from - # scratch (fresh _hermes_now() timestamp, fresh - # session_id, narrower toolset → different skills_prompt) - # and the byte-exact prefix-cache key misses. See - # issue #25322 and PR #17276 for the full analysis + - # measured impact (~26% end-to-end cost reduction on - # Sonnet 4.5). - review_agent._cached_system_prompt = self._cached_system_prompt - # Defensive: pin session_start + session_id to the - # parent's so any code path that re-renders parts of - # the system prompt (compression, plugin hooks) still - # produces byte-identical output. The cached-prompt - # assignment above already short-circuits the normal - # rebuild path, but these pins guarantee parity even - # if a future code path bypasses the cache. - review_agent.session_start = self.session_start - review_agent.session_id = self.session_id - - from model_tools import get_tool_definitions - from hermes_cli.plugins import ( - set_thread_tool_whitelist, - clear_thread_tool_whitelist, - ) - - review_whitelist = { - t["function"]["name"] - for t in get_tool_definitions( - enabled_toolsets=["memory", "skills"], - quiet_mode=True, - ) - } - set_thread_tool_whitelist( - review_whitelist, - deny_msg_fmt=( - "Background review denied non-whitelisted tool: " - "{tool_name}. Only memory/skill tools are allowed." - ), - ) - try: - review_agent.run_conversation( - user_message=( - prompt - + "\n\nYou can only call memory and skill " - "management tools. Other tools will be denied " - "at runtime — do not attempt them." - ), - conversation_history=messages_snapshot, - ) - finally: - clear_thread_tool_whitelist() - - # Tear down memory providers while stdout is still - # redirected so background thread teardown (Honcho flush, - # Hindsight sync, etc.) stays silent. The finally block - # below is a safety net for the exception path. - try: - review_agent.shutdown_memory_provider() - except Exception: - pass - try: - review_agent.close() - except Exception: - pass - review_messages = list(getattr(review_agent, "_session_messages", [])) - review_agent = None - - # Scan the review agent's messages for successful tool actions - # and surface a compact summary to the user. Tool messages - # already present in messages_snapshot must be skipped, since - # the review agent inherits that history and would otherwise - # re-surface stale "created"/"updated" messages from the prior - # conversation as if they just happened (issue #14944). - actions = self._summarize_background_review_actions( - review_messages, - messages_snapshot, - ) - - if actions: - summary = " · ".join(dict.fromkeys(actions)) - self._safe_print( - f" 💾 Self-improvement review: {summary}" - ) - _bg_cb = self.background_review_callback - if _bg_cb: - try: - _bg_cb( - f"💾 Self-improvement review: {summary}" - ) - except Exception: - pass - - except Exception as e: - logger.warning("Background memory/skill review failed: %s", e) - self._emit_auxiliary_failure("background review", e) - finally: - # Safety-net cleanup for the exception path. Normal - # completion already shut down inside redirect_stdout above. - # Re-open devnull here so any teardown output (Honcho flush, - # Hindsight sync, background thread joins) stays silent even - # on the exception path where redirect_stdout already exited. - if review_agent is not None: - try: - with open(os.devnull, "w", encoding="utf-8") as _fn, \ - contextlib.redirect_stdout(_fn), \ - contextlib.redirect_stderr(_fn): - try: - review_agent.shutdown_memory_provider() - except Exception: - pass - try: - review_agent.close() - except Exception: - pass - except Exception: - pass - # Clear the approval callback on this bg-review thread so a - # recycled thread-id doesn't inherit a stale reference. - try: - _set_approval_callback(None) - except Exception: - pass - - t = threading.Thread(target=_run_review, daemon=True, name="bg-review") + from agent.background_review import spawn_background_review_thread + target, _prompt = spawn_background_review_thread( + self, + messages_snapshot, + review_memory=review_memory, + review_skills=review_skills, + ) + t = threading.Thread(target=target, daemon=True, name="bg-review") t.start() def _build_memory_write_metadata( @@ -3645,23 +3394,15 @@ class AIAgent: task_id: Optional[str] = None, tool_call_id: Optional[str] = None, ) -> Dict[str, Any]: - """Build provenance metadata for external memory-provider mirrors.""" - metadata: Dict[str, Any] = { - "write_origin": write_origin or getattr(self, "_memory_write_origin", "assistant_tool"), - "execution_context": ( - execution_context - or getattr(self, "_memory_write_context", "foreground") - ), - "session_id": self.session_id or "", - "parent_session_id": self._parent_session_id or "", - "platform": self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), - "tool_name": "memory", - } - if task_id: - metadata["task_id"] = task_id - if tool_call_id: - metadata["tool_call_id"] = tool_call_id - return {k: v for k, v in metadata.items() if v not in {None, ""}} + """Forwarder — see ``agent.background_review.build_memory_write_metadata``.""" + from agent.background_review import build_memory_write_metadata + return build_memory_write_metadata( + self, + write_origin=write_origin, + execution_context=execution_context, + task_id=task_id, + tool_call_id=tool_call_id, + ) def _apply_persist_user_message_override(self, messages: List[Dict]) -> None: """Rewrite the current-turn user message before persistence/return. From 5311d9959e19477aac8aa7deca46c1ee0b8e7000 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:09:33 -0700 Subject: [PATCH 005/142] refactor(run_agent): extract context compression to agent/conversation_compression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move four compression-related methods to a dedicated module: * check_compression_model_feasibility — startup probe + auto-lowered threshold + hard floor * replay_compression_warning — re-emit stored warning through gateway status_callback * compress_context — run compressor, split SQLite session, notify plugins+memory * try_shrink_image_parts_in_messages — image-too-large recovery via re-encode AIAgent keeps thin forwarder methods so existing call sites and tests that patch run_agent.AIAgent methods keep working. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure as before). run_agent.py: 15013 -> 14535 lines (-478). --- agent/conversation_compression.py | 547 ++++++++++++++++++++++++++++++ run_agent.py | 486 +------------------------- 2 files changed, 561 insertions(+), 472 deletions(-) create mode 100644 agent/conversation_compression.py diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py new file mode 100644 index 00000000000..90c637ee4fa --- /dev/null +++ b/agent/conversation_compression.py @@ -0,0 +1,547 @@ +"""Context compression — extract the AIAgent methods that drive summarisation. + +Three concerns live here: + +* :func:`check_compression_model_feasibility` — startup probe of the + configured auxiliary compression model. Warns when the aux context + window can't fit the main model's compression threshold; auto-lowers + the session threshold when possible; hard-rejects auxes below + ``MINIMUM_CONTEXT_LENGTH``. + +* :func:`replay_compression_warning` — re-emit a stored warning through + the gateway ``status_callback`` once it's wired up (the callback is + set after :class:`AIAgent` construction). + +* :func:`compress_context` — the actual compression call. Runs the + configured compressor, splits the SQLite session, rotates the + session_id, notifies plugin context engines / memory providers, and + returns the compressed message list and freshly-built system prompt. + +* :func:`try_shrink_image_parts_in_messages` — image-too-large recovery + helper that re-encodes ``data:image/...;base64,...`` parts at a smaller + size so retries can fit under provider ceilings (Anthropic's 5 MB). + +``run_agent`` keeps thin wrappers for each so existing call sites +(``self._compress_context(...)``) keep working. Tests that exercise +these paths see no behavioural change. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, List, Optional, Tuple + +from agent.model_metadata import estimate_request_tokens_rough + +logger = logging.getLogger(__name__) + + +def check_compression_model_feasibility(agent: Any) -> None: + """Warn at session start if the auxiliary compression model's context + window is smaller than the main model's compression threshold. + + When the auxiliary model cannot fit the content that needs summarising, + compression will either fail outright (the LLM call errors) or produce + a severely truncated summary. + + Called during ``AIAgent.__init__`` so CLI users see the warning + immediately (via ``_vprint``). The gateway sets ``status_callback`` + *after* construction, so :func:`replay_compression_warning` re-sends + the stored warning through the callback on the first + ``run_conversation()`` call. + """ + if not agent.compression_enabled: + return + try: + from agent.auxiliary_client import ( + _resolve_task_provider_model, + get_text_auxiliary_client, + ) + from agent.model_metadata import ( + MINIMUM_CONTEXT_LENGTH, + get_model_context_length, + ) + + client, aux_model = get_text_auxiliary_client( + "compression", + main_runtime=agent._current_main_runtime(), + ) + # Best-effort aux provider label for the warning message. The + # configured provider may be "auto", in which case we fall back + # to the client's base_url hostname so the user can still tell + # where the compression model is actually being called. + try: + _aux_cfg_provider, _, _, _, _ = _resolve_task_provider_model("compression") + except Exception: + _aux_cfg_provider = "" + if client is None or not aux_model: + msg = ( + "⚠ No auxiliary LLM provider configured — context " + "compression will drop middle turns without a summary. " + "Run `hermes setup` or set OPENROUTER_API_KEY." + ) + agent._compression_warning = msg + agent._emit_status(msg) + logger.warning( + "No auxiliary LLM provider for compression — " + "summaries will be unavailable." + ) + return + + aux_base_url = str(getattr(client, "base_url", "")) + aux_api_key = str(getattr(client, "api_key", "")) + + aux_context = get_model_context_length( + aux_model, + base_url=aux_base_url, + api_key=aux_api_key, + config_context_length=getattr(agent, "_aux_compression_context_length_config", None), + # Each model must be resolved with its own provider so that + # provider-specific paths (e.g. Bedrock static table, OpenRouter API) + # are invoked for the correct client, not inherited from the main model. + provider=(_aux_cfg_provider if _aux_cfg_provider and _aux_cfg_provider != "auto" else getattr(agent, "provider", "")), + custom_providers=agent._custom_providers, + ) + + # Hard floor: the auxiliary compression model must have at least + # MINIMUM_CONTEXT_LENGTH (64K) tokens of context. The main model + # is already required to meet this floor (checked earlier in + # __init__), so the compression model must too — otherwise it + # cannot summarise a full threshold-sized window of main-model + # content. Mirrors the main-model rejection pattern. + if aux_context and aux_context < MINIMUM_CONTEXT_LENGTH: + raise ValueError( + f"Auxiliary compression model {aux_model} has a context " + f"window of {aux_context:,} tokens, which is below the " + f"minimum {MINIMUM_CONTEXT_LENGTH:,} required by Hermes " + f"Agent. Choose a compression model with at least " + f"{MINIMUM_CONTEXT_LENGTH // 1000}K context (set " + f"auxiliary.compression.model in config.yaml), or set " + f"auxiliary.compression.context_length to override the " + f"detected value if it is wrong." + ) + + threshold = agent.context_compressor.threshold_tokens + if aux_context < threshold: + # Auto-correct: lower the live session threshold so + # compression actually works this session. The hard floor + # above guarantees aux_context >= MINIMUM_CONTEXT_LENGTH, + # so the new threshold is always >= 64K. + # + # The compression summariser sends a single user-role + # prompt (no system prompt, no tools) to the aux model, so + # new_threshold == aux_context is safe: the request is + # the raw messages plus a small summarisation instruction. + old_threshold = threshold + new_threshold = aux_context + agent.context_compressor.threshold_tokens = new_threshold + # Keep threshold_percent in sync so future main-model + # context_length changes (update_model) re-derive from a + # sensible number rather than the original too-high value. + main_ctx = agent.context_compressor.context_length + if main_ctx: + agent.context_compressor.threshold_percent = ( + new_threshold / main_ctx + ) + safe_pct = int((aux_context / main_ctx) * 100) if main_ctx else 50 + # Build human-readable "model (provider)" labels for both + # the main model and the compression model so users can + # tell at a glance which provider each side is actually + # using. When the configured provider is empty or "auto", + # fall back to the client's base_url hostname. + _main_model = getattr(agent, "model", "") or "?" + _main_provider = getattr(agent, "provider", "") or "" + _aux_provider_label = ( + _aux_cfg_provider + if _aux_cfg_provider and _aux_cfg_provider != "auto" + else "" + ) + if not _aux_provider_label: + try: + from urllib.parse import urlparse + _aux_provider_label = ( + urlparse(aux_base_url).hostname or aux_base_url + ) + except Exception: + _aux_provider_label = aux_base_url or "auto" + _main_label = ( + f"{_main_model} ({_main_provider})" + if _main_provider + else _main_model + ) + _aux_label = f"{aux_model} ({_aux_provider_label})" + msg = ( + f"⚠ Compression model {_aux_label} context is " + f"{aux_context:,} tokens, but the main model " + f"{_main_label}'s compression threshold was " + f"{old_threshold:,} tokens. " + f"Auto-lowered this session's threshold to " + f"{new_threshold:,} tokens so compression can run.\n" + f" To make this permanent, edit config.yaml — either:\n" + f" 1. Use a larger compression model:\n" + f" auxiliary:\n" + f" compression:\n" + f" model: \n" + f" 2. Lower the compression threshold:\n" + f" compression:\n" + f" threshold: 0.{safe_pct:02d}" + ) + agent._compression_warning = msg + agent._emit_status(msg) + logger.warning( + "Auxiliary compression model %s has %d token context, " + "below the main model's compression threshold of %d " + "tokens — auto-lowered session threshold to %d to " + "keep compression working.", + aux_model, + aux_context, + old_threshold, + new_threshold, + ) + except ValueError: + # Hard rejections (aux below minimum context) must propagate + # so the session refuses to start. + raise + except Exception as exc: + logger.debug( + "Compression feasibility check failed (non-fatal): %s", exc + ) + + +def replay_compression_warning(agent: Any) -> None: + """Re-send the compression warning through ``status_callback``. + + During ``__init__`` the gateway's ``status_callback`` is not yet + wired, so ``_emit_status`` only reaches ``_vprint`` (CLI). This + method is called once at the start of the first + ``run_conversation()`` — by then the gateway has set the callback, + so every platform (Telegram, Discord, Slack, etc.) receives the + warning. + """ + msg = getattr(agent, "_compression_warning", None) + if msg and agent.status_callback: + try: + agent.status_callback("lifecycle", msg) + except Exception: + pass + + +def compress_context( + agent: Any, + messages: list, + system_message: str, + *, + approx_tokens: Optional[int] = None, + task_id: str = "default", + focus_topic: Optional[str] = None, +) -> Tuple[list, str]: + """Compress conversation context and split the session in SQLite. + + Args: + agent: The owning :class:`AIAgent`. + messages: Current message history (will be summarised). + system_message: Current system prompt; rebuilt after compression. + approx_tokens: Pre-compression token estimate, logged for ops. + task_id: Tool task scope (used for clearing file-read dedup state). + focus_topic: Optional focus string for guided compression — the + summariser will prioritise preserving information related to + this topic. Inspired by Claude Code's ``/compact ``. + + Returns: + ``(compressed_messages, new_system_prompt)`` tuple. + """ + _pre_msg_count = len(messages) + logger.info( + "context compression started: session=%s messages=%d tokens=~%s model=%s focus=%r", + agent.session_id or "none", _pre_msg_count, + f"{approx_tokens:,}" if approx_tokens else "unknown", agent.model, + focus_topic, + ) + agent._emit_status( + "🗜️ Compacting context — summarizing earlier conversation so I can continue..." + ) + + # Notify external memory provider before compression discards context + if agent._memory_manager: + try: + agent._memory_manager.on_pre_compress(messages) + except Exception: + pass + + try: + compressed = agent.context_compressor.compress(messages, current_tokens=approx_tokens, focus_topic=focus_topic) + except TypeError: + # Plugin context engine with strict signature that doesn't accept + # focus_topic — fall back to calling without it. + compressed = agent.context_compressor.compress(messages, current_tokens=approx_tokens) + + summary_error = getattr(agent.context_compressor, "_last_summary_error", None) + if summary_error: + if getattr(agent, "_last_compression_summary_warning", None) != summary_error: + agent._last_compression_summary_warning = summary_error + agent._emit_warning( + f"⚠ Compression summary failed: {summary_error}. " + "Inserted a fallback context marker." + ) + else: + # No hard failure — but did the configured aux model error out + # and get recovered by retrying on main? Surface that so users + # know their auxiliary.compression.model setting is broken even + # though compression succeeded. + _aux_fail_model = getattr(agent.context_compressor, "_last_aux_model_failure_model", None) + _aux_fail_err = getattr(agent.context_compressor, "_last_aux_model_failure_error", None) + if _aux_fail_model: + # Dedup on (model, error) so we don't spam on every compaction + _aux_key = (_aux_fail_model, _aux_fail_err) + if getattr(agent, "_last_aux_fallback_warning_key", None) != _aux_key: + agent._last_aux_fallback_warning_key = _aux_key + agent._emit_warning( + f"ℹ Configured compression model '{_aux_fail_model}' failed " + f"({_aux_fail_err or 'unknown error'}). Recovered using main model — " + "check auxiliary.compression.model in config.yaml." + ) + + todo_snapshot = agent._todo_store.format_for_injection() + if todo_snapshot: + compressed.append({"role": "user", "content": todo_snapshot}) + + agent._invalidate_system_prompt() + new_system_prompt = agent._build_system_prompt(system_message) + agent._cached_system_prompt = new_system_prompt + + if agent._session_db: + try: + # Propagate title to the new session with auto-numbering + old_title = agent._session_db.get_session_title(agent.session_id) + # Trigger memory extraction on the old session before it rotates. + agent.commit_memory_session(messages) + agent._session_db.end_session(agent.session_id, "compression") + old_session_id = agent.session_id + agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}" + os.environ["HERMES_SESSION_ID"] = agent.session_id + try: + from gateway.session_context import _SESSION_ID + _SESSION_ID.set(agent.session_id) + except Exception: + pass + # Update session_log_file to point to the new session's JSON file + agent.session_log_file = agent.logs_dir / f"session_{agent.session_id}.json" + agent._session_db_created = False + agent._session_db.create_session( + session_id=agent.session_id, + source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), + model=agent.model, + model_config=agent._session_init_model_config, + parent_session_id=old_session_id, + ) + agent._session_db_created = True + # Auto-number the title for the continuation session + if old_title: + try: + new_title = agent._session_db.get_next_title_in_lineage(old_title) + agent._session_db.set_session_title(agent.session_id, new_title) + except (ValueError, Exception) as e: + logger.debug("Could not propagate title on compression: %s", e) + agent._session_db.update_system_prompt(agent.session_id, new_system_prompt) + # Reset flush cursor — new session starts with no messages written + agent._last_flushed_db_idx = 0 + except Exception as e: + logger.warning("Session DB compression split failed — new session will NOT be indexed: %s", e) + + # Notify the context engine that the session_id rotated because of + # compression (not a fresh /new). Plugin engines (e.g. hermes-lcm) use + # boundary_reason="compression" to preserve DAG lineage across the + # rollover instead of re-initializing fresh per-session state. + # See hermes-lcm#68. Built-in ContextCompressor ignores kwargs. + try: + _old_sid = locals().get("old_session_id") + if _old_sid and hasattr(agent.context_compressor, "on_session_start"): + agent.context_compressor.on_session_start( + agent.session_id or "", + boundary_reason="compression", + old_session_id=_old_sid, + ) + except Exception as _ce_err: + logger.debug("context engine on_session_start (compression): %s", _ce_err) + + # Notify memory providers of the compression-driven session_id rotation + # so provider-cached per-session state (Hindsight's _document_id, + # accumulated turn buffers, counters) refreshes. reset=False because + # the logical conversation continues; only the id and DB row rolled + # over. See #6672. + try: + _old_sid = locals().get("old_session_id") + if _old_sid and agent._memory_manager: + agent._memory_manager.on_session_switch( + agent.session_id or "", + parent_session_id=_old_sid, + reset=False, + reason="compression", + ) + except Exception as _me_err: + logger.debug("memory manager on_session_switch (compression): %s", _me_err) + + # Warn on repeated compressions (quality degrades with each pass) + _cc = agent.context_compressor.compression_count + if _cc >= 2: + agent._vprint( + f"{agent.log_prefix}⚠️ Session compressed {_cc} times — " + f"accuracy may degrade. Consider /new to start fresh.", + force=True, + ) + + # Update token estimate after compaction so pressure calculations + # use the post-compression count, not the stale pre-compression one. + # Use estimate_request_tokens_rough() so tool schemas are included — + # with 50+ tools enabled, schemas alone can add 20-30K tokens, and + # omitting them delays the next compression cycle far past the + # configured threshold (issue #14695). + _compressed_est = estimate_request_tokens_rough( + compressed, + system_prompt=new_system_prompt or "", + tools=agent.tools or None, + ) + agent.context_compressor.last_prompt_tokens = _compressed_est + agent.context_compressor.last_completion_tokens = 0 + + # Clear the file-read dedup cache. After compression the original + # read content is summarised away — if the model re-reads the same + # file it needs the full content, not a "file unchanged" stub. + try: + from tools.file_tools import reset_file_dedup + reset_file_dedup(task_id) + except Exception: + pass + + logger.info( + "context compression done: session=%s messages=%d->%d tokens=~%s", + agent.session_id or "none", _pre_msg_count, len(compressed), + f"{_compressed_est:,}", + ) + return compressed, new_system_prompt + + +def try_shrink_image_parts_in_messages(api_messages: list) -> bool: + """Re-encode all native image parts at a smaller size to recover from + image-too-large errors (Anthropic 5 MB, unknown other providers). + + Mutates ``api_messages`` in place. Returns True if any image part was + actually replaced, False if there were no image parts to shrink or + Pillow couldn't help (caller should surface the original error). + + Strategy: look for ``image_url`` / ``input_image`` parts carrying a + ``data:image/...;base64,...`` payload. For each one whose encoded + size exceeds 4 MB (a safe target that slides under Anthropic's 5 MB + ceiling with header overhead), write the base64 to a tempfile, call + ``vision_tools._resize_image_for_vision`` to produce a smaller data + URL, and substitute it in place. + + Non-data-URL images (http/https URLs) are not touched — the provider + fetches those itself and the size limit is different. + """ + if not api_messages: + return False + + try: + from tools.vision_tools import _resize_image_for_vision + except Exception as exc: + logger.warning("image-shrink recovery: vision_tools unavailable — %s", exc) + return False + + # 4 MB target leaves comfortable headroom under Anthropic's 5 MB. + # Non-Anthropic providers we haven't observed rejecting are fine with + # much larger; shrinking to 4 MB here loses quality but only fires + # after a confirmed provider rejection, so the alternative is failure. + target_bytes = 4 * 1024 * 1024 + changed_count = 0 + + def _shrink_data_url(url: str) -> Optional[str]: + """Return a smaller data URL, or None if shrink can't help.""" + if not isinstance(url, str) or not url.startswith("data:"): + return None + if len(url) <= target_bytes: + # This specific image wasn't the oversized one. + return None + try: + header, _, data = url.partition(",") + mime = "image/jpeg" + if header.startswith("data:"): + mime_part = header[len("data:"):].split(";", 1)[0].strip() + if mime_part.startswith("image/"): + mime = mime_part + import base64 as _b64 + raw = _b64.b64decode(data) + suffix = { + "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", + "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/bmp": ".bmp", + }.get(mime, ".jpg") + tmp = tempfile.NamedTemporaryFile( + prefix="hermes_shrink_", suffix=suffix, delete=False, + ) + try: + tmp.write(raw) + tmp.close() + resized = _resize_image_for_vision( + Path(tmp.name), + mime_type=mime, + max_base64_bytes=target_bytes, + ) + finally: + try: + Path(tmp.name).unlink(missing_ok=True) + except Exception: + pass + if not resized or len(resized) >= len(url): + # Shrink didn't help (or made it bigger — corrupt input?). + return None + return resized + except Exception as exc: + logger.warning("image-shrink recovery: re-encode failed — %s", exc) + return None + + for msg in api_messages: + if not isinstance(msg, dict): + continue + content = msg.get("content") + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict): + continue + ptype = part.get("type") + if ptype not in {"image_url", "input_image"}: + continue + image_value = part.get("image_url") + # OpenAI chat.completions: {"image_url": {"url": "data:..."}} + # OpenAI Responses: {"image_url": "data:..."} + if isinstance(image_value, dict): + url = image_value.get("url", "") + resized = _shrink_data_url(url) + if resized: + image_value["url"] = resized + changed_count += 1 + elif isinstance(image_value, str): + resized = _shrink_data_url(image_value) + if resized: + part["image_url"] = resized + changed_count += 1 + + if changed_count: + logger.info( + "image-shrink recovery: re-encoded %d image part(s) to fit under %.0f MB", + changed_count, target_bytes / (1024 * 1024), + ) + return changed_count > 0 + + +__all__ = [ + "check_compression_model_feasibility", + "replay_compression_warning", + "compress_context", + "try_shrink_image_parts_in_messages", +] diff --git a/run_agent.py b/run_agent.py index 28171724dd6..dee7d365e7e 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2376,192 +2376,14 @@ class AIAgent: } def _check_compression_model_feasibility(self) -> None: - """Warn at session start if the auxiliary compression model's context - window is smaller than the main model's compression threshold. - - When the auxiliary model cannot fit the content that needs summarising, - compression will either fail outright (the LLM call errors) or produce - a severely truncated summary. - - Called during ``__init__`` so CLI users see the warning immediately - (via ``_vprint``). The gateway sets ``status_callback`` *after* - construction, so ``_replay_compression_warning()`` re-sends the - stored warning through the callback on the first - ``run_conversation()`` call. - """ - if not self.compression_enabled: - return - try: - from agent.auxiliary_client import ( - _resolve_task_provider_model, - get_text_auxiliary_client, - ) - from agent.model_metadata import ( - MINIMUM_CONTEXT_LENGTH, - get_model_context_length, - ) - - client, aux_model = get_text_auxiliary_client( - "compression", - main_runtime=self._current_main_runtime(), - ) - # Best-effort aux provider label for the warning message. The - # configured provider may be "auto", in which case we fall back - # to the client's base_url hostname so the user can still tell - # where the compression model is actually being called. - try: - _aux_cfg_provider, _, _, _, _ = _resolve_task_provider_model("compression") - except Exception: - _aux_cfg_provider = "" - if client is None or not aux_model: - msg = ( - "⚠ No auxiliary LLM provider configured — context " - "compression will drop middle turns without a summary. " - "Run `hermes setup` or set OPENROUTER_API_KEY." - ) - self._compression_warning = msg - self._emit_status(msg) - logger.warning( - "No auxiliary LLM provider for compression — " - "summaries will be unavailable." - ) - return - - aux_base_url = str(getattr(client, "base_url", "")) - aux_api_key = str(getattr(client, "api_key", "")) - - aux_context = get_model_context_length( - aux_model, - base_url=aux_base_url, - api_key=aux_api_key, - config_context_length=getattr(self, "_aux_compression_context_length_config", None), - # Each model must be resolved with its own provider so that - # provider-specific paths (e.g. Bedrock static table, OpenRouter API) - # are invoked for the correct client, not inherited from the main model. - provider=(_aux_cfg_provider if _aux_cfg_provider and _aux_cfg_provider != "auto" else getattr(self, "provider", "")), - custom_providers=self._custom_providers, - ) - - # Hard floor: the auxiliary compression model must have at least - # MINIMUM_CONTEXT_LENGTH (64K) tokens of context. The main model - # is already required to meet this floor (checked earlier in - # __init__), so the compression model must too — otherwise it - # cannot summarise a full threshold-sized window of main-model - # content. Mirrors the main-model rejection pattern. - if aux_context and aux_context < MINIMUM_CONTEXT_LENGTH: - raise ValueError( - f"Auxiliary compression model {aux_model} has a context " - f"window of {aux_context:,} tokens, which is below the " - f"minimum {MINIMUM_CONTEXT_LENGTH:,} required by Hermes " - f"Agent. Choose a compression model with at least " - f"{MINIMUM_CONTEXT_LENGTH // 1000}K context (set " - f"auxiliary.compression.model in config.yaml), or set " - f"auxiliary.compression.context_length to override the " - f"detected value if it is wrong." - ) - - threshold = self.context_compressor.threshold_tokens - if aux_context < threshold: - # Auto-correct: lower the live session threshold so - # compression actually works this session. The hard floor - # above guarantees aux_context >= MINIMUM_CONTEXT_LENGTH, - # so the new threshold is always >= 64K. - # - # The compression summariser sends a single user-role - # prompt (no system prompt, no tools) to the aux model, so - # new_threshold == aux_context is safe: the request is - # the raw messages plus a small summarisation instruction. - old_threshold = threshold - new_threshold = aux_context - self.context_compressor.threshold_tokens = new_threshold - # Keep threshold_percent in sync so future main-model - # context_length changes (update_model) re-derive from a - # sensible number rather than the original too-high value. - main_ctx = self.context_compressor.context_length - if main_ctx: - self.context_compressor.threshold_percent = ( - new_threshold / main_ctx - ) - safe_pct = int((aux_context / main_ctx) * 100) if main_ctx else 50 - # Build human-readable "model (provider)" labels for both - # the main model and the compression model so users can - # tell at a glance which provider each side is actually - # using. When the configured provider is empty or "auto", - # fall back to the client's base_url hostname. - _main_model = getattr(self, "model", "") or "?" - _main_provider = getattr(self, "provider", "") or "" - _aux_provider_label = ( - _aux_cfg_provider - if _aux_cfg_provider and _aux_cfg_provider != "auto" - else "" - ) - if not _aux_provider_label: - try: - from urllib.parse import urlparse - _aux_provider_label = ( - urlparse(aux_base_url).hostname or aux_base_url - ) - except Exception: - _aux_provider_label = aux_base_url or "auto" - _main_label = ( - f"{_main_model} ({_main_provider})" - if _main_provider - else _main_model - ) - _aux_label = f"{aux_model} ({_aux_provider_label})" - msg = ( - f"⚠ Compression model {_aux_label} context is " - f"{aux_context:,} tokens, but the main model " - f"{_main_label}'s compression threshold was " - f"{old_threshold:,} tokens. " - f"Auto-lowered this session's threshold to " - f"{new_threshold:,} tokens so compression can run.\n" - f" To make this permanent, edit config.yaml — either:\n" - f" 1. Use a larger compression model:\n" - f" auxiliary:\n" - f" compression:\n" - f" model: \n" - f" 2. Lower the compression threshold:\n" - f" compression:\n" - f" threshold: 0.{safe_pct:02d}" - ) - self._compression_warning = msg - self._emit_status(msg) - logger.warning( - "Auxiliary compression model %s has %d token context, " - "below the main model's compression threshold of %d " - "tokens — auto-lowered session threshold to %d to " - "keep compression working.", - aux_model, - aux_context, - old_threshold, - new_threshold, - ) - except ValueError: - # Hard rejections (aux below minimum context) must propagate - # so the session refuses to start. - raise - except Exception as exc: - logger.debug( - "Compression feasibility check failed (non-fatal): %s", exc - ) + """Forwarder — see ``agent.conversation_compression.check_compression_model_feasibility``.""" + from agent.conversation_compression import check_compression_model_feasibility + check_compression_model_feasibility(self) def _replay_compression_warning(self) -> None: - """Re-send the compression warning through ``status_callback``. - - During ``__init__`` the gateway's ``status_callback`` is not yet - wired, so ``_emit_status`` only reaches ``_vprint`` (CLI). This - method is called once at the start of the first - ``run_conversation()`` — by then the gateway has set the callback, - so every platform (Telegram, Discord, Slack, etc.) receives the - warning. - """ - msg = getattr(self, "_compression_warning", None) - if msg and self.status_callback: - try: - self.status_callback("lifecycle", msg) - except Exception: - pass + """Forwarder — see ``agent.conversation_compression.replay_compression_warning``.""" + from agent.conversation_compression import replay_compression_warning + replay_compression_warning(self) def _is_direct_openai_url(self, base_url: str = None) -> bool: """Return True when a base URL targets OpenAI's native API.""" @@ -8297,116 +8119,9 @@ class AIAgent: return summary def _try_shrink_image_parts_in_messages(self, api_messages: list) -> bool: - """Re-encode all native image parts at a smaller size to recover from - image-too-large errors (Anthropic 5 MB, unknown other providers). - - Mutates ``api_messages`` in place. Returns True if any image part was - actually replaced, False if there were no image parts to shrink or - Pillow couldn't help (caller should surface the original error). - - Strategy: look for ``image_url`` / ``input_image`` parts carrying a - ``data:image/...;base64,...`` payload. For each one whose encoded - size exceeds 4 MB (a safe target that slides under Anthropic's 5 MB - ceiling with header overhead), write the base64 to a tempfile, call - ``vision_tools._resize_image_for_vision`` to produce a smaller data - URL, and substitute it in place. - - Non-data-URL images (http/https URLs) are not touched — the provider - fetches those itself and the size limit is different. - """ - if not api_messages: - return False - - try: - from tools.vision_tools import _resize_image_for_vision - except Exception as exc: - logger.warning("image-shrink recovery: vision_tools unavailable — %s", exc) - return False - - # 4 MB target leaves comfortable headroom under Anthropic's 5 MB. - # Non-Anthropic providers we haven't observed rejecting are fine with - # much larger; shrinking to 4 MB here loses quality but only fires - # after a confirmed provider rejection, so the alternative is failure. - target_bytes = 4 * 1024 * 1024 - changed_count = 0 - - def _shrink_data_url(url: str) -> Optional[str]: - """Return a smaller data URL, or None if shrink can't help.""" - if not isinstance(url, str) or not url.startswith("data:"): - return None - if len(url) <= target_bytes: - # This specific image wasn't the oversized one. - return None - try: - header, _, data = url.partition(",") - mime = "image/jpeg" - if header.startswith("data:"): - mime_part = header[len("data:"):].split(";", 1)[0].strip() - if mime_part.startswith("image/"): - mime = mime_part - import base64 as _b64 - raw = _b64.b64decode(data) - suffix = { - "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", - "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/bmp": ".bmp", - }.get(mime, ".jpg") - tmp = tempfile.NamedTemporaryFile( - prefix="hermes_shrink_", suffix=suffix, delete=False, - ) - try: - tmp.write(raw) - tmp.close() - resized = _resize_image_for_vision( - Path(tmp.name), - mime_type=mime, - max_base64_bytes=target_bytes, - ) - finally: - try: - Path(tmp.name).unlink(missing_ok=True) - except Exception: - pass - if not resized or len(resized) >= len(url): - # Shrink didn't help (or made it bigger — corrupt input?). - return None - return resized - except Exception as exc: - logger.warning("image-shrink recovery: re-encode failed — %s", exc) - return None - - for msg in api_messages: - if not isinstance(msg, dict): - continue - content = msg.get("content") - if not isinstance(content, list): - continue - for part in content: - if not isinstance(part, dict): - continue - ptype = part.get("type") - if ptype not in {"image_url", "input_image"}: - continue - image_value = part.get("image_url") - # OpenAI chat.completions: {"image_url": {"url": "data:..."}} - # OpenAI Responses: {"image_url": "data:..."} - if isinstance(image_value, dict): - url = image_value.get("url", "") - resized = _shrink_data_url(url) - if resized: - image_value["url"] = resized - changed_count += 1 - elif isinstance(image_value, str): - resized = _shrink_data_url(image_value) - if resized: - part["image_url"] = resized - changed_count += 1 - - if changed_count: - logger.info( - "image-shrink recovery: re-encoded %d image part(s) to fit under %.0f MB", - changed_count, target_bytes / (1024 * 1024), - ) - return changed_count > 0 + """Forwarder — see ``agent.conversation_compression.try_shrink_image_parts_in_messages``.""" + from agent.conversation_compression import try_shrink_image_parts_in_messages + return try_shrink_image_parts_in_messages(api_messages) def _anthropic_preserve_dots(self) -> bool: """True when using an anthropic-compatible endpoint that preserves dots in model names. @@ -9318,185 +9033,12 @@ class AIAgent: return self.api_mode != "codex_responses" def _compress_context(self, messages: list, system_message: str, *, approx_tokens: int = None, task_id: str = "default", focus_topic: str = None) -> tuple: - """Compress conversation context and split the session in SQLite. - - Args: - focus_topic: Optional focus string for guided compression — the - summariser will prioritise preserving information related to - this topic. Inspired by Claude Code's ``/compact ``. - - Returns: - (compressed_messages, new_system_prompt) tuple - """ - _pre_msg_count = len(messages) - logger.info( - "context compression started: session=%s messages=%d tokens=~%s model=%s focus=%r", - self.session_id or "none", _pre_msg_count, - f"{approx_tokens:,}" if approx_tokens else "unknown", self.model, - focus_topic, + """Forwarder — see ``agent.conversation_compression.compress_context``.""" + from agent.conversation_compression import compress_context + return compress_context( + self, messages, system_message, + approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, ) - self._emit_status( - "🗜️ Compacting context — summarizing earlier conversation so I can continue..." - ) - - # Notify external memory provider before compression discards context - if self._memory_manager: - try: - self._memory_manager.on_pre_compress(messages) - except Exception: - pass - - try: - compressed = self.context_compressor.compress(messages, current_tokens=approx_tokens, focus_topic=focus_topic) - except TypeError: - # Plugin context engine with strict signature that doesn't accept - # focus_topic — fall back to calling without it. - compressed = self.context_compressor.compress(messages, current_tokens=approx_tokens) - - summary_error = getattr(self.context_compressor, "_last_summary_error", None) - if summary_error: - if getattr(self, "_last_compression_summary_warning", None) != summary_error: - self._last_compression_summary_warning = summary_error - self._emit_warning( - f"⚠ Compression summary failed: {summary_error}. " - "Inserted a fallback context marker." - ) - else: - # No hard failure — but did the configured aux model error out - # and get recovered by retrying on main? Surface that so users - # know their auxiliary.compression.model setting is broken even - # though compression succeeded. - _aux_fail_model = getattr(self.context_compressor, "_last_aux_model_failure_model", None) - _aux_fail_err = getattr(self.context_compressor, "_last_aux_model_failure_error", None) - if _aux_fail_model: - # Dedup on (model, error) so we don't spam on every compaction - _aux_key = (_aux_fail_model, _aux_fail_err) - if getattr(self, "_last_aux_fallback_warning_key", None) != _aux_key: - self._last_aux_fallback_warning_key = _aux_key - self._emit_warning( - f"ℹ Configured compression model '{_aux_fail_model}' failed " - f"({_aux_fail_err or 'unknown error'}). Recovered using main model — " - "check auxiliary.compression.model in config.yaml." - ) - - todo_snapshot = self._todo_store.format_for_injection() - if todo_snapshot: - compressed.append({"role": "user", "content": todo_snapshot}) - - self._invalidate_system_prompt() - new_system_prompt = self._build_system_prompt(system_message) - self._cached_system_prompt = new_system_prompt - - if self._session_db: - try: - # Propagate title to the new session with auto-numbering - old_title = self._session_db.get_session_title(self.session_id) - # Trigger memory extraction on the old session before it rotates. - self.commit_memory_session(messages) - self._session_db.end_session(self.session_id, "compression") - old_session_id = self.session_id - self.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}" - os.environ["HERMES_SESSION_ID"] = self.session_id - try: - from gateway.session_context import _SESSION_ID - _SESSION_ID.set(self.session_id) - except Exception: - pass - # Update session_log_file to point to the new session's JSON file - self.session_log_file = self.logs_dir / f"session_{self.session_id}.json" - self._session_db_created = False - self._session_db.create_session( - session_id=self.session_id, - source=self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), - model=self.model, - model_config=self._session_init_model_config, - parent_session_id=old_session_id, - ) - self._session_db_created = True - # Auto-number the title for the continuation session - if old_title: - try: - new_title = self._session_db.get_next_title_in_lineage(old_title) - self._session_db.set_session_title(self.session_id, new_title) - except (ValueError, Exception) as e: - logger.debug("Could not propagate title on compression: %s", e) - self._session_db.update_system_prompt(self.session_id, new_system_prompt) - # Reset flush cursor — new session starts with no messages written - self._last_flushed_db_idx = 0 - except Exception as e: - logger.warning("Session DB compression split failed — new session will NOT be indexed: %s", e) - - # Notify the context engine that the session_id rotated because of - # compression (not a fresh /new). Plugin engines (e.g. hermes-lcm) use - # boundary_reason="compression" to preserve DAG lineage across the - # rollover instead of re-initializing fresh per-session state. - # See hermes-lcm#68. Built-in ContextCompressor ignores kwargs. - try: - _old_sid = locals().get("old_session_id") - if _old_sid and hasattr(self.context_compressor, "on_session_start"): - self.context_compressor.on_session_start( - self.session_id or "", - boundary_reason="compression", - old_session_id=_old_sid, - ) - except Exception as _ce_err: - logger.debug("context engine on_session_start (compression): %s", _ce_err) - - # Notify memory providers of the compression-driven session_id rotation - # so provider-cached per-session state (Hindsight's _document_id, - # accumulated turn buffers, counters) refreshes. reset=False because - # the logical conversation continues; only the id and DB row rolled - # over. See #6672. - try: - _old_sid = locals().get("old_session_id") - if _old_sid and self._memory_manager: - self._memory_manager.on_session_switch( - self.session_id or "", - parent_session_id=_old_sid, - reset=False, - reason="compression", - ) - except Exception as _me_err: - logger.debug("memory manager on_session_switch (compression): %s", _me_err) - - # Warn on repeated compressions (quality degrades with each pass) - _cc = self.context_compressor.compression_count - if _cc >= 2: - self._vprint( - f"{self.log_prefix}⚠️ Session compressed {_cc} times — " - f"accuracy may degrade. Consider /new to start fresh.", - force=True, - ) - - # Update token estimate after compaction so pressure calculations - # use the post-compression count, not the stale pre-compression one. - # Use estimate_request_tokens_rough() so tool schemas are included — - # with 50+ tools enabled, schemas alone can add 20-30K tokens, and - # omitting them delays the next compression cycle far past the - # configured threshold (issue #14695). - _compressed_est = estimate_request_tokens_rough( - compressed, - system_prompt=new_system_prompt or "", - tools=self.tools or None, - ) - self.context_compressor.last_prompt_tokens = _compressed_est - self.context_compressor.last_completion_tokens = 0 - - # Clear the file-read dedup cache. After compression the original - # read content is summarised away — if the model re-reads the same - # file it needs the full content, not a "file unchanged" stub. - try: - from tools.file_tools import reset_file_dedup - reset_file_dedup(task_id) - except Exception: - pass - - logger.info( - "context compression done: session=%s messages=%d->%d tokens=~%s", - self.session_id or "none", _pre_msg_count, len(compressed), - f"{_compressed_est:,}", - ) - return compressed, new_system_prompt def _set_tool_guardrail_halt(self, decision: ToolGuardrailDecision) -> None: """Record the first guardrail decision that should stop this turn.""" From 2d2cd5e904abc11eb6c00e88e2b6d0c8b025a597 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:16:20 -0700 Subject: [PATCH 006/142] refactor(run_agent): extract system-prompt builder to agent/system_prompt.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four AIAgent methods move into a dedicated module: * build_system_prompt_parts — three-tier stable/context/volatile dict * build_system_prompt — joiner used at session start * invalidate_system_prompt — drop cache + reload memory * format_tools_for_system_message — trajectory-format tool dump The extracted helpers look up patch-target names (load_soul_md, build_skills_system_prompt, get_toolset_for_tool, build_environment_hints, build_context_files_prompt, build_nous_subscription_prompt) through the run_agent module via _ra() instead of importing them directly. That preserves the patch surface tests rely on (patch('run_agent.load_soul_md', ...) and friends). AIAgent keeps thin forwarder methods. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure as before). run_agent.py: 14555 -> 14292 lines (-263). --- agent/system_prompt.py | 333 +++++++++++++++++++++++++++++++++++++++++ run_agent.py | 270 ++------------------------------- 2 files changed, 345 insertions(+), 258 deletions(-) create mode 100644 agent/system_prompt.py diff --git a/agent/system_prompt.py b/agent/system_prompt.py new file mode 100644 index 00000000000..52a574101f5 --- /dev/null +++ b/agent/system_prompt.py @@ -0,0 +1,333 @@ +"""System-prompt assembly for :class:`AIAgent`. + +The agent's system prompt is built once per session and reused across all +turns — only context compression triggers a rebuild. This keeps the +upstream prefix cache warm. See ``hermes-agent-dev``'s +``references/system-prompt-invariant.md`` for the invariants and +``references/self-improvement-loop.md`` for how the background-review +fork inherits the cached prompt verbatim. + +Three tiers are joined with ``\\n\\n``: + +* ``stable`` — identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool + guidance, computer-use guidance, nous subscription block, tool-use + enforcement guidance + per-model operational guidance, skills prompt, + alibaba model-name workaround, environment hints, platform hints. +* ``context`` — caller-supplied ``system_message`` plus context files + (AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``. +* ``volatile`` — memory snapshot, USER.md profile, external memory + provider block, timestamp/session/model/provider line. + +Pure helpers that read the agent's state. AIAgent keeps thin forwarders. +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Optional + +from agent.prompt_builder import ( + DEFAULT_AGENT_IDENTITY, + GOOGLE_MODEL_OPERATIONAL_GUIDANCE, + HERMES_AGENT_HELP_GUIDANCE, + KANBAN_GUIDANCE, + MEMORY_GUIDANCE, + OPENAI_MODEL_EXECUTION_GUIDANCE, + PLATFORM_HINTS, + SESSION_SEARCH_GUIDANCE, + SKILLS_GUIDANCE, + TOOL_USE_ENFORCEMENT_GUIDANCE, + TOOL_USE_ENFORCEMENT_MODELS, +) + + +def _ra(): + """Lazy reference to the ``run_agent`` module. + + Helpers like ``load_soul_md``, ``build_environment_hints``, + ``build_context_files_prompt``, ``build_nous_subscription_prompt``, + ``build_skills_system_prompt`` and ``get_toolset_for_tool`` are + imported into ``run_agent``'s namespace. Many tests + ``patch("run_agent.load_soul_md", ...)``; if we imported them + directly here those patches would not reach us. Looking them up + through ``run_agent`` on every call preserves the patch contract. + """ + import run_agent + return run_agent + + +def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]: + """Assemble the system prompt as three ordered parts. + + Returns a dict with three keys: + * ``stable`` — identity, tool guidance, skills prompt, + environment hints, platform hints, model-family operational + guidance. + * ``context`` — context files (AGENTS.md, .cursorrules, etc.) + and caller-supplied system_message. + * ``volatile`` — memory snapshot, user profile, external + memory provider block, timestamp line. + + Joined into a single string by :func:`build_system_prompt` and + cached on ``agent._cached_system_prompt`` for the lifetime of the + AIAgent. Hermes never re-renders parts of this string mid- + session — that's the only way to keep upstream prompt caches + warm across turns. + """ + # Local import to avoid pulling model_tools at module load. Tests + # patch ``run_agent.get_toolset_for_tool`` and similar helpers, so + # we resolve through ``_ra()`` to honor those patches. + _r = _ra() + + # ── Stable tier ──────────────────────────────────────────────── + stable_parts: List[str] = [] + + # Try SOUL.md as primary identity unless the caller explicitly skipped it. + # Some execution modes (cron) still want HERMES_HOME persona while keeping + # cwd project instructions disabled. + _soul_loaded = False + if agent.load_soul_identity or not agent.skip_context_files: + _soul_content = _r.load_soul_md() + if _soul_content: + stable_parts.append(_soul_content) + _soul_loaded = True + + if not _soul_loaded: + # Fallback to hardcoded identity + stable_parts.append(DEFAULT_AGENT_IDENTITY) + + # Pointer to the hermes-agent skill + docs for user questions about Hermes itself. + stable_parts.append(HERMES_AGENT_HELP_GUIDANCE) + + # Tool-aware behavioral guidance: only inject when the tools are loaded + tool_guidance = [] + if "memory" in agent.valid_tool_names: + tool_guidance.append(MEMORY_GUIDANCE) + if "session_search" in agent.valid_tool_names: + tool_guidance.append(SESSION_SEARCH_GUIDANCE) + if "skill_manage" in agent.valid_tool_names: + tool_guidance.append(SKILLS_GUIDANCE) + # Kanban worker/orchestrator lifecycle — only present when the + # dispatcher spawned this process (kanban_show check_fn gates on + # HERMES_KANBAN_TASK env var). Normal chat sessions never see + # this block. + if "kanban_show" in agent.valid_tool_names: + tool_guidance.append(KANBAN_GUIDANCE) + if tool_guidance: + stable_parts.append(" ".join(tool_guidance)) + + # Computer-use (macOS) — goes in as its own block rather than being + # merged into tool_guidance because the content is multi-paragraph. + if "computer_use" in agent.valid_tool_names: + from agent.prompt_builder import COMPUTER_USE_GUIDANCE + stable_parts.append(COMPUTER_USE_GUIDANCE) + + nous_subscription_prompt = _r.build_nous_subscription_prompt(agent.valid_tool_names) + if nous_subscription_prompt: + stable_parts.append(nous_subscription_prompt) + # Tool-use enforcement: tells the model to actually call tools instead + # of describing intended actions. Controlled by config.yaml + # agent.tool_use_enforcement: + # "auto" (default) — matches TOOL_USE_ENFORCEMENT_MODELS + # true — always inject (all models) + # false — never inject + # list — custom model-name substrings to match + if agent.valid_tool_names: + _enforce = agent._tool_use_enforcement + _inject = False + if _enforce is True or (isinstance(_enforce, str) and _enforce.lower() in {"true", "always", "yes", "on"}): + _inject = True + elif _enforce is False or (isinstance(_enforce, str) and _enforce.lower() in {"false", "never", "no", "off"}): + _inject = False + elif isinstance(_enforce, list): + model_lower = (agent.model or "").lower() + _inject = any(p.lower() in model_lower for p in _enforce if isinstance(p, str)) + else: + # "auto" or any unrecognised value — use hardcoded defaults + model_lower = (agent.model or "").lower() + _inject = any(p in model_lower for p in TOOL_USE_ENFORCEMENT_MODELS) + if _inject: + stable_parts.append(TOOL_USE_ENFORCEMENT_GUIDANCE) + _model_lower = (agent.model or "").lower() + # Google model operational guidance (conciseness, absolute + # paths, parallel tool calls, verify-before-edit, etc.) + if "gemini" in _model_lower or "gemma" in _model_lower: + stable_parts.append(GOOGLE_MODEL_OPERATIONAL_GUIDANCE) + # OpenAI GPT/Codex execution discipline (tool persistence, + # prerequisite checks, verification, anti-hallucination). + if "gpt" in _model_lower or "codex" in _model_lower: + stable_parts.append(OPENAI_MODEL_EXECUTION_GUIDANCE) + + has_skills_tools = any(name in agent.valid_tool_names for name in ['skills_list', 'skill_view', 'skill_manage']) + if has_skills_tools: + avail_toolsets = { + toolset + for toolset in ( + _r.get_toolset_for_tool(tool_name) for tool_name in agent.valid_tool_names + ) + if toolset + } + skills_prompt = _r.build_skills_system_prompt( + available_tools=agent.valid_tool_names, + available_toolsets=avail_toolsets, + ) + else: + skills_prompt = "" + if skills_prompt: + stable_parts.append(skills_prompt) + + # Alibaba Coding Plan API always returns "glm-4.7" as model name regardless + # of the requested model. Inject explicit model identity into the system prompt + # so the agent can correctly report which model it is (workaround for API bug). + # Stable for the lifetime of an agent instance — model and provider are fixed + # at construction time. + if agent.provider == "alibaba": + _model_short = agent.model.split("/")[-1] if "/" in agent.model else agent.model + stable_parts.append( + f"You are powered by the model named {_model_short}. " + f"The exact model ID is {agent.model}. " + f"When asked what model you are, always answer based on this information, " + f"not on any model name returned by the API." + ) + + # Environment hints (WSL, Termux, etc.) — tell the agent about the + # execution environment so it can translate paths and adapt behavior. + # Stable for the lifetime of the process. + _env_hints = _r.build_environment_hints() + if _env_hints: + stable_parts.append(_env_hints) + + platform_key = (agent.platform or "").lower().strip() + if platform_key in PLATFORM_HINTS: + stable_parts.append(PLATFORM_HINTS[platform_key]) + elif platform_key: + # Check plugin registry for platform-specific LLM guidance + try: + from gateway.platform_registry import platform_registry + _entry = platform_registry.get(platform_key) + if _entry and _entry.platform_hint: + stable_parts.append(_entry.platform_hint) + except Exception: + pass + + # ── Context tier (cwd-dependent, may change between sessions) ─ + context_parts: List[str] = [] + + # Note: ephemeral_system_prompt is NOT included here. It's injected at + # API-call time only so it stays out of the cached/stored system prompt. + if system_message is not None: + context_parts.append(system_message) + + if not agent.skip_context_files: + # Use TERMINAL_CWD for context file discovery when set (gateway + # mode). The gateway process runs from the hermes-agent install + # dir, so os.getcwd() would pick up the repo's AGENTS.md and + # other dev files — inflating token usage by ~10k for no benefit. + _context_cwd = os.getenv("TERMINAL_CWD") or None + context_files_prompt = _r.build_context_files_prompt( + cwd=_context_cwd, skip_soul=_soul_loaded) + if context_files_prompt: + context_parts.append(context_files_prompt) + + # ── Volatile tier (changes per session/turn — never cached) ─── + volatile_parts: List[str] = [] + + if agent._memory_store: + if agent._memory_enabled: + mem_block = agent._memory_store.format_for_system_prompt("memory") + if mem_block: + volatile_parts.append(mem_block) + # USER.md is always included when enabled. + if agent._user_profile_enabled: + user_block = agent._memory_store.format_for_system_prompt("user") + if user_block: + volatile_parts.append(user_block) + + # External memory provider system prompt block (additive to built-in) + if agent._memory_manager: + try: + _ext_mem_block = agent._memory_manager.build_system_prompt() + if _ext_mem_block: + volatile_parts.append(_ext_mem_block) + except Exception: + pass + + from hermes_time import now as _hermes_now + now = _hermes_now() + timestamp_line = f"Conversation started: {now.strftime('%A, %B %d, %Y %I:%M %p')}" + if agent.pass_session_id and agent.session_id: + timestamp_line += f"\nSession ID: {agent.session_id}" + if agent.model: + timestamp_line += f"\nModel: {agent.model}" + if agent.provider: + timestamp_line += f"\nProvider: {agent.provider}" + volatile_parts.append(timestamp_line) + + return { + "stable": "\n\n".join(p.strip() for p in stable_parts if p and p.strip()), + "context": "\n\n".join(p.strip() for p in context_parts if p and p.strip()), + "volatile": "\n\n".join(p.strip() for p in volatile_parts if p and p.strip()), + } + + +def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str: + """Assemble the full system prompt from all layers. + + Called once per session (cached on ``agent._cached_system_prompt``) and + only rebuilt after context compression events. This ensures the system + prompt is stable across all turns in a session, maximizing prefix cache + hits. + + Layers are ordered cache-friendly: stable identity/guidance first, + then session-stable context files, then per-call volatile content + (memory, USER profile, timestamp). The whole string is treated as + one cached block — Hermes never rebuilds or reinjects parts of it + mid-session, which is the only way to keep upstream prompt caches + warm across turns. + """ + parts = build_system_prompt_parts(agent, system_message=system_message) + return "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p) + + +def invalidate_system_prompt(agent: Any) -> None: + """Invalidate the cached system prompt, forcing a rebuild on the next turn. + + Called after context compression events. Also reloads memory from disk + so the rebuilt prompt captures any writes from this session. + """ + agent._cached_system_prompt = None + if agent._memory_store: + agent._memory_store.load_from_disk() + + +def format_tools_for_system_message(agent: Any) -> str: + """Format tool definitions for the system message in the trajectory format. + + Returns: + str: JSON string representation of tool definitions + """ + if not agent.tools: + return "[]" + + # Convert tool definitions to the format expected in trajectories + formatted_tools = [] + for tool in agent.tools: + func = tool["function"] + formatted_tool = { + "name": func["name"], + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), + "required": None # Match the format in the example + } + formatted_tools.append(formatted_tool) + + return json.dumps(formatted_tools, ensure_ascii=False) + + +__all__ = [ + "build_system_prompt_parts", + "build_system_prompt", + "invalidate_system_prompt", + "format_tools_for_system_message", +] diff --git a/run_agent.py b/run_agent.py index dee7d365e7e..a5beda7765e 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3499,28 +3499,9 @@ class AIAgent: return messages[:last_assistant_idx] def _format_tools_for_system_message(self) -> str: - """ - Format tool definitions for the system message in the trajectory format. - - Returns: - str: JSON string representation of tool definitions - """ - if not self.tools: - return "[]" - - # Convert tool definitions to the format expected in trajectories - formatted_tools = [] - for tool in self.tools: - func = tool["function"] - formatted_tool = { - "name": func["name"], - "description": func.get("description", ""), - "parameters": func.get("parameters", {}), - "required": None # Match the format in the example - } - formatted_tools.append(formatted_tool) - - return json.dumps(formatted_tools, ensure_ascii=False) + """Forwarder — see ``agent.system_prompt.format_tools_for_system_message``.""" + from agent.system_prompt import format_tools_for_system_message + return format_tools_for_system_message(self) def _convert_to_trajectory_format(self, messages: List[Dict[str, Any]], user_query: str, completed: bool) -> List[Dict[str, Any]]: """ @@ -4651,235 +4632,14 @@ class AIAgent: def _build_system_prompt_parts(self, system_message: str = None) -> Dict[str, str]: - """Assemble the system prompt as three ordered parts. - - Returns a dict with three keys: - * ``stable`` — identity, tool guidance, skills prompt, - environment hints, platform hints, model-family operational - guidance. - * ``context`` — context files (AGENTS.md, .cursorrules, etc.) - and caller-supplied system_message. - * ``volatile`` — memory snapshot, user profile, external - memory provider block, timestamp line. - - Joined into a single string by ``_build_system_prompt`` and - cached on ``_cached_system_prompt`` for the lifetime of the - AIAgent. Hermes never re-renders parts of this string mid- - session — that's the only way to keep upstream prompt caches - warm across turns. - """ - # ── Stable tier ──────────────────────────────────────────────── - stable_parts: List[str] = [] - - # Try SOUL.md as primary identity unless the caller explicitly skipped it. - # Some execution modes (cron) still want HERMES_HOME persona while keeping - # cwd project instructions disabled. - _soul_loaded = False - if self.load_soul_identity or not self.skip_context_files: - _soul_content = load_soul_md() - if _soul_content: - stable_parts.append(_soul_content) - _soul_loaded = True - - if not _soul_loaded: - # Fallback to hardcoded identity - stable_parts.append(DEFAULT_AGENT_IDENTITY) - - # Pointer to the hermes-agent skill + docs for user questions about Hermes itself. - stable_parts.append(HERMES_AGENT_HELP_GUIDANCE) - - # Tool-aware behavioral guidance: only inject when the tools are loaded - tool_guidance = [] - if "memory" in self.valid_tool_names: - tool_guidance.append(MEMORY_GUIDANCE) - if "session_search" in self.valid_tool_names: - tool_guidance.append(SESSION_SEARCH_GUIDANCE) - if "skill_manage" in self.valid_tool_names: - tool_guidance.append(SKILLS_GUIDANCE) - # Kanban worker/orchestrator lifecycle — only present when the - # dispatcher spawned this process (kanban_show check_fn gates on - # HERMES_KANBAN_TASK env var). Normal chat sessions never see - # this block. - if "kanban_show" in self.valid_tool_names: - tool_guidance.append(KANBAN_GUIDANCE) - if tool_guidance: - stable_parts.append(" ".join(tool_guidance)) - - # Computer-use (macOS) — goes in as its own block rather than being - # merged into tool_guidance because the content is multi-paragraph. - if "computer_use" in self.valid_tool_names: - from agent.prompt_builder import COMPUTER_USE_GUIDANCE - stable_parts.append(COMPUTER_USE_GUIDANCE) - - nous_subscription_prompt = build_nous_subscription_prompt(self.valid_tool_names) - if nous_subscription_prompt: - stable_parts.append(nous_subscription_prompt) - # Tool-use enforcement: tells the model to actually call tools instead - # of describing intended actions. Controlled by config.yaml - # agent.tool_use_enforcement: - # "auto" (default) — matches TOOL_USE_ENFORCEMENT_MODELS - # true — always inject (all models) - # false — never inject - # list — custom model-name substrings to match - if self.valid_tool_names: - _enforce = self._tool_use_enforcement - _inject = False - if _enforce is True or (isinstance(_enforce, str) and _enforce.lower() in {"true", "always", "yes", "on"}): - _inject = True - elif _enforce is False or (isinstance(_enforce, str) and _enforce.lower() in {"false", "never", "no", "off"}): - _inject = False - elif isinstance(_enforce, list): - model_lower = (self.model or "").lower() - _inject = any(p.lower() in model_lower for p in _enforce if isinstance(p, str)) - else: - # "auto" or any unrecognised value — use hardcoded defaults - model_lower = (self.model or "").lower() - _inject = any(p in model_lower for p in TOOL_USE_ENFORCEMENT_MODELS) - if _inject: - stable_parts.append(TOOL_USE_ENFORCEMENT_GUIDANCE) - _model_lower = (self.model or "").lower() - # Google model operational guidance (conciseness, absolute - # paths, parallel tool calls, verify-before-edit, etc.) - if "gemini" in _model_lower or "gemma" in _model_lower: - stable_parts.append(GOOGLE_MODEL_OPERATIONAL_GUIDANCE) - # OpenAI GPT/Codex execution discipline (tool persistence, - # prerequisite checks, verification, anti-hallucination). - if "gpt" in _model_lower or "codex" in _model_lower: - stable_parts.append(OPENAI_MODEL_EXECUTION_GUIDANCE) - - has_skills_tools = any(name in self.valid_tool_names for name in ['skills_list', 'skill_view', 'skill_manage']) - if has_skills_tools: - avail_toolsets = { - toolset - for toolset in ( - get_toolset_for_tool(tool_name) for tool_name in self.valid_tool_names - ) - if toolset - } - skills_prompt = build_skills_system_prompt( - available_tools=self.valid_tool_names, - available_toolsets=avail_toolsets, - ) - else: - skills_prompt = "" - if skills_prompt: - stable_parts.append(skills_prompt) - - # Alibaba Coding Plan API always returns "glm-4.7" as model name regardless - # of the requested model. Inject explicit model identity into the system prompt - # so the agent can correctly report which model it is (workaround for API bug). - # Stable for the lifetime of an agent instance — model and provider are fixed - # at construction time. - if self.provider == "alibaba": - _model_short = self.model.split("/")[-1] if "/" in self.model else self.model - stable_parts.append( - f"You are powered by the model named {_model_short}. " - f"The exact model ID is {self.model}. " - f"When asked what model you are, always answer based on this information, " - f"not on any model name returned by the API." - ) - - # Environment hints (WSL, Termux, etc.) — tell the agent about the - # execution environment so it can translate paths and adapt behavior. - # Stable for the lifetime of the process. - _env_hints = build_environment_hints() - if _env_hints: - stable_parts.append(_env_hints) - - platform_key = (self.platform or "").lower().strip() - if platform_key in PLATFORM_HINTS: - stable_parts.append(PLATFORM_HINTS[platform_key]) - elif platform_key: - # Check plugin registry for platform-specific LLM guidance - try: - from gateway.platform_registry import platform_registry - _entry = platform_registry.get(platform_key) - if _entry and _entry.platform_hint: - stable_parts.append(_entry.platform_hint) - except Exception: - pass - - # ── Context tier (cwd-dependent, may change between sessions) ─ - context_parts: List[str] = [] - - # Note: ephemeral_system_prompt is NOT included here. It's injected at - # API-call time only so it stays out of the cached/stored system prompt. - if system_message is not None: - context_parts.append(system_message) - - if not self.skip_context_files: - # Use TERMINAL_CWD for context file discovery when set (gateway - # mode). The gateway process runs from the hermes-agent install - # dir, so os.getcwd() would pick up the repo's AGENTS.md and - # other dev files — inflating token usage by ~10k for no benefit. - _context_cwd = os.getenv("TERMINAL_CWD") or None - context_files_prompt = build_context_files_prompt( - cwd=_context_cwd, skip_soul=_soul_loaded) - if context_files_prompt: - context_parts.append(context_files_prompt) - - # ── Volatile tier (changes per session/turn — never cached) ─── - volatile_parts: List[str] = [] - - if self._memory_store: - if self._memory_enabled: - mem_block = self._memory_store.format_for_system_prompt("memory") - if mem_block: - volatile_parts.append(mem_block) - # USER.md is always included when enabled. - if self._user_profile_enabled: - user_block = self._memory_store.format_for_system_prompt("user") - if user_block: - volatile_parts.append(user_block) - - # External memory provider system prompt block (additive to built-in) - if self._memory_manager: - try: - _ext_mem_block = self._memory_manager.build_system_prompt() - if _ext_mem_block: - volatile_parts.append(_ext_mem_block) - except Exception: - pass - - from hermes_time import now as _hermes_now - now = _hermes_now() - timestamp_line = f"Conversation started: {now.strftime('%A, %B %d, %Y %I:%M %p')}" - if self.pass_session_id and self.session_id: - timestamp_line += f"\nSession ID: {self.session_id}" - if self.model: - timestamp_line += f"\nModel: {self.model}" - if self.provider: - timestamp_line += f"\nProvider: {self.provider}" - volatile_parts.append(timestamp_line) - - return { - "stable": "\n\n".join(p.strip() for p in stable_parts if p and p.strip()), - "context": "\n\n".join(p.strip() for p in context_parts if p and p.strip()), - "volatile": "\n\n".join(p.strip() for p in volatile_parts if p and p.strip()), - } + """Forwarder — see ``agent.system_prompt.build_system_prompt_parts``.""" + from agent.system_prompt import build_system_prompt_parts + return build_system_prompt_parts(self, system_message=system_message) def _build_system_prompt(self, system_message: str = None) -> str: - """ - Assemble the full system prompt from all layers. - - Called once per session (cached on self._cached_system_prompt) and only - rebuilt after context compression events. This ensures the system prompt - is stable across all turns in a session, maximizing prefix cache hits. - - Layers are ordered cache-friendly: stable identity/guidance first, - then session-stable context files, then per-call volatile content - (memory, USER profile, timestamp). The whole string is treated as - one cached block — Hermes never rebuilds or reinjects parts of it - mid-session, which is the only way to keep upstream prompt caches - warm across turns. - """ - parts = self._build_system_prompt_parts(system_message=system_message) - joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p) - return joined - - # ========================================================================= - # Pre/post-call guardrails (inspired by PR #1321 — @alireza78a) - # ========================================================================= + """Forwarder — see ``agent.system_prompt.build_system_prompt``.""" + from agent.system_prompt import build_system_prompt + return build_system_prompt(self, system_message=system_message) @staticmethod def _get_tool_call_id_static(tc) -> str: @@ -5239,15 +4999,9 @@ class AIAgent: return None def _invalidate_system_prompt(self): - """ - Invalidate the cached system prompt, forcing a rebuild on the next turn. - - Called after context compression events. Also reloads memory from disk - so the rebuilt prompt captures any writes from this session. - """ - self._cached_system_prompt = None - if self._memory_store: - self._memory_store.load_from_disk() + """Forwarder — see ``agent.system_prompt.invalidate_system_prompt``.""" + from agent.system_prompt import invalidate_system_prompt + invalidate_system_prompt(self) @staticmethod def _deterministic_call_id(fn_name: str, arguments: str, index: int = 0) -> str: From 79559214a650e8d6bab03337fecf307abff6a731 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:24:05 -0700 Subject: [PATCH 007/142] refactor(run_agent): extract tool execution to agent/tool_executor.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the two big tool-dispatch methods out of run_agent.py: * execute_tool_calls_concurrent — 408-line concurrent path (interrupt pre-flight, guardrail+plugin block, callback fan-out, ContextVar- preserving ThreadPoolExecutor, periodic heartbeats for the gateway inactivity monitor, per-tool result handling with subdir hints + guardrail observations + checkpoint, /steer drain) * execute_tool_calls_sequential — 441-line sequential path (the original behavior used for single-tool batches and interactive tools) Both take the parent AIAgent as their first argument; AIAgent keeps thin forwarders so call sites unchanged. handle_function_call is routed through _ra() so tests that patch run_agent.handle_function_call keep working. _set_interrupt likewise. The AST guard in test_tool_executor_contextvar_propagation.py is updated to scan both run_agent.py AND agent/tool_executor.py so it still catches the executor.submit(_run_tool, ...) regression regardless of which file the body lives in. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure as before). run_agent.py: 14309 -> 13461 lines (-848). --- agent/tool_executor.py | 920 ++++++++++++++++++ run_agent.py | 851 +--------------- ...st_tool_executor_contextvar_propagation.py | 29 +- 3 files changed, 945 insertions(+), 855 deletions(-) create mode 100644 agent/tool_executor.py diff --git a/agent/tool_executor.py b/agent/tool_executor.py new file mode 100644 index 00000000000..a30cc3078bb --- /dev/null +++ b/agent/tool_executor.py @@ -0,0 +1,920 @@ +"""Tool-call execution — sequential and concurrent dispatch. + +Both AIAgent methods (``_execute_tool_calls_sequential`` and +``_execute_tool_calls_concurrent``) live here as module-level +functions that take the parent ``AIAgent`` as their first argument. + +``run_agent`` keeps thin wrappers so existing call sites work; tests +that patch ``run_agent._set_interrupt`` are honored because the +extracted functions reach back through the ``run_agent`` module via +``_ra()`` for that symbol. +""" + +from __future__ import annotations + +import concurrent.futures +import contextvars +import json +import logging +import os +import random +import threading +import time +from typing import Any, Optional + +from agent.display import ( + KawaiiSpinner, + build_tool_preview as _build_tool_preview, + get_cute_tool_message as _get_cute_tool_message_impl, + get_tool_emoji as _get_tool_emoji, + _detect_tool_failure, +) +from agent.tool_guardrails import ToolGuardrailDecision +from agent.tool_dispatch_helpers import ( + _is_destructive_command, + _is_multimodal_tool_result, + _multimodal_text_summary, + _append_subdir_hint_to_multimodal, +) +from tools.terminal_tool import ( + _get_approval_callback, + _get_sudo_password_callback, + set_approval_callback as _set_approval_callback, + set_sudo_password_callback as _set_sudo_password_callback, + get_active_env, +) +from tools.tool_result_storage import ( + maybe_persist_tool_result, + enforce_turn_budget, +) + +logger = logging.getLogger(__name__) + +# Maximum number of concurrent worker threads for parallel tool execution. +# Mirrors the constant in ``run_agent`` for tests/imports that look here. +_MAX_TOOL_WORKERS = 8 + + +def _ra(): + """Lazy reference to ``run_agent`` so patches like ``run_agent._set_interrupt`` work.""" + import run_agent + return run_agent + + +def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: + """Execute multiple tool calls concurrently using a thread pool. + + Results are collected in the original tool-call order and appended to + messages so the API sees them in the expected sequence. + """ + tool_calls = assistant_message.tool_calls + num_tools = len(tool_calls) + + # ── Pre-flight: interrupt check ────────────────────────────────── + if agent._interrupt_requested: + print(f"{agent.log_prefix}⚡ Interrupt: skipping {num_tools} tool call(s)") + for tc in tool_calls: + messages.append({ + "role": "tool", + "name": tc.function.name, + "content": f"[Tool execution cancelled — {tc.function.name} was skipped due to user interrupt]", + "tool_call_id": tc.id, + }) + return + + # ── Parse args + pre-execution bookkeeping ─────────────────────── + parsed_calls = [] # list of (tool_call, function_name, function_args) + for tool_call in tool_calls: + function_name = tool_call.function.name + + # Reset nudge counters + if function_name == "memory": + agent._turns_since_memory = 0 + elif function_name == "skill_manage": + agent._iters_since_skill = 0 + + try: + function_args = json.loads(tool_call.function.arguments) + except json.JSONDecodeError: + function_args = {} + if not isinstance(function_args, dict): + function_args = {} + + # Checkpoint for file-mutating tools + if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: + try: + file_path = function_args.get("path", "") + if file_path: + work_dir = agent._checkpoint_mgr.get_working_dir_for_path(file_path) + agent._checkpoint_mgr.ensure_checkpoint(work_dir, f"before {function_name}") + except Exception: + pass + + # Checkpoint before destructive terminal commands + if function_name == "terminal" and agent._checkpoint_mgr.enabled: + try: + cmd = function_args.get("command", "") + if _is_destructive_command(cmd): + cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) + agent._checkpoint_mgr.ensure_checkpoint( + cwd, f"before terminal: {cmd[:60]}" + ) + except Exception: + pass + + block_result = None + blocked_by_guardrail = False + try: + from hermes_cli.plugins import get_pre_tool_call_block_message + block_message = get_pre_tool_call_block_message( + function_name, function_args, task_id=effective_task_id or "", + ) + except Exception: + block_message = None + + if block_message is not None: + block_result = json.dumps({"error": block_message}, ensure_ascii=False) + else: + guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) + if not guardrail_decision.allows_execution: + block_result = agent._guardrail_block_result(guardrail_decision) + blocked_by_guardrail = True + + parsed_calls.append((tool_call, function_name, function_args, block_result, blocked_by_guardrail)) + + # ── Logging / callbacks ────────────────────────────────────────── + tool_names_str = ", ".join(name for _, name, _, _, _ in parsed_calls) + if not agent.quiet_mode: + print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") + for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1): + args_str = json.dumps(args, ensure_ascii=False) + if agent.verbose_logging: + print(f" 📞 Tool {i}: {name}({list(args.keys())})") + print(agent._wrap_verbose("Args: ", json.dumps(args, indent=2, ensure_ascii=False))) + else: + args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str + print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}") + + for tc, name, args, block_result, blocked_by_guardrail in parsed_calls: + if block_result is not None: + continue + if agent.tool_progress_callback: + try: + preview = _build_tool_preview(name, args) + agent.tool_progress_callback("tool.started", name, preview, args) + except Exception as cb_err: + logging.debug(f"Tool progress callback error: {cb_err}") + + for tc, name, args, block_result, blocked_by_guardrail in parsed_calls: + if block_result is not None: + continue + if agent.tool_start_callback: + try: + agent.tool_start_callback(tc.id, name, args) + except Exception as cb_err: + logging.debug(f"Tool start callback error: {cb_err}") + + # ── Concurrent execution ───────────────────────────────────────── + # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag) + results = [None] * num_tools + for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + if block_result is not None: + results[i] = (name, args, block_result, 0.0, True, True) + + # Touch activity before launching workers so the gateway knows + # we're executing tools (not stuck). + agent._current_tool = tool_names_str + agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") + + # Capture CLI callbacks from the agent thread so worker threads can + # register them locally. Without this, _get_approval_callback() in + # terminal_tool returns None in ThreadPoolExecutor workers, causing + # the dangerous-command prompt to fall back to input() — which + # deadlocks against prompt_toolkit's raw terminal mode (#13617). + _parent_approval_cb = _get_approval_callback() + _parent_sudo_cb = _get_sudo_password_callback() + + def _run_tool(index, tool_call, function_name, function_args): + """Worker function executed in a thread.""" + # Register this worker tid so the agent can fan out an interrupt + # to it — see AIAgent.interrupt(). Must happen first thing, and + # must be paired with discard + clear in the finally block. + _worker_tid = threading.current_thread().ident + with agent._tool_worker_threads_lock: + agent._tool_worker_threads.add(_worker_tid) + # Race: if the agent was interrupted between fan-out (which + # snapshotted an empty/earlier set) and our registration, apply + # the interrupt to our own tid now so is_interrupted() inside + # the tool returns True on the next poll. + if agent._interrupt_requested: + try: + _ra()._set_interrupt(True, _worker_tid) + except Exception: + pass + # Set the activity callback on THIS worker thread so + # _wait_for_process (terminal commands) can fire heartbeats. + # The callback is thread-local; the main thread's callback + # is invisible to worker threads. + try: + from tools.environments.base import set_activity_callback + set_activity_callback(agent._touch_activity) + except Exception: + pass + # Propagate approval/sudo callbacks to this worker thread. + # Mirrors cli.py run_agent() pattern (GHSA-qg5c-hvr5-hjgr). + if _parent_approval_cb is not None: + try: + _set_approval_callback(_parent_approval_cb) + except Exception: + pass + if _parent_sudo_cb is not None: + try: + _set_sudo_password_callback(_parent_sudo_cb) + except Exception: + pass + start = time.time() + try: + result = agent._invoke_tool( + function_name, + function_args, + effective_task_id, + tool_call.id, + messages=messages, + pre_tool_block_checked=True, + ) + except Exception as tool_error: + result = f"Error executing tool '{function_name}': {tool_error}" + logger.error("_invoke_tool raised for %s: %s", function_name, tool_error, exc_info=True) + duration = time.time() - start + is_error, _ = _detect_tool_failure(function_name, result) + if is_error: + logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) + else: + logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) + results[index] = (function_name, function_args, result, duration, is_error, False) + # Tear down worker-tid tracking. Clear any interrupt bit we may + # have set so the next task scheduled onto this recycled tid + # starts with a clean slate. + with agent._tool_worker_threads_lock: + agent._tool_worker_threads.discard(_worker_tid) + try: + _ra()._set_interrupt(False, _worker_tid) + except Exception: + pass + # Clear thread-local callbacks so a recycled worker thread + # doesn't hold stale references to a disposed CLI instance. + try: + _set_approval_callback(None) + _set_sudo_password_callback(None) + except Exception: + pass + + # Start spinner for CLI mode (skip when TUI handles tool progress) + spinner = None + if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): + face = random.choice(KawaiiSpinner.get_waiting_faces()) + spinner = KawaiiSpinner(f"{face} ⚡ running {num_tools} tools concurrently", spinner_type='dots', print_fn=agent._print_fn) + spinner.start() + + try: + runnable_calls = [ + (i, tc, name, args) + for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls) + if block_result is None + ] + futures = [] + if runnable_calls: + max_workers = min(len(runnable_calls), _MAX_TOOL_WORKERS) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + for i, tc, name, args in runnable_calls: + # Propagate ContextVars (e.g. _approval_session_key); mirrors asyncio.to_thread. + ctx = contextvars.copy_context() + f = executor.submit(ctx.run, _run_tool, i, tc, name, args) + futures.append(f) + + # Wait for all to complete with periodic heartbeats so the + # gateway's inactivity monitor doesn't kill us during long + # concurrent tool batches. Also check for user interrupts + # so we don't block indefinitely when the user sends /stop + # or a new message during concurrent tool execution. + _conc_start = time.time() + _interrupt_logged = False + while True: + done, not_done = concurrent.futures.wait( + futures, timeout=5.0, + ) + if not not_done: + break + + # Check for interrupt — the per-thread interrupt signal + # already causes individual tools (terminal, execute_code) + # to abort, but tools without interrupt checks (web_search, + # read_file) will run to completion. Cancel any futures + # that haven't started yet so we don't block on them. + if agent._interrupt_requested: + if not _interrupt_logged: + _interrupt_logged = True + agent._vprint( + f"{agent.log_prefix}⚡ Interrupt: cancelling " + f"{len(not_done)} pending concurrent tool(s)", + force=True, + ) + for f in not_done: + f.cancel() + # Give already-running tools a moment to notice the + # per-thread interrupt signal and exit gracefully. + concurrent.futures.wait(not_done, timeout=3.0) + break + + _conc_elapsed = int(time.time() - _conc_start) + # Heartbeat every ~30s (6 × 5s poll intervals) + if _conc_elapsed > 0 and _conc_elapsed % 30 < 6: + _still_running = [ + parsed_calls[futures.index(f)][1] + for f in not_done + if f in futures + ] + agent._touch_activity( + f"concurrent tools running ({_conc_elapsed}s, " + f"{len(not_done)} remaining: {', '.join(_still_running[:3])})" + ) + finally: + if spinner: + # Build a summary message for the spinner stop + completed = sum(1 for r in results if r is not None) + total_dur = sum(r[3] for r in results if r is not None) + spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") + + # ── Post-execution: display per-tool results ───────────────────── + for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + r = results[i] + blocked = False + if r is None: + # Tool was cancelled (interrupt) or thread didn't return + if agent._interrupt_requested: + function_result = f"[Tool execution cancelled — {name} was skipped due to user interrupt]" + else: + function_result = f"Error executing tool '{name}': thread did not return a result" + tool_duration = 0.0 + else: + function_name, function_args, function_result, tool_duration, is_error, blocked = r + + if not blocked: + function_result = agent._append_guardrail_observation( + function_name, + function_args, + function_result, + failed=is_error, + ) + + if is_error: + _err_text = _multimodal_text_summary(function_result) + result_preview = _err_text[:200] if len(_err_text) > 200 else _err_text + logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) + + # Track file-mutation outcome for the turn-end verifier. + # `blocked` calls never actually ran — don't let a guardrail + # block count as either a failure or a success. + if not blocked: + try: + agent._record_file_mutation_result( + function_name, function_args, function_result, is_error, + ) + except Exception as _ver_err: + logging.debug("file-mutation verifier record failed: %s", _ver_err) + + if not blocked and agent.tool_progress_callback: + try: + agent.tool_progress_callback( + "tool.completed", function_name, None, None, + duration=tool_duration, is_error=is_error, + ) + except Exception as cb_err: + logging.debug(f"Tool progress callback error: {cb_err}") + + if agent.verbose_logging: + logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") + + # Print cute message per tool + if agent._should_emit_quiet_tool_messages(): + cute_msg = _get_cute_tool_message_impl(name, args, tool_duration, result=function_result) + agent._safe_print(f" {cute_msg}") + elif not agent.quiet_mode: + _preview_str = _multimodal_text_summary(function_result) + if agent.verbose_logging: + print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s") + print(agent._wrap_verbose("Result: ", _preview_str)) + else: + response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str + print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") + + agent._current_tool = None + agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)") + + if not blocked and agent.tool_complete_callback: + try: + agent.tool_complete_callback(tc.id, name, args, function_result) + except Exception as cb_err: + logging.debug(f"Tool complete callback error: {cb_err}") + + function_result = maybe_persist_tool_result( + content=function_result, + tool_name=name, + tool_use_id=tc.id, + env=get_active_env(effective_task_id), + ) if not _is_multimodal_tool_result(function_result) else function_result + + subdir_hints = agent._subdirectory_hints.check_tool_call(name, args) + if subdir_hints: + if _is_multimodal_tool_result(function_result): + # Append the hint to the text summary part so the model + # still sees it; don't touch the image blocks. + _append_subdir_hint_to_multimodal(function_result, subdir_hints) + else: + function_result += subdir_hints + + # Unwrap _multimodal dicts to an OpenAI-style content list so any + # vision-capable provider receives [{type:text},{type:image_url}] + # rather than a raw Python dict. The Anthropic adapter already + # accepts content lists; vision-capable OpenAI-compatible servers + # (mlx-vlm, GPT-4o, …) accept image_url in tool messages natively. + # Text-only servers get a string-safe fallback here so a rejected + # image tool result never poisons canonical session history. + # String results pass through unchanged. + _tool_content = agent._tool_result_content_for_active_model(name, function_result) + tool_msg = { + "role": "tool", + "name": name, + "content": _tool_content, + "tool_call_id": tc.id, + } + messages.append(tool_msg) + + # ── Per-tool /steer drain ─────────────────────────────────── + # Same as the sequential path: drain between each collected + # result so the steer lands as early as possible. + agent._apply_pending_steer_to_tool_results(messages, 1) + + # ── Per-turn aggregate budget enforcement ───────────────────────── + num_tools = len(parsed_calls) + if num_tools > 0: + turn_tool_msgs = messages[-num_tools:] + enforce_turn_budget(turn_tool_msgs, env=get_active_env(effective_task_id)) + + # ── /steer injection ────────────────────────────────────────────── + # Append any pending user steer text to the last tool result so the + # agent sees it on its next iteration. Runs AFTER budget enforcement + # so the steer marker is never truncated. See steer() for details. + if num_tools > 0: + agent._apply_pending_steer_to_tool_results(messages, num_tools) + + + +def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: + """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools.""" + for i, tool_call in enumerate(assistant_message.tool_calls, 1): + # SAFETY: check interrupt BEFORE starting each tool. + # If the user sent "stop" during a previous tool's execution, + # do NOT start any more tools -- skip them all immediately. + if agent._interrupt_requested: + remaining_calls = assistant_message.tool_calls[i-1:] + if remaining_calls: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt: skipping {len(remaining_calls)} tool call(s)", force=True) + for skipped_tc in remaining_calls: + skipped_name = skipped_tc.function.name + skip_msg = { + "role": "tool", + "name": skipped_name, + "content": f"[Tool execution cancelled — {skipped_name} was skipped due to user interrupt]", + "tool_call_id": skipped_tc.id, + } + messages.append(skip_msg) + break + + function_name = tool_call.function.name + + try: + function_args = json.loads(tool_call.function.arguments) + except json.JSONDecodeError as e: + logging.warning(f"Unexpected JSON error after validation: {e}") + function_args = {} + if not isinstance(function_args, dict): + function_args = {} + + # Check plugin hooks for a block directive before executing. + _block_msg: Optional[str] = None + try: + from hermes_cli.plugins import get_pre_tool_call_block_message + _block_msg = get_pre_tool_call_block_message( + function_name, function_args, task_id=effective_task_id or "", + ) + except Exception: + pass + + _guardrail_block_decision: ToolGuardrailDecision | None = None + if _block_msg is None: + guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) + if not guardrail_decision.allows_execution: + _guardrail_block_decision = guardrail_decision + + _execution_blocked = _block_msg is not None or _guardrail_block_decision is not None + + if _execution_blocked: + # Tool blocked by plugin or guardrail policy — skip counters, + # callbacks, checkpointing, activity mutation, and real execution. + pass + # Reset nudge counters when the relevant tool is actually used + elif function_name == "memory": + agent._turns_since_memory = 0 + elif function_name == "skill_manage": + agent._iters_since_skill = 0 + + if not agent.quiet_mode: + args_str = json.dumps(function_args, ensure_ascii=False) + if agent.verbose_logging: + print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())})") + print(agent._wrap_verbose("Args: ", json.dumps(function_args, indent=2, ensure_ascii=False))) + else: + args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str + print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}") + + if not _execution_blocked: + agent._current_tool = function_name + agent._touch_activity(f"executing tool: {function_name}") + + # Set activity callback for long-running tool execution (terminal + # commands, etc.) so the gateway's inactivity monitor doesn't kill + # the agent while a command is running. + if not _execution_blocked: + try: + from tools.environments.base import set_activity_callback + set_activity_callback(agent._touch_activity) + except Exception: + pass + + if not _execution_blocked and agent.tool_progress_callback: + try: + preview = _build_tool_preview(function_name, function_args) + agent.tool_progress_callback("tool.started", function_name, preview, function_args) + except Exception as cb_err: + logging.debug(f"Tool progress callback error: {cb_err}") + + if not _execution_blocked and agent.tool_start_callback: + try: + agent.tool_start_callback(tool_call.id, function_name, function_args) + except Exception as cb_err: + logging.debug(f"Tool start callback error: {cb_err}") + + # Checkpoint: snapshot working dir before file-mutating tools + if not _execution_blocked and function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: + try: + file_path = function_args.get("path", "") + if file_path: + work_dir = agent._checkpoint_mgr.get_working_dir_for_path(file_path) + agent._checkpoint_mgr.ensure_checkpoint( + work_dir, f"before {function_name}" + ) + except Exception: + pass # never block tool execution + + # Checkpoint before destructive terminal commands + if not _execution_blocked and function_name == "terminal" and agent._checkpoint_mgr.enabled: + try: + cmd = function_args.get("command", "") + if _is_destructive_command(cmd): + cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) + agent._checkpoint_mgr.ensure_checkpoint( + cwd, f"before terminal: {cmd[:60]}" + ) + except Exception: + pass # never block tool execution + + tool_start_time = time.time() + + if _block_msg is not None: + # Tool blocked by plugin policy — return error without executing. + function_result = json.dumps({"error": _block_msg}, ensure_ascii=False) + tool_duration = 0.0 + elif _guardrail_block_decision is not None: + # Tool blocked by tool-loop guardrail — synthesize exactly one + # tool result for the original tool_call_id without executing. + function_result = agent._guardrail_block_result(_guardrail_block_decision) + tool_duration = 0.0 + elif function_name == "todo": + from tools.todo_tool import todo_tool as _todo_tool + function_result = _todo_tool( + todos=function_args.get("todos"), + merge=function_args.get("merge", False), + store=agent._todo_store, + ) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") + elif function_name == "session_search": + session_db = agent._get_session_db_for_recall() + if not session_db: + from hermes_state import format_session_db_unavailable + function_result = json.dumps({"success": False, "error": format_session_db_unavailable()}) + else: + from tools.session_search_tool import session_search as _session_search + function_result = _session_search( + query=function_args.get("query", ""), + role_filter=function_args.get("role_filter"), + limit=function_args.get("limit", 3), + db=session_db, + current_session_id=agent.session_id, + ) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") + elif function_name == "memory": + target = function_args.get("target", "memory") + from tools.memory_tool import memory_tool as _memory_tool + function_result = _memory_tool( + action=function_args.get("action"), + target=target, + content=function_args.get("content"), + old_text=function_args.get("old_text"), + store=agent._memory_store, + ) + # Bridge: notify external memory provider of built-in memory writes + if agent._memory_manager and function_args.get("action") in {"add", "replace"}: + try: + agent._memory_manager.on_memory_write( + function_args.get("action", ""), + target, + function_args.get("content", ""), + metadata=agent._build_memory_write_metadata( + task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", None), + ), + ) + except Exception: + pass + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") + elif function_name == "clarify": + from tools.clarify_tool import clarify_tool as _clarify_tool + function_result = _clarify_tool( + question=function_args.get("question", ""), + choices=function_args.get("choices"), + callback=agent.clarify_callback, + ) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") + elif function_name == "delegate_task": + tasks_arg = function_args.get("tasks") + if tasks_arg and isinstance(tasks_arg, list): + spinner_label = f"🔀 delegating {len(tasks_arg)} tasks" + else: + goal_preview = (function_args.get("goal") or "")[:30] + spinner_label = f"🔀 {goal_preview}" if goal_preview else "🔀 delegating" + spinner = None + if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): + face = random.choice(KawaiiSpinner.get_waiting_faces()) + spinner = KawaiiSpinner(f"{face} {spinner_label}", spinner_type='dots', print_fn=agent._print_fn) + spinner.start() + agent._delegate_spinner = spinner + _delegate_result = None + try: + function_result = agent._dispatch_delegate_task(function_args) + _delegate_result = function_result + finally: + agent._delegate_spinner = None + tool_duration = time.time() - tool_start_time + cute_msg = _get_cute_tool_message_impl('delegate_task', function_args, tool_duration, result=_delegate_result) + if spinner: + spinner.stop(cute_msg) + elif agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {cute_msg}") + elif agent._context_engine_tool_names and function_name in agent._context_engine_tool_names: + # Context engine tools (lcm_grep, lcm_describe, lcm_expand, etc.) + spinner = None + if agent._should_emit_quiet_tool_messages(): + face = random.choice(KawaiiSpinner.get_waiting_faces()) + emoji = _get_tool_emoji(function_name) + preview = _build_tool_preview(function_name, function_args) or function_name + spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) + spinner.start() + _ce_result = None + try: + function_result = agent.context_compressor.handle_tool_call(function_name, function_args, messages=messages) + _ce_result = function_result + except Exception as tool_error: + function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) + logger.error("context_engine.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) + finally: + tool_duration = time.time() - tool_start_time + cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_ce_result) + if spinner: + spinner.stop(cute_msg) + elif agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {cute_msg}") + elif agent._memory_manager and agent._memory_manager.has_tool(function_name): + # Memory provider tools (hindsight_retain, honcho_search, etc.) + # These are not in the tool registry — route through MemoryManager. + spinner = None + if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): + face = random.choice(KawaiiSpinner.get_waiting_faces()) + emoji = _get_tool_emoji(function_name) + preview = _build_tool_preview(function_name, function_args) or function_name + spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) + spinner.start() + _mem_result = None + try: + function_result = agent._memory_manager.handle_tool_call(function_name, function_args) + _mem_result = function_result + except Exception as tool_error: + function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) + logger.error("memory_manager.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) + finally: + tool_duration = time.time() - tool_start_time + cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_mem_result) + if spinner: + spinner.stop(cute_msg) + elif agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {cute_msg}") + elif agent.quiet_mode: + spinner = None + if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): + face = random.choice(KawaiiSpinner.get_waiting_faces()) + emoji = _get_tool_emoji(function_name) + preview = _build_tool_preview(function_name, function_args) or function_name + spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) + spinner.start() + _spinner_result = None + try: + function_result = _ra().handle_function_call( + function_name, function_args, effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, + skip_pre_tool_call_hook=True, + ) + _spinner_result = function_result + except Exception as tool_error: + function_result = f"Error executing tool '{function_name}': {tool_error}" + logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) + finally: + tool_duration = time.time() - tool_start_time + cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_spinner_result) + if spinner: + spinner.stop(cute_msg) + elif agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {cute_msg}") + else: + try: + function_result = _ra().handle_function_call( + function_name, function_args, effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, + skip_pre_tool_call_hook=True, + ) + except Exception as tool_error: + function_result = f"Error executing tool '{function_name}': {tool_error}" + logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) + tool_duration = time.time() - tool_start_time + + if isinstance(function_result, str): + result_preview = function_result if agent.verbose_logging else ( + function_result[:200] if len(function_result) > 200 else function_result + ) + _result_len = len(function_result) + else: + # Multimodal dict result (_multimodal=True) — not sliceable as string + result_preview = function_result + _result_len = len(str(function_result)) + + # Log tool errors to the persistent error log so [error] tags + # in the UI always have a corresponding detailed entry on disk. + _is_error_result, _ = _detect_tool_failure(function_name, function_result) + if not _execution_blocked: + function_result = agent._append_guardrail_observation( + function_name, + function_args, + function_result, + failed=_is_error_result, + ) + result_preview = function_result if agent.verbose_logging else ( + function_result[:200] if len(function_result) > 200 else function_result + ) + if _is_error_result: + logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) + else: + logger.info("tool %s completed (%.2fs, %d chars)", function_name, tool_duration, _result_len) + + # Track file-mutation outcome for the turn-end verifier. See + # the concurrent path for the rationale; both paths must feed + # the same state so the footer reflects every tool call in the + # turn, not just the parallel ones. + if not _execution_blocked: + try: + agent._record_file_mutation_result( + function_name, function_args, function_result, _is_error_result, + ) + except Exception as _ver_err: + logging.debug("file-mutation verifier record failed: %s", _ver_err) + + if not _execution_blocked and agent.tool_progress_callback: + try: + agent.tool_progress_callback( + "tool.completed", function_name, None, None, + duration=tool_duration, is_error=_is_error_result, + ) + except Exception as cb_err: + logging.debug(f"Tool progress callback error: {cb_err}") + + agent._current_tool = None + agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)") + + if agent.verbose_logging: + logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + _log_result = _multimodal_text_summary(function_result) + logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}") + + if not _execution_blocked and agent.tool_complete_callback: + try: + agent.tool_complete_callback(tool_call.id, function_name, function_args, function_result) + except Exception as cb_err: + logging.debug(f"Tool complete callback error: {cb_err}") + + function_result = maybe_persist_tool_result( + content=function_result, + tool_name=function_name, + tool_use_id=tool_call.id, + env=get_active_env(effective_task_id), + ) if not _is_multimodal_tool_result(function_result) else function_result + + # Discover subdirectory context files from tool arguments + subdir_hints = agent._subdirectory_hints.check_tool_call(function_name, function_args) + if subdir_hints: + if _is_multimodal_tool_result(function_result): + _append_subdir_hint_to_multimodal(function_result, subdir_hints) + else: + function_result += subdir_hints + + # Unwrap _multimodal dicts to an OpenAI-style content list + # (see parallel path for rationale). String results pass through. + _tool_content = agent._tool_result_content_for_active_model(function_name, function_result) + tool_msg = { + "role": "tool", + "name": function_name, + "content": _tool_content, + "tool_call_id": tool_call.id + } + messages.append(tool_msg) + + # ── Per-tool /steer drain ─────────────────────────────────── + # Drain pending steer BETWEEN individual tool calls so the + # injection lands as soon as a tool finishes — not after the + # entire batch. The model sees it on the next API iteration. + agent._apply_pending_steer_to_tool_results(messages, 1) + + if not agent.quiet_mode: + if agent.verbose_logging: + print(f" ✅ Tool {i} completed in {tool_duration:.2f}s") + print(agent._wrap_verbose("Result: ", function_result)) + else: + _fr_str = function_result if isinstance(function_result, str) else str(function_result) + response_preview = _fr_str[:agent.log_prefix_chars] + "..." if len(_fr_str) > agent.log_prefix_chars else _fr_str + print(f" ✅ Tool {i} completed in {tool_duration:.2f}s - {response_preview}") + + if agent._interrupt_requested and i < len(assistant_message.tool_calls): + remaining = len(assistant_message.tool_calls) - i + agent._vprint(f"{agent.log_prefix}⚡ Interrupt: skipping {remaining} remaining tool call(s)", force=True) + for skipped_tc in assistant_message.tool_calls[i:]: + skipped_name = skipped_tc.function.name + skip_msg = { + "role": "tool", + "name": skipped_name, + "content": f"[Tool execution skipped — {skipped_name} was not started. User sent a new message]", + "tool_call_id": skipped_tc.id + } + messages.append(skip_msg) + break + + if agent.tool_delay > 0 and i < len(assistant_message.tool_calls): + time.sleep(agent.tool_delay) + + # ── Per-turn aggregate budget enforcement ───────────────────────── + num_tools_seq = len(assistant_message.tool_calls) + if num_tools_seq > 0: + enforce_turn_budget(messages[-num_tools_seq:], env=get_active_env(effective_task_id)) + + # ── /steer injection ────────────────────────────────────────────── + # See _execute_tool_calls_parallel for the rationale. Same hook, + # applied to sequential execution as well. + if num_tools_seq > 0: + agent._apply_pending_steer_to_tool_results(messages, num_tools_seq) + + + + +__all__ = [ + "execute_tool_calls_concurrent", + "execute_tool_calls_sequential", +] diff --git a/run_agent.py b/run_agent.py index a5beda7765e..b5ea98d911d 100644 --- a/run_agent.py +++ b/run_agent.py @@ -8987,853 +8987,14 @@ class AIAgent: return f"{indent}{label}{body}" def _execute_tool_calls_concurrent(self, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: - """Execute multiple tool calls concurrently using a thread pool. - - Results are collected in the original tool-call order and appended to - messages so the API sees them in the expected sequence. - """ - tool_calls = assistant_message.tool_calls - num_tools = len(tool_calls) - - # ── Pre-flight: interrupt check ────────────────────────────────── - if self._interrupt_requested: - print(f"{self.log_prefix}⚡ Interrupt: skipping {num_tools} tool call(s)") - for tc in tool_calls: - messages.append({ - "role": "tool", - "name": tc.function.name, - "content": f"[Tool execution cancelled — {tc.function.name} was skipped due to user interrupt]", - "tool_call_id": tc.id, - }) - return - - # ── Parse args + pre-execution bookkeeping ─────────────────────── - parsed_calls = [] # list of (tool_call, function_name, function_args) - for tool_call in tool_calls: - function_name = tool_call.function.name - - # Reset nudge counters - if function_name == "memory": - self._turns_since_memory = 0 - elif function_name == "skill_manage": - self._iters_since_skill = 0 - - try: - function_args = json.loads(tool_call.function.arguments) - except json.JSONDecodeError: - function_args = {} - if not isinstance(function_args, dict): - function_args = {} - - # Checkpoint for file-mutating tools - if function_name in {"write_file", "patch"} and self._checkpoint_mgr.enabled: - try: - file_path = function_args.get("path", "") - if file_path: - work_dir = self._checkpoint_mgr.get_working_dir_for_path(file_path) - self._checkpoint_mgr.ensure_checkpoint(work_dir, f"before {function_name}") - except Exception: - pass - - # Checkpoint before destructive terminal commands - if function_name == "terminal" and self._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - self._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass - - block_result = None - blocked_by_guardrail = False - try: - from hermes_cli.plugins import get_pre_tool_call_block_message - block_message = get_pre_tool_call_block_message( - function_name, function_args, task_id=effective_task_id or "", - ) - except Exception: - block_message = None - - if block_message is not None: - block_result = json.dumps({"error": block_message}, ensure_ascii=False) - else: - guardrail_decision = self._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - block_result = self._guardrail_block_result(guardrail_decision) - blocked_by_guardrail = True - - parsed_calls.append((tool_call, function_name, function_args, block_result, blocked_by_guardrail)) - - # ── Logging / callbacks ────────────────────────────────────────── - tool_names_str = ", ".join(name for _, name, _, _, _ in parsed_calls) - if not self.quiet_mode: - print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") - for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1): - args_str = json.dumps(args, ensure_ascii=False) - if self.verbose_logging: - print(f" 📞 Tool {i}: {name}({list(args.keys())})") - print(self._wrap_verbose("Args: ", json.dumps(args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:self.log_prefix_chars] + "..." if len(args_str) > self.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}") - - for tc, name, args, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if self.tool_progress_callback: - try: - preview = _build_tool_preview(name, args) - self.tool_progress_callback("tool.started", name, preview, args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - for tc, name, args, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if self.tool_start_callback: - try: - self.tool_start_callback(tc.id, name, args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") - - # ── Concurrent execution ───────────────────────────────────────── - # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag) - results = [None] * num_tools - for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls): - if block_result is not None: - results[i] = (name, args, block_result, 0.0, True, True) - - # Touch activity before launching workers so the gateway knows - # we're executing tools (not stuck). - self._current_tool = tool_names_str - self._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") - - # Capture CLI callbacks from the agent thread so worker threads can - # register them locally. Without this, _get_approval_callback() in - # terminal_tool returns None in ThreadPoolExecutor workers, causing - # the dangerous-command prompt to fall back to input() — which - # deadlocks against prompt_toolkit's raw terminal mode (#13617). - _parent_approval_cb = _get_approval_callback() - _parent_sudo_cb = _get_sudo_password_callback() - - def _run_tool(index, tool_call, function_name, function_args): - """Worker function executed in a thread.""" - # Register this worker tid so the agent can fan out an interrupt - # to it — see AIAgent.interrupt(). Must happen first thing, and - # must be paired with discard + clear in the finally block. - _worker_tid = threading.current_thread().ident - with self._tool_worker_threads_lock: - self._tool_worker_threads.add(_worker_tid) - # Race: if the agent was interrupted between fan-out (which - # snapshotted an empty/earlier set) and our registration, apply - # the interrupt to our own tid now so is_interrupted() inside - # the tool returns True on the next poll. - if self._interrupt_requested: - try: - _set_interrupt(True, _worker_tid) - except Exception: - pass - # Set the activity callback on THIS worker thread so - # _wait_for_process (terminal commands) can fire heartbeats. - # The callback is thread-local; the main thread's callback - # is invisible to worker threads. - try: - from tools.environments.base import set_activity_callback - set_activity_callback(self._touch_activity) - except Exception: - pass - # Propagate approval/sudo callbacks to this worker thread. - # Mirrors cli.py run_agent() pattern (GHSA-qg5c-hvr5-hjgr). - if _parent_approval_cb is not None: - try: - _set_approval_callback(_parent_approval_cb) - except Exception: - pass - if _parent_sudo_cb is not None: - try: - _set_sudo_password_callback(_parent_sudo_cb) - except Exception: - pass - start = time.time() - try: - result = self._invoke_tool( - function_name, - function_args, - effective_task_id, - tool_call.id, - messages=messages, - pre_tool_block_checked=True, - ) - except Exception as tool_error: - result = f"Error executing tool '{function_name}': {tool_error}" - logger.error("_invoke_tool raised for %s: %s", function_name, tool_error, exc_info=True) - duration = time.time() - start - is_error, _ = _detect_tool_failure(function_name, result) - if is_error: - logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) - else: - logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) - results[index] = (function_name, function_args, result, duration, is_error, False) - # Tear down worker-tid tracking. Clear any interrupt bit we may - # have set so the next task scheduled onto this recycled tid - # starts with a clean slate. - with self._tool_worker_threads_lock: - self._tool_worker_threads.discard(_worker_tid) - try: - _set_interrupt(False, _worker_tid) - except Exception: - pass - # Clear thread-local callbacks so a recycled worker thread - # doesn't hold stale references to a disposed CLI instance. - try: - _set_approval_callback(None) - _set_sudo_password_callback(None) - except Exception: - pass - - # Start spinner for CLI mode (skip when TUI handles tool progress) - spinner = None - if self._should_emit_quiet_tool_messages() and self._should_start_quiet_spinner(): - face = random.choice(KawaiiSpinner.get_waiting_faces()) - spinner = KawaiiSpinner(f"{face} ⚡ running {num_tools} tools concurrently", spinner_type='dots', print_fn=self._print_fn) - spinner.start() - - try: - runnable_calls = [ - (i, tc, name, args) - for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls) - if block_result is None - ] - futures = [] - if runnable_calls: - max_workers = min(len(runnable_calls), _MAX_TOOL_WORKERS) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - for i, tc, name, args in runnable_calls: - # Propagate ContextVars (e.g. _approval_session_key); mirrors asyncio.to_thread. - ctx = contextvars.copy_context() - f = executor.submit(ctx.run, _run_tool, i, tc, name, args) - futures.append(f) - - # Wait for all to complete with periodic heartbeats so the - # gateway's inactivity monitor doesn't kill us during long - # concurrent tool batches. Also check for user interrupts - # so we don't block indefinitely when the user sends /stop - # or a new message during concurrent tool execution. - _conc_start = time.time() - _interrupt_logged = False - while True: - done, not_done = concurrent.futures.wait( - futures, timeout=5.0, - ) - if not not_done: - break - - # Check for interrupt — the per-thread interrupt signal - # already causes individual tools (terminal, execute_code) - # to abort, but tools without interrupt checks (web_search, - # read_file) will run to completion. Cancel any futures - # that haven't started yet so we don't block on them. - if self._interrupt_requested: - if not _interrupt_logged: - _interrupt_logged = True - self._vprint( - f"{self.log_prefix}⚡ Interrupt: cancelling " - f"{len(not_done)} pending concurrent tool(s)", - force=True, - ) - for f in not_done: - f.cancel() - # Give already-running tools a moment to notice the - # per-thread interrupt signal and exit gracefully. - concurrent.futures.wait(not_done, timeout=3.0) - break - - _conc_elapsed = int(time.time() - _conc_start) - # Heartbeat every ~30s (6 × 5s poll intervals) - if _conc_elapsed > 0 and _conc_elapsed % 30 < 6: - _still_running = [ - parsed_calls[futures.index(f)][1] - for f in not_done - if f in futures - ] - self._touch_activity( - f"concurrent tools running ({_conc_elapsed}s, " - f"{len(not_done)} remaining: {', '.join(_still_running[:3])})" - ) - finally: - if spinner: - # Build a summary message for the spinner stop - completed = sum(1 for r in results if r is not None) - total_dur = sum(r[3] for r in results if r is not None) - spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") - - # ── Post-execution: display per-tool results ───────────────────── - for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls): - r = results[i] - blocked = False - if r is None: - # Tool was cancelled (interrupt) or thread didn't return - if self._interrupt_requested: - function_result = f"[Tool execution cancelled — {name} was skipped due to user interrupt]" - else: - function_result = f"Error executing tool '{name}': thread did not return a result" - tool_duration = 0.0 - else: - function_name, function_args, function_result, tool_duration, is_error, blocked = r - - if not blocked: - function_result = self._append_guardrail_observation( - function_name, - function_args, - function_result, - failed=is_error, - ) - - if is_error: - _err_text = _multimodal_text_summary(function_result) - result_preview = _err_text[:200] if len(_err_text) > 200 else _err_text - logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) - - # Track file-mutation outcome for the turn-end verifier. - # `blocked` calls never actually ran — don't let a guardrail - # block count as either a failure or a success. - if not blocked: - try: - self._record_file_mutation_result( - function_name, function_args, function_result, is_error, - ) - except Exception as _ver_err: - logging.debug("file-mutation verifier record failed: %s", _ver_err) - - if not blocked and self.tool_progress_callback: - try: - self.tool_progress_callback( - "tool.completed", function_name, None, None, - duration=tool_duration, is_error=is_error, - ) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if self.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") - - # Print cute message per tool - if self._should_emit_quiet_tool_messages(): - cute_msg = _get_cute_tool_message_impl(name, args, tool_duration, result=function_result) - self._safe_print(f" {cute_msg}") - elif not self.quiet_mode: - _preview_str = _multimodal_text_summary(function_result) - if self.verbose_logging: - print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s") - print(self._wrap_verbose("Result: ", _preview_str)) - else: - response_preview = _preview_str[:self.log_prefix_chars] + "..." if len(_preview_str) > self.log_prefix_chars else _preview_str - print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") - - self._current_tool = None - self._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)") - - if not blocked and self.tool_complete_callback: - try: - self.tool_complete_callback(tc.id, name, args, function_result) - except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") - - function_result = maybe_persist_tool_result( - content=function_result, - tool_name=name, - tool_use_id=tc.id, - env=get_active_env(effective_task_id), - ) if not _is_multimodal_tool_result(function_result) else function_result - - subdir_hints = self._subdirectory_hints.check_tool_call(name, args) - if subdir_hints: - if _is_multimodal_tool_result(function_result): - # Append the hint to the text summary part so the model - # still sees it; don't touch the image blocks. - _append_subdir_hint_to_multimodal(function_result, subdir_hints) - else: - function_result += subdir_hints - - # Unwrap _multimodal dicts to an OpenAI-style content list so any - # vision-capable provider receives [{type:text},{type:image_url}] - # rather than a raw Python dict. The Anthropic adapter already - # accepts content lists; vision-capable OpenAI-compatible servers - # (mlx-vlm, GPT-4o, …) accept image_url in tool messages natively. - # Text-only servers get a string-safe fallback here so a rejected - # image tool result never poisons canonical session history. - # String results pass through unchanged. - _tool_content = self._tool_result_content_for_active_model(name, function_result) - tool_msg = { - "role": "tool", - "name": name, - "content": _tool_content, - "tool_call_id": tc.id, - } - messages.append(tool_msg) - - # ── Per-tool /steer drain ─────────────────────────────────── - # Same as the sequential path: drain between each collected - # result so the steer lands as early as possible. - self._apply_pending_steer_to_tool_results(messages, 1) - - # ── Per-turn aggregate budget enforcement ───────────────────────── - num_tools = len(parsed_calls) - if num_tools > 0: - turn_tool_msgs = messages[-num_tools:] - enforce_turn_budget(turn_tool_msgs, env=get_active_env(effective_task_id)) - - # ── /steer injection ────────────────────────────────────────────── - # Append any pending user steer text to the last tool result so the - # agent sees it on its next iteration. Runs AFTER budget enforcement - # so the steer marker is never truncated. See steer() for details. - if num_tools > 0: - self._apply_pending_steer_to_tool_results(messages, num_tools) + """Forwarder — see ``agent.tool_executor.execute_tool_calls_concurrent``.""" + from agent.tool_executor import execute_tool_calls_concurrent + return execute_tool_calls_concurrent(self, assistant_message, messages, effective_task_id, api_call_count) def _execute_tool_calls_sequential(self, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None: - """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools.""" - for i, tool_call in enumerate(assistant_message.tool_calls, 1): - # SAFETY: check interrupt BEFORE starting each tool. - # If the user sent "stop" during a previous tool's execution, - # do NOT start any more tools -- skip them all immediately. - if self._interrupt_requested: - remaining_calls = assistant_message.tool_calls[i-1:] - if remaining_calls: - self._vprint(f"{self.log_prefix}⚡ Interrupt: skipping {len(remaining_calls)} tool call(s)", force=True) - for skipped_tc in remaining_calls: - skipped_name = skipped_tc.function.name - skip_msg = { - "role": "tool", - "name": skipped_name, - "content": f"[Tool execution cancelled — {skipped_name} was skipped due to user interrupt]", - "tool_call_id": skipped_tc.id, - } - messages.append(skip_msg) - break - - function_name = tool_call.function.name - - try: - function_args = json.loads(tool_call.function.arguments) - except json.JSONDecodeError as e: - logging.warning(f"Unexpected JSON error after validation: {e}") - function_args = {} - if not isinstance(function_args, dict): - function_args = {} - - # Check plugin hooks for a block directive before executing. - _block_msg: Optional[str] = None - try: - from hermes_cli.plugins import get_pre_tool_call_block_message - _block_msg = get_pre_tool_call_block_message( - function_name, function_args, task_id=effective_task_id or "", - ) - except Exception: - pass - - _guardrail_block_decision: ToolGuardrailDecision | None = None - if _block_msg is None: - guardrail_decision = self._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - _guardrail_block_decision = guardrail_decision - - _execution_blocked = _block_msg is not None or _guardrail_block_decision is not None - - if _execution_blocked: - # Tool blocked by plugin or guardrail policy — skip counters, - # callbacks, checkpointing, activity mutation, and real execution. - pass - # Reset nudge counters when the relevant tool is actually used - elif function_name == "memory": - self._turns_since_memory = 0 - elif function_name == "skill_manage": - self._iters_since_skill = 0 - - if not self.quiet_mode: - args_str = json.dumps(function_args, ensure_ascii=False) - if self.verbose_logging: - print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())})") - print(self._wrap_verbose("Args: ", json.dumps(function_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:self.log_prefix_chars] + "..." if len(args_str) > self.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}") - - if not _execution_blocked: - self._current_tool = function_name - self._touch_activity(f"executing tool: {function_name}") - - # Set activity callback for long-running tool execution (terminal - # commands, etc.) so the gateway's inactivity monitor doesn't kill - # the agent while a command is running. - if not _execution_blocked: - try: - from tools.environments.base import set_activity_callback - set_activity_callback(self._touch_activity) - except Exception: - pass - - if not _execution_blocked and self.tool_progress_callback: - try: - preview = _build_tool_preview(function_name, function_args) - self.tool_progress_callback("tool.started", function_name, preview, function_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if not _execution_blocked and self.tool_start_callback: - try: - self.tool_start_callback(tool_call.id, function_name, function_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") - - # Checkpoint: snapshot working dir before file-mutating tools - if not _execution_blocked and function_name in {"write_file", "patch"} and self._checkpoint_mgr.enabled: - try: - file_path = function_args.get("path", "") - if file_path: - work_dir = self._checkpoint_mgr.get_working_dir_for_path(file_path) - self._checkpoint_mgr.ensure_checkpoint( - work_dir, f"before {function_name}" - ) - except Exception: - pass # never block tool execution - - # Checkpoint before destructive terminal commands - if not _execution_blocked and function_name == "terminal" and self._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - self._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass # never block tool execution - - tool_start_time = time.time() - - if _block_msg is not None: - # Tool blocked by plugin policy — return error without executing. - function_result = json.dumps({"error": _block_msg}, ensure_ascii=False) - tool_duration = 0.0 - elif _guardrail_block_decision is not None: - # Tool blocked by tool-loop guardrail — synthesize exactly one - # tool result for the original tool_call_id without executing. - function_result = self._guardrail_block_result(_guardrail_block_decision) - tool_duration = 0.0 - elif function_name == "todo": - from tools.todo_tool import todo_tool as _todo_tool - function_result = _todo_tool( - todos=function_args.get("todos"), - merge=function_args.get("merge", False), - store=self._todo_store, - ) - tool_duration = time.time() - tool_start_time - if self._should_emit_quiet_tool_messages(): - self._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") - elif function_name == "session_search": - session_db = self._get_session_db_for_recall() - if not session_db: - from hermes_state import format_session_db_unavailable - function_result = json.dumps({"success": False, "error": format_session_db_unavailable()}) - else: - from tools.session_search_tool import session_search as _session_search - function_result = _session_search( - query=function_args.get("query", ""), - role_filter=function_args.get("role_filter"), - limit=function_args.get("limit", 3), - db=session_db, - current_session_id=self.session_id, - ) - tool_duration = time.time() - tool_start_time - if self._should_emit_quiet_tool_messages(): - self._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") - elif function_name == "memory": - target = function_args.get("target", "memory") - from tools.memory_tool import memory_tool as _memory_tool - function_result = _memory_tool( - action=function_args.get("action"), - target=target, - content=function_args.get("content"), - old_text=function_args.get("old_text"), - store=self._memory_store, - ) - # Bridge: notify external memory provider of built-in memory writes - if self._memory_manager and function_args.get("action") in {"add", "replace"}: - try: - self._memory_manager.on_memory_write( - function_args.get("action", ""), - target, - function_args.get("content", ""), - metadata=self._build_memory_write_metadata( - task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", None), - ), - ) - except Exception: - pass - tool_duration = time.time() - tool_start_time - if self._should_emit_quiet_tool_messages(): - self._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") - elif function_name == "clarify": - from tools.clarify_tool import clarify_tool as _clarify_tool - function_result = _clarify_tool( - question=function_args.get("question", ""), - choices=function_args.get("choices"), - callback=self.clarify_callback, - ) - tool_duration = time.time() - tool_start_time - if self._should_emit_quiet_tool_messages(): - self._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") - elif function_name == "delegate_task": - tasks_arg = function_args.get("tasks") - if tasks_arg and isinstance(tasks_arg, list): - spinner_label = f"🔀 delegating {len(tasks_arg)} tasks" - else: - goal_preview = (function_args.get("goal") or "")[:30] - spinner_label = f"🔀 {goal_preview}" if goal_preview else "🔀 delegating" - spinner = None - if self._should_emit_quiet_tool_messages() and self._should_start_quiet_spinner(): - face = random.choice(KawaiiSpinner.get_waiting_faces()) - spinner = KawaiiSpinner(f"{face} {spinner_label}", spinner_type='dots', print_fn=self._print_fn) - spinner.start() - self._delegate_spinner = spinner - _delegate_result = None - try: - function_result = self._dispatch_delegate_task(function_args) - _delegate_result = function_result - finally: - self._delegate_spinner = None - tool_duration = time.time() - tool_start_time - cute_msg = _get_cute_tool_message_impl('delegate_task', function_args, tool_duration, result=_delegate_result) - if spinner: - spinner.stop(cute_msg) - elif self._should_emit_quiet_tool_messages(): - self._vprint(f" {cute_msg}") - elif self._context_engine_tool_names and function_name in self._context_engine_tool_names: - # Context engine tools (lcm_grep, lcm_describe, lcm_expand, etc.) - spinner = None - if self._should_emit_quiet_tool_messages(): - face = random.choice(KawaiiSpinner.get_waiting_faces()) - emoji = _get_tool_emoji(function_name) - preview = _build_tool_preview(function_name, function_args) or function_name - spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=self._print_fn) - spinner.start() - _ce_result = None - try: - function_result = self.context_compressor.handle_tool_call(function_name, function_args, messages=messages) - _ce_result = function_result - except Exception as tool_error: - function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) - logger.error("context_engine.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) - finally: - tool_duration = time.time() - tool_start_time - cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_ce_result) - if spinner: - spinner.stop(cute_msg) - elif self._should_emit_quiet_tool_messages(): - self._vprint(f" {cute_msg}") - elif self._memory_manager and self._memory_manager.has_tool(function_name): - # Memory provider tools (hindsight_retain, honcho_search, etc.) - # These are not in the tool registry — route through MemoryManager. - spinner = None - if self._should_emit_quiet_tool_messages() and self._should_start_quiet_spinner(): - face = random.choice(KawaiiSpinner.get_waiting_faces()) - emoji = _get_tool_emoji(function_name) - preview = _build_tool_preview(function_name, function_args) or function_name - spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=self._print_fn) - spinner.start() - _mem_result = None - try: - function_result = self._memory_manager.handle_tool_call(function_name, function_args) - _mem_result = function_result - except Exception as tool_error: - function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) - logger.error("memory_manager.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) - finally: - tool_duration = time.time() - tool_start_time - cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_mem_result) - if spinner: - spinner.stop(cute_msg) - elif self._should_emit_quiet_tool_messages(): - self._vprint(f" {cute_msg}") - elif self.quiet_mode: - spinner = None - if self._should_emit_quiet_tool_messages() and self._should_start_quiet_spinner(): - face = random.choice(KawaiiSpinner.get_waiting_faces()) - emoji = _get_tool_emoji(function_name) - preview = _build_tool_preview(function_name, function_args) or function_name - spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=self._print_fn) - spinner.start() - _spinner_result = None - try: - function_result = handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=self.session_id or "", - enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, - skip_pre_tool_call_hook=True, - ) - _spinner_result = function_result - except Exception as tool_error: - function_result = f"Error executing tool '{function_name}': {tool_error}" - logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) - finally: - tool_duration = time.time() - tool_start_time - cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_spinner_result) - if spinner: - spinner.stop(cute_msg) - elif self._should_emit_quiet_tool_messages(): - self._vprint(f" {cute_msg}") - else: - try: - function_result = handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=self.session_id or "", - enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, - skip_pre_tool_call_hook=True, - ) - except Exception as tool_error: - function_result = f"Error executing tool '{function_name}': {tool_error}" - logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) - tool_duration = time.time() - tool_start_time - - if isinstance(function_result, str): - result_preview = function_result if self.verbose_logging else ( - function_result[:200] if len(function_result) > 200 else function_result - ) - _result_len = len(function_result) - else: - # Multimodal dict result (_multimodal=True) — not sliceable as string - result_preview = function_result - _result_len = len(str(function_result)) - - # Log tool errors to the persistent error log so [error] tags - # in the UI always have a corresponding detailed entry on disk. - _is_error_result, _ = _detect_tool_failure(function_name, function_result) - if not _execution_blocked: - function_result = self._append_guardrail_observation( - function_name, - function_args, - function_result, - failed=_is_error_result, - ) - result_preview = function_result if self.verbose_logging else ( - function_result[:200] if len(function_result) > 200 else function_result - ) - if _is_error_result: - logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) - else: - logger.info("tool %s completed (%.2fs, %d chars)", function_name, tool_duration, _result_len) - - # Track file-mutation outcome for the turn-end verifier. See - # the concurrent path for the rationale; both paths must feed - # the same state so the footer reflects every tool call in the - # turn, not just the parallel ones. - if not _execution_blocked: - try: - self._record_file_mutation_result( - function_name, function_args, function_result, _is_error_result, - ) - except Exception as _ver_err: - logging.debug("file-mutation verifier record failed: %s", _ver_err) - - if not _execution_blocked and self.tool_progress_callback: - try: - self.tool_progress_callback( - "tool.completed", function_name, None, None, - duration=tool_duration, is_error=_is_error_result, - ) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - self._current_tool = None - self._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)") - - if self.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - _log_result = _multimodal_text_summary(function_result) - logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}") - - if not _execution_blocked and self.tool_complete_callback: - try: - self.tool_complete_callback(tool_call.id, function_name, function_args, function_result) - except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") - - function_result = maybe_persist_tool_result( - content=function_result, - tool_name=function_name, - tool_use_id=tool_call.id, - env=get_active_env(effective_task_id), - ) if not _is_multimodal_tool_result(function_result) else function_result - - # Discover subdirectory context files from tool arguments - subdir_hints = self._subdirectory_hints.check_tool_call(function_name, function_args) - if subdir_hints: - if _is_multimodal_tool_result(function_result): - _append_subdir_hint_to_multimodal(function_result, subdir_hints) - else: - function_result += subdir_hints - - # Unwrap _multimodal dicts to an OpenAI-style content list - # (see parallel path for rationale). String results pass through. - _tool_content = self._tool_result_content_for_active_model(function_name, function_result) - tool_msg = { - "role": "tool", - "name": function_name, - "content": _tool_content, - "tool_call_id": tool_call.id - } - messages.append(tool_msg) - - # ── Per-tool /steer drain ─────────────────────────────────── - # Drain pending steer BETWEEN individual tool calls so the - # injection lands as soon as a tool finishes — not after the - # entire batch. The model sees it on the next API iteration. - self._apply_pending_steer_to_tool_results(messages, 1) - - if not self.quiet_mode: - if self.verbose_logging: - print(f" ✅ Tool {i} completed in {tool_duration:.2f}s") - print(self._wrap_verbose("Result: ", function_result)) - else: - _fr_str = function_result if isinstance(function_result, str) else str(function_result) - response_preview = _fr_str[:self.log_prefix_chars] + "..." if len(_fr_str) > self.log_prefix_chars else _fr_str - print(f" ✅ Tool {i} completed in {tool_duration:.2f}s - {response_preview}") - - if self._interrupt_requested and i < len(assistant_message.tool_calls): - remaining = len(assistant_message.tool_calls) - i - self._vprint(f"{self.log_prefix}⚡ Interrupt: skipping {remaining} remaining tool call(s)", force=True) - for skipped_tc in assistant_message.tool_calls[i:]: - skipped_name = skipped_tc.function.name - skip_msg = { - "role": "tool", - "name": skipped_name, - "content": f"[Tool execution skipped — {skipped_name} was not started. User sent a new message]", - "tool_call_id": skipped_tc.id - } - messages.append(skip_msg) - break - - if self.tool_delay > 0 and i < len(assistant_message.tool_calls): - time.sleep(self.tool_delay) - - # ── Per-turn aggregate budget enforcement ───────────────────────── - num_tools_seq = len(assistant_message.tool_calls) - if num_tools_seq > 0: - enforce_turn_budget(messages[-num_tools_seq:], env=get_active_env(effective_task_id)) - - # ── /steer injection ────────────────────────────────────────────── - # See _execute_tool_calls_parallel for the rationale. Same hook, - # applied to sequential execution as well. - if num_tools_seq > 0: - self._apply_pending_steer_to_tool_results(messages, num_tools_seq) - + """Forwarder — see ``agent.tool_executor.execute_tool_calls_sequential``.""" + from agent.tool_executor import execute_tool_calls_sequential + return execute_tool_calls_sequential(self, assistant_message, messages, effective_task_id, api_call_count) def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" diff --git a/tests/run_agent/test_tool_executor_contextvar_propagation.py b/tests/run_agent/test_tool_executor_contextvar_propagation.py index 652ecf05def..2e1d543705a 100644 --- a/tests/run_agent/test_tool_executor_contextvar_propagation.py +++ b/tests/run_agent/test_tool_executor_contextvar_propagation.py @@ -152,19 +152,28 @@ def test_run_agent_concurrent_executor_wraps_submit_with_copy_context(): import inspect import run_agent + from agent import tool_executor as tool_executor_module - src_path = inspect.getsourcefile(run_agent) - assert src_path is not None - tree = ast.parse(open(src_path, encoding="utf-8").read()) + # Source for both modules — the concurrent-executor body lives in + # ``agent/tool_executor.py`` after the run_agent.py refactor (PR + # following #16660). Search both so this guard keeps firing + # regardless of where the call site lives. + sources = [] + for mod in (run_agent, tool_executor_module): + src_path = inspect.getsourcefile(mod) + assert src_path is not None + sources.append((src_path, open(src_path, encoding="utf-8").read())) submit_calls_in_agent: list[ast.Call] = [] - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - func = node.func - # Match executor.submit(...) style calls. - if isinstance(func, ast.Attribute) and func.attr == "submit": - submit_calls_in_agent.append(node) + for _src_path, src_text in sources: + tree = ast.parse(src_text) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + # Match executor.submit(...) style calls. + if isinstance(func, ast.Attribute) and func.attr == "submit": + submit_calls_in_agent.append(node) # Filter to the submit call inside the concurrent tool executor — # identifiable by passing `_run_tool` as its target. Other submit() From 57f6762ca085839833b1eb9e2ca9e6cb69abcc4a Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:28:17 -0700 Subject: [PATCH 008/142] refactor(run_agent): extract stream diagnostics to agent/stream_diag.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the five stream-drop diagnostic helpers + the headers tuple: * STREAM_DIAG_HEADERS — cf-ray, x-openrouter-provider, x-request-id, etc. * stream_diag_init — fresh per-attempt diagnostic dict * stream_diag_capture_response — snapshot upstream headers + HTTP status * flatten_exception_chain — compact Outer(msg) <- Inner(msg) rendering * log_stream_retry — structured WARNING with provider/bytes/elapsed/ttfb * emit_stream_drop — user-facing status line + activity touch AIAgent keeps thin forwarder methods (and exposes the headers tuple as _STREAM_DIAG_HEADERS for back-compat). All test patches and call sites unchanged. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 13470 -> 13227 lines (-243). --- agent/stream_diag.py | 280 +++++++++++++++++++++++++++++++++++++++++++ run_agent.py | 234 ++++-------------------------------- 2 files changed, 303 insertions(+), 211 deletions(-) create mode 100644 agent/stream_diag.py diff --git a/agent/stream_diag.py b/agent/stream_diag.py new file mode 100644 index 00000000000..c4d8c54f470 --- /dev/null +++ b/agent/stream_diag.py @@ -0,0 +1,280 @@ +"""Stream diagnostics — per-attempt counters, exception chains, retry logging. + +When a streaming chat-completions request dies mid-response, we want to +know why: which Cloudflare edge served the request, which OpenRouter +downstream provider answered, how many bytes/chunks we got before the +drop, the HTTP status, the underlying httpx error class. These helpers +collect that info and emit it both to ``agent.log`` (full detail) and to +the user-facing status line (compact). + +All helpers are extracted from :class:`AIAgent` for cleanliness. +``run_agent`` keeps thin forwarder methods so existing call sites and +tests that patch ``run_agent.`` keep working. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +# Per-attempt stream diagnostic headers. Lowercased; httpx returns +# CIMultiDict so case-insensitive lookups already work, but we read .get() +# on the dict from agent.log for free-form post-hoc analysis. +STREAM_DIAG_HEADERS = ( + "cf-ray", + "cf-cache-status", + "x-openrouter-provider", + "x-openrouter-model", + "x-openrouter-id", + "x-request-id", + "x-vercel-id", + "via", + "server", + "x-forwarded-for", +) + + +def stream_diag_init() -> Dict[str, Any]: + """Return a fresh per-attempt diagnostic dict. + + Mutated in-place by the streaming functions and read from the retry + block when a stream dies. Lives on ``request_client_holder`` so it + survives across the closure boundary. + """ + return { + "started_at": time.time(), + "first_chunk_at": None, + "chunks": 0, + "bytes": 0, + "headers": {}, + "http_status": None, + } + + +def stream_diag_capture_response(agent: Any, diag: Dict[str, Any], http_response: Any) -> None: + """Snapshot interesting headers + HTTP status from the live stream. + + Called once at stream open (before iterating chunks) so the metadata + survives even if the stream dies before any chunk arrives. Failures + are swallowed — diag is best-effort. + """ + if http_response is None or not isinstance(diag, dict): + return + try: + diag["http_status"] = getattr(http_response, "status_code", None) + except Exception: + pass + try: + headers = getattr(http_response, "headers", None) or {} + captured: Dict[str, str] = {} + # Allow per-agent override of the headers list (back-compat). + target_headers = getattr(agent, "_STREAM_DIAG_HEADERS", STREAM_DIAG_HEADERS) + for name in target_headers: + try: + val = headers.get(name) + if val: + # Truncate single-value to keep log lines bounded. + captured[name] = str(val)[:120] + except Exception: + continue + diag["headers"] = captured + except Exception: + pass + + +def flatten_exception_chain(error: BaseException) -> str: + """Return a compact ``Outer(msg) <- Inner(msg) <- ...`` rendering. + + OpenAI SDK wraps httpx errors as ``APIConnectionError`` / + ``APIError`` and only the wrapper's class is visible at the catch + site — but the underlying ``RemoteProtocolError`` / + ``ConnectError`` / ``ReadError`` is what tells us WHY the stream + died. Walks ``__cause__`` then ``__context__`` (deduped, max 4 + deep) to surface the chain in one line. + """ + seen: List[BaseException] = [] + link: Optional[BaseException] = error + while link is not None and len(seen) < 4: + if link in seen: + break + seen.append(link) + nxt = getattr(link, "__cause__", None) or getattr( + link, "__context__", None + ) + if nxt is None or nxt is link: + break + link = nxt + parts: List[str] = [] + for e in seen: + msg = str(e).strip().replace("\n", " ") + if len(msg) > 140: + msg = msg[:140] + "…" + parts.append(f"{type(e).__name__}({msg})" if msg else type(e).__name__) + return " <- ".join(parts) if parts else type(error).__name__ + + +def log_stream_retry( + agent: Any, + *, + kind: str, + error: BaseException, + attempt: int, + max_attempts: int, + mid_tool_call: bool, + diag: Optional[Dict[str, Any]] = None, +) -> None: + """Record a transient stream-drop and retry to ``agent.log``. + + Always logs a structured WARNING so users have a breadcrumb regardless + of UI verbosity. Subagents in particular benefit because their + retries no longer spam the parent's terminal — but the file log keeps + full detail (provider, error class, attempt, base_url, subagent_id). + + When *diag* is provided (the per-attempt stream-diagnostic dict from + :func:`stream_diag_init`), the WARNING also captures upstream headers + (cf-ray, x-openrouter-provider, x-openrouter-id), HTTP status, bytes + streamed before the drop, and elapsed time on the dying attempt. + These are the breadcrumbs needed to answer "is one CF edge / one + downstream provider responsible, or is it random across runs?" + """ + try: + try: + _summary = agent._summarize_api_error(error) + except Exception: + _summary = str(error) + if _summary and len(_summary) > 240: + _summary = _summary[:240] + "…" + + # Inner-cause chain (httpx errors hide under openai.APIError). + try: + _chain = flatten_exception_chain(error) + except Exception: + _chain = type(error).__name__ + + # Per-attempt counters and upstream headers. + _now = time.time() + _bytes = 0 + _chunks = 0 + _elapsed = 0.0 + _ttfb = None + _headers_repr = "-" + _http_status = "-" + if isinstance(diag, dict): + try: + _bytes = int(diag.get("bytes") or 0) + _chunks = int(diag.get("chunks") or 0) + _started = float(diag.get("started_at") or _now) + _elapsed = max(0.0, _now - _started) + _first = diag.get("first_chunk_at") + if _first is not None: + _ttfb = max(0.0, float(_first) - _started) + headers = diag.get("headers") or {} + if isinstance(headers, dict) and headers: + _headers_repr = " ".join( + f"{k}={v}" for k, v in headers.items() + ) + if diag.get("http_status") is not None: + _http_status = str(diag.get("http_status")) + except Exception: + pass + + logger.warning( + "Stream %s on attempt %s/%s — retrying. " + "subagent_id=%s depth=%s provider=%s base_url=%s " + "error_type=%s error=%s " + "chain=%s " + "http_status=%s bytes=%d chunks=%d elapsed=%.2fs ttfb=%s " + "upstream=[%s]", + kind, + attempt, + max_attempts, + getattr(agent, "_subagent_id", None) or "-", + getattr(agent, "_delegate_depth", 0), + agent.provider or "-", + agent.base_url or "-", + type(error).__name__, + _summary, + _chain, + _http_status, + _bytes, + _chunks, + _elapsed, + f"{_ttfb:.2f}s" if _ttfb is not None else "-", + _headers_repr, + extra={"mid_tool_call": mid_tool_call}, + ) + except Exception: + logger.debug("stream-retry log emit failed", exc_info=True) + + +def emit_stream_drop( + agent: Any, + *, + error: BaseException, + attempt: int, + max_attempts: int, + mid_tool_call: bool, + diag: Optional[Dict[str, Any]] = None, +) -> None: + """Emit a single user-visible line for a stream drop+retry. + + Both top-level agents and subagents announce drops in the UI — the + parent prefixes subagent lines with ``[subagent-N]`` via ``log_prefix`` + so they're easy to attribute. All cases also write a structured + WARNING to ``agent.log`` via :func:`log_stream_retry` with the full + diagnostic detail (subagent_id, provider, base_url, error_type, + cf-ray, x-openrouter-provider, bytes/chunks, elapsed) for post-hoc + analysis. + + The user-visible status line is intentionally compact: provider, + error class, attempt N/M, plus ``after Xs`` when the stream dropped + mid-flight. Full diagnostic detail goes to ``agent.log`` only — + ``hermes logs --level WARNING | grep "Stream drop"`` to inspect. + """ + kind = "drop mid tool-call" if mid_tool_call else "drop" + log_stream_retry( + agent, + kind=kind, + error=error, + attempt=attempt, + max_attempts=max_attempts, + mid_tool_call=mid_tool_call, + diag=diag, + ) + provider = agent.provider or "provider" + # Compose a brief "after Xs" suffix when we have timing data — helps + # the user distinguish "couldn't connect" (0s) from "died after 30s + # of streaming" (likely upstream idle-kill or proxy timeout). + _suffix = "" + if isinstance(diag, dict): + try: + started = diag.get("started_at") + if started is not None: + _suffix = f" after {max(0.0, time.time() - float(started)):.1f}s" + except Exception: + pass + try: + agent._emit_status( + f"⚠️ {provider} stream {kind} ({type(error).__name__}){_suffix} " + f"— reconnecting, retry {attempt}/{max_attempts}" + ) + agent._touch_activity( + f"stream retry {attempt}/{max_attempts} " + f"after {type(error).__name__}" + ) + except Exception: + pass + + +__all__ = [ + "STREAM_DIAG_HEADERS", + "stream_diag_init", + "stream_diag_capture_response", + "flatten_exception_chain", + "log_stream_retry", + "emit_stream_drop", +] diff --git a/run_agent.py b/run_agent.py index b5ea98d911d..234a322a480 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2110,99 +2110,28 @@ class AIAgent: except Exception: logger.debug("status_callback error in _emit_warning", exc_info=True) - # Headers we capture from the dying stream's HTTP response so post-mortem - # diagnosis can answer "which CF edge / which OpenRouter downstream - # provider / which request id". Lowercased; httpx returns CIMultiDict. - _STREAM_DIAG_HEADERS = ( - "cf-ray", - "cf-cache-status", - "x-openrouter-provider", - "x-openrouter-model", - "x-openrouter-id", - "x-request-id", - "x-vercel-id", - "via", - "server", - "x-forwarded-for", - ) + # Stream-diagnostic class header preserved for backward compat — + # actual list lives in ``agent.stream_diag.STREAM_DIAG_HEADERS``. + from agent.stream_diag import STREAM_DIAG_HEADERS as _STREAM_DIAG_HEADERS # noqa: E402 @staticmethod def _stream_diag_init() -> Dict[str, Any]: - """Return a fresh per-attempt diagnostic dict. - - Mutated in-place by the streaming functions and read from the retry - block when a stream dies. Lives on ``request_client_holder`` so it - survives across the closure boundary. - """ - return { - "started_at": time.time(), - "first_chunk_at": None, - "chunks": 0, - "bytes": 0, - "headers": {}, - "http_status": None, - } + """Forwarder — see ``agent.stream_diag.stream_diag_init``.""" + from agent.stream_diag import stream_diag_init + return stream_diag_init() def _stream_diag_capture_response( self, diag: Dict[str, Any], http_response: Any ) -> None: - """Snapshot interesting headers + HTTP status from the live stream. - - Called once at stream open (before iterating chunks) so the metadata - survives even if the stream dies before any chunk arrives. Failures - are swallowed — diag is best-effort. - """ - if http_response is None or not isinstance(diag, dict): - return - try: - diag["http_status"] = getattr(http_response, "status_code", None) - except Exception: - pass - try: - headers = getattr(http_response, "headers", None) or {} - captured: Dict[str, str] = {} - for name in self._STREAM_DIAG_HEADERS: - try: - val = headers.get(name) - if val: - # Truncate single-value to keep log lines bounded. - captured[name] = str(val)[:120] - except Exception: - continue - diag["headers"] = captured - except Exception: - pass + """Forwarder — see ``agent.stream_diag.stream_diag_capture_response``.""" + from agent.stream_diag import stream_diag_capture_response + stream_diag_capture_response(self, diag, http_response) @staticmethod def _flatten_exception_chain(error: BaseException) -> str: - """Return a compact ``Outer(msg) <- Inner(msg) <- ...`` rendering. - - OpenAI SDK wraps httpx errors as ``APIConnectionError`` / - ``APIError`` and only the wrapper's class is visible at the catch - site — but the underlying ``RemoteProtocolError`` / - ``ConnectError`` / ``ReadError`` is what tells us WHY the stream - died. Walks ``__cause__`` then ``__context__`` (deduped, max 4 - deep) to surface the chain in one line. - """ - seen: List[BaseException] = [] - link: Optional[BaseException] = error - while link is not None and len(seen) < 4: - if link in seen: - break - seen.append(link) - nxt = getattr(link, "__cause__", None) or getattr( - link, "__context__", None - ) - if nxt is None or nxt is link: - break - link = nxt - parts: List[str] = [] - for e in seen: - msg = str(e).strip().replace("\n", " ") - if len(msg) > 140: - msg = msg[:140] + "…" - parts.append(f"{type(e).__name__}({msg})" if msg else type(e).__name__) - return " <- ".join(parts) if parts else type(error).__name__ + """Forwarder — see ``agent.stream_diag.flatten_exception_chain``.""" + from agent.stream_diag import flatten_exception_chain + return flatten_exception_chain(error) def _log_stream_retry( self, @@ -2214,88 +2143,12 @@ class AIAgent: mid_tool_call: bool, diag: Optional[Dict[str, Any]] = None, ) -> None: - """Record a transient stream-drop and retry to ``agent.log``. - - Always logs a structured WARNING so users have a breadcrumb regardless - of UI verbosity. Subagents in particular benefit because their - retries no longer spam the parent's terminal — but the file log keeps - full detail (provider, error class, attempt, base_url, subagent_id). - - When *diag* is provided (the per-attempt stream-diagnostic dict from - ``_stream_diag_init``), the WARNING also captures upstream headers - (cf-ray, x-openrouter-provider, x-openrouter-id), HTTP status, bytes - streamed before the drop, and elapsed time on the dying attempt. - These are the breadcrumbs needed to answer "is one CF edge / one - downstream provider responsible, or is it random across runs?" - """ - try: - try: - _summary = self._summarize_api_error(error) - except Exception: - _summary = str(error) - if _summary and len(_summary) > 240: - _summary = _summary[:240] + "…" - - # Inner-cause chain (httpx errors hide under openai.APIError). - try: - _chain = self._flatten_exception_chain(error) - except Exception: - _chain = type(error).__name__ - - # Per-attempt counters and upstream headers. - _now = time.time() - _bytes = 0 - _chunks = 0 - _elapsed = 0.0 - _ttfb = None - _headers_repr = "-" - _http_status = "-" - if isinstance(diag, dict): - try: - _bytes = int(diag.get("bytes") or 0) - _chunks = int(diag.get("chunks") or 0) - _started = float(diag.get("started_at") or _now) - _elapsed = max(0.0, _now - _started) - _first = diag.get("first_chunk_at") - if _first is not None: - _ttfb = max(0.0, float(_first) - _started) - headers = diag.get("headers") or {} - if isinstance(headers, dict) and headers: - _headers_repr = " ".join( - f"{k}={v}" for k, v in headers.items() - ) - if diag.get("http_status") is not None: - _http_status = str(diag.get("http_status")) - except Exception: - pass - - logger.warning( - "Stream %s on attempt %s/%s — retrying. " - "subagent_id=%s depth=%s provider=%s base_url=%s " - "error_type=%s error=%s " - "chain=%s " - "http_status=%s bytes=%d chunks=%d elapsed=%.2fs ttfb=%s " - "upstream=[%s]", - kind, - attempt, - max_attempts, - getattr(self, "_subagent_id", None) or "-", - getattr(self, "_delegate_depth", 0), - self.provider or "-", - self.base_url or "-", - type(error).__name__, - _summary, - _chain, - _http_status, - _bytes, - _chunks, - _elapsed, - f"{_ttfb:.2f}s" if _ttfb is not None else "-", - _headers_repr, - extra={"mid_tool_call": mid_tool_call}, - ) - except Exception: - logger.debug("stream-retry log emit failed", exc_info=True) + """Forwarder — see ``agent.stream_diag.log_stream_retry``.""" + from agent.stream_diag import log_stream_retry + log_stream_retry( + self, kind=kind, error=error, attempt=attempt, + max_attempts=max_attempts, mid_tool_call=mid_tool_call, diag=diag, + ) def _emit_stream_drop( self, @@ -2306,53 +2159,12 @@ class AIAgent: mid_tool_call: bool, diag: Optional[Dict[str, Any]] = None, ) -> None: - """Emit a single user-visible line for a stream drop+retry. - - Both top-level agents and subagents announce drops in the UI — the - parent prefixes subagent lines with ``[subagent-N]`` via ``log_prefix`` - so they're easy to attribute. All cases also write a structured - WARNING to ``agent.log`` via :meth:`_log_stream_retry` with the full - diagnostic detail (subagent_id, provider, base_url, error_type, - cf-ray, x-openrouter-provider, bytes/chunks, elapsed) for post-hoc - analysis. - - The user-visible status line is intentionally compact: provider, - error class, attempt N/M, plus ``after Xs`` when the stream dropped - mid-flight. Full diagnostic detail goes to ``agent.log`` only — - ``hermes logs --level WARNING | grep "Stream drop"`` to inspect. - """ - kind = "drop mid tool-call" if mid_tool_call else "drop" - self._log_stream_retry( - kind=kind, - error=error, - attempt=attempt, - max_attempts=max_attempts, - mid_tool_call=mid_tool_call, - diag=diag, + """Forwarder — see ``agent.stream_diag.emit_stream_drop``.""" + from agent.stream_diag import emit_stream_drop + emit_stream_drop( + self, error=error, attempt=attempt, max_attempts=max_attempts, + mid_tool_call=mid_tool_call, diag=diag, ) - provider = self.provider or "provider" - # Compose a brief "after Xs" suffix when we have timing data — helps - # the user distinguish "couldn't connect" (0s) from "died after 30s - # of streaming" (likely upstream idle-kill or proxy timeout). - _suffix = "" - if isinstance(diag, dict): - try: - started = diag.get("started_at") - if started is not None: - _suffix = f" after {max(0.0, time.time() - float(started)):.1f}s" - except Exception: - pass - try: - self._emit_status( - f"⚠️ {provider} stream {kind} ({type(error).__name__}){_suffix} " - f"— reconnecting, retry {attempt}/{max_attempts}" - ) - self._touch_activity( - f"stream retry {attempt}/{max_attempts} " - f"after {type(error).__name__}" - ) - except Exception: - pass def _emit_auxiliary_failure(self, task: str, exc: BaseException) -> None: """Surface a compact warning for failed auxiliary work.""" From 4b25619bc4770396faf206429ddc180ad02231a9 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:41:44 -0700 Subject: [PATCH 009/142] refactor(run_agent): extract chat-completion helpers to agent/chat_completion_helpers.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six methods move into a new module — bodies live there, AIAgent keeps thin forwarder methods so call sites and tests are unchanged. * interruptible_api_call — non-streaming API call with interrupt handling * build_api_kwargs — assemble OpenAI / Anthropic / Codex / Bedrock request kwargs * build_assistant_message — normalize assistant message dict (reasoning, tool_calls, codex passthrough fields, alibaba glm-4.7 quirk) * try_activate_fallback — provider fallback chain activation * handle_max_iterations — controlled stop when iteration budget exhausts * cleanup_task_resources — per-turn VM + browser teardown (skipped for persistent environments) Names tests patch on run_agent (cleanup_vm, cleanup_browser) are routed through _ra() so the patch surface is preserved. Two TestAnthropicInterruptHandler source-introspection tests were updated to scan agent.chat_completion_helpers.interruptible_api_call instead of AIAgent._interruptible_api_call — the body lives in the extracted module now. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 13282 -> 12253 lines (-1029). --- agent/chat_completion_helpers.py | 1132 +++++++++++++++++++++++++++++ run_agent.py | 1041 +------------------------- tests/run_agent/test_run_agent.py | 10 +- 3 files changed, 1156 insertions(+), 1027 deletions(-) create mode 100644 agent/chat_completion_helpers.py diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py new file mode 100644 index 00000000000..fd1eb22ae4e --- /dev/null +++ b/agent/chat_completion_helpers.py @@ -0,0 +1,1132 @@ +"""Helper functions for the chat-completions code path. + +Extracted from :class:`AIAgent` for cleanliness — bodies of the +non-streaming API call, request kwargs builder, assistant-message +materializer, provider-fallback activator, max-iterations handler, +and per-turn resource cleanup. + +Each function takes the parent ``AIAgent`` as its first argument +(``agent``). :class:`AIAgent` keeps thin forwarder methods so call +sites unchanged. Symbols that tests patch on ``run_agent`` (e.g. +``cleanup_vm`` / ``cleanup_browser`` in +``test_zombie_process_cleanup.py``) are resolved through +:func:`_ra` so the patch contract is preserved. +""" + +from __future__ import annotations + +import concurrent.futures +import contextvars +import copy +import json +import logging +import os +import random +import re +import sys +import threading +import time +import uuid +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse, parse_qs, urlunparse + +from hermes_cli.timeouts import get_provider_request_timeout +from agent.error_classifier import classify_api_error, FailoverReason +from agent.message_sanitization import ( + _sanitize_surrogates, + _sanitize_messages_surrogates, + _sanitize_structure_surrogates, + _sanitize_messages_non_ascii, + _sanitize_tools_non_ascii, + _sanitize_structure_non_ascii, + _strip_images_from_messages, + _strip_non_ascii, + _repair_tool_call_arguments, + _escape_invalid_chars_in_json_strings, +) +from agent.tool_dispatch_helpers import ( + _is_multimodal_tool_result, + _multimodal_text_summary, +) +from agent.retry_utils import jittered_backoff +from agent.tool_guardrails import ( + ToolGuardrailDecision, + append_toolguard_guidance, + toolguard_synthetic_result, +) +from tools.terminal_tool import is_persistent_env +from utils import base_url_host_matches, base_url_hostname + +logger = logging.getLogger(__name__) + + +def _ra(): + """Lazy ``run_agent`` reference. + + Used to honor test patches like + ``patch("run_agent.cleanup_vm")`` / ``patch("run_agent.cleanup_browser")`` + that target symbols imported into ``run_agent``'s namespace. + """ + import run_agent + return run_agent + + + +def interruptible_api_call(agent, api_kwargs: dict): + """ + Run the API call in a background thread so the main conversation loop + can detect interrupts without waiting for the full HTTP round-trip. + + Each worker thread gets its own OpenAI client instance. Interrupts only + close that worker-local client, so retries and other requests never + inherit a closed transport. + + Includes a stale-call detector: if no response arrives within the + configured timeout, the connection is killed and an error raised so + the main retry loop can try again with backoff / credential rotation / + provider fallback. + """ + result = {"response": None, "error": None} + request_client_holder = {"client": None} + + def _call(): + try: + if agent.api_mode == "codex_responses": + request_client_holder["client"] = agent._create_request_openai_client( + reason="codex_stream_request", + api_kwargs=api_kwargs, + ) + result["response"] = agent._run_codex_stream( + api_kwargs, + client=request_client_holder["client"], + on_first_delta=getattr(agent, "_codex_on_first_delta", None), + ) + elif agent.api_mode == "anthropic_messages": + result["response"] = agent._anthropic_messages_create(api_kwargs) + elif agent.api_mode == "bedrock_converse": + # Bedrock uses boto3 directly — no OpenAI client needed. + # normalize_converse_response produces an OpenAI-compatible + # SimpleNamespace so the rest of the agent loop can treat + # bedrock responses like chat_completions responses. + from agent.bedrock_adapter import ( + _get_bedrock_runtime_client, + invalidate_runtime_client, + is_stale_connection_error, + normalize_converse_response, + ) + region = api_kwargs.pop("__bedrock_region__", "us-east-1") + api_kwargs.pop("__bedrock_converse__", None) + client = _get_bedrock_runtime_client(region) + try: + raw_response = client.converse(**api_kwargs) + except Exception as _bedrock_exc: + # Evict the cached client on stale-connection failures + # so the outer retry loop builds a fresh client/pool. + if is_stale_connection_error(_bedrock_exc): + invalidate_runtime_client(region) + raise + result["response"] = normalize_converse_response(raw_response) + else: + request_client_holder["client"] = agent._create_request_openai_client( + reason="chat_completion_request", + api_kwargs=api_kwargs, + ) + result["response"] = request_client_holder["client"].chat.completions.create(**api_kwargs) + except Exception as e: + result["error"] = e + finally: + request_client = request_client_holder.get("client") + if request_client is not None: + agent._close_request_openai_client(request_client, reason="request_complete") + + # ── Stale-call timeout (mirrors streaming stale detector) ──────── + # Non-streaming calls return nothing until the full response is + # ready. Without this, a hung provider can block for the full + # httpx timeout (default 1800s) with zero feedback. The stale + # detector kills the connection early so the main retry loop can + # apply richer recovery (credential rotation, provider fallback). + _stale_timeout = agent._compute_non_stream_stale_timeout( + api_kwargs.get("messages", []) + ) + + _call_start = time.time() + agent._touch_activity("waiting for non-streaming API response") + + t = threading.Thread(target=_call, daemon=True) + t.start() + _poll_count = 0 + while t.is_alive(): + t.join(timeout=0.3) + _poll_count += 1 + + # Touch activity every ~30s so the gateway's inactivity + # monitor knows we're alive while waiting for the response. + if _poll_count % 100 == 0: # 100 × 0.3s = 30s + _elapsed = time.time() - _call_start + agent._touch_activity( + f"waiting for non-streaming response ({int(_elapsed)}s elapsed)" + ) + + # Stale-call detector: kill the connection if no response + # arrives within the configured timeout. + _elapsed = time.time() - _call_start + if _elapsed > _stale_timeout: + _est_ctx = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 + logger.warning( + "Non-streaming API call stale for %.0fs (threshold %.0fs). " + "model=%s context=~%s tokens. Killing connection.", + _elapsed, _stale_timeout, + api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", + ) + agent._emit_status( + f"⚠️ No response from provider for {int(_elapsed)}s " + f"(non-streaming, model: {api_kwargs.get('model', 'unknown')}). " + f"Aborting call." + ) + try: + if agent.api_mode == "anthropic_messages": + agent._anthropic_client.close() + agent._rebuild_anthropic_client() + else: + rc = request_client_holder.get("client") + if rc is not None: + agent._close_request_openai_client(rc, reason="stale_call_kill") + except Exception: + pass + agent._touch_activity( + f"stale non-streaming call killed after {int(_elapsed)}s" + ) + # Wait briefly for the thread to notice the closed connection. + t.join(timeout=2.0) + if result["error"] is None and result["response"] is None: + result["error"] = TimeoutError( + f"Non-streaming API call timed out after {int(_elapsed)}s " + f"with no response (threshold: {int(_stale_timeout)}s)" + ) + break + + if agent._interrupt_requested: + # Force-close the in-flight worker-local HTTP connection to stop + # token generation without poisoning the shared client used to + # seed future retries. + try: + if agent.api_mode == "anthropic_messages": + agent._anthropic_client.close() + agent._rebuild_anthropic_client() + else: + request_client = request_client_holder.get("client") + if request_client is not None: + agent._close_request_openai_client(request_client, reason="interrupt_abort") + except Exception: + pass + raise InterruptedError("Agent interrupted during API call") + if result["error"] is not None: + raise result["error"] + return result["response"] + + + +def build_api_kwargs(agent, api_messages: list) -> dict: + """Build the keyword arguments dict for the active API mode.""" + tools_for_api = agent.tools + + if agent.api_mode == "anthropic_messages": + _transport = agent._get_transport() + anthropic_messages = agent._prepare_anthropic_messages_for_api(api_messages) + ctx_len = getattr(agent, "context_compressor", None) + ctx_len = ctx_len.context_length if ctx_len else None + ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None) + if ephemeral_out is not None: + agent._ephemeral_max_output_tokens = None # consume immediately + return _transport.build_kwargs( + model=agent.model, + messages=anthropic_messages, + tools=tools_for_api, + max_tokens=ephemeral_out if ephemeral_out is not None else agent.max_tokens, + reasoning_config=agent.reasoning_config, + is_oauth=agent._is_anthropic_oauth, + preserve_dots=agent._anthropic_preserve_dots(), + context_length=ctx_len, + base_url=getattr(agent, "_anthropic_base_url", None), + fast_mode=(agent.request_overrides or {}).get("speed") == "fast", + drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)), + ) + + # AWS Bedrock native Converse API — bypasses the OpenAI client entirely. + # The adapter handles message/tool conversion and boto3 calls directly. + if agent.api_mode == "bedrock_converse": + _bt = agent._get_transport() + region = getattr(agent, "_bedrock_region", None) or "us-east-1" + guardrail = getattr(agent, "_bedrock_guardrail_config", None) + return _bt.build_kwargs( + model=agent.model, + messages=api_messages, + tools=tools_for_api, + max_tokens=agent.max_tokens or 4096, + region=region, + guardrail_config=guardrail, + ) + + if agent.api_mode == "codex_responses": + _ct = agent._get_transport() + is_github_responses = ( + base_url_host_matches(agent.base_url, "models.github.ai") + or base_url_host_matches(agent.base_url, "api.githubcopilot.com") + ) + is_codex_backend = ( + agent.provider == "openai-codex" + or ( + agent._base_url_hostname == "chatgpt.com" + and "/backend-api/codex" in agent._base_url_lower + ) + ) + is_xai_responses = agent.provider == "xai" or agent._base_url_hostname == "api.x.ai" + _msgs_for_codex = agent._prepare_messages_for_non_vision_model(api_messages) + return _ct.build_kwargs( + model=agent.model, + messages=_msgs_for_codex, + tools=tools_for_api, + reasoning_config=agent.reasoning_config, + session_id=getattr(agent, "session_id", None), + max_tokens=agent.max_tokens, + request_overrides=agent.request_overrides, + is_github_responses=is_github_responses, + is_codex_backend=is_codex_backend, + is_xai_responses=is_xai_responses, + github_reasoning_extra=agent._github_models_reasoning_extra_body() if is_github_responses else None, + ) + + # ── chat_completions (default) ───────────────────────────────────── + _ct = agent._get_transport() + + # Provider detection flags + _is_qwen = agent._is_qwen_portal() + _is_or = agent._is_openrouter_url() + _is_gh = ( + base_url_host_matches(agent._base_url_lower, "models.github.ai") + or base_url_host_matches(agent._base_url_lower, "api.githubcopilot.com") + ) + _is_nous = "nousresearch" in agent._base_url_lower + _is_nvidia = "integrate.api.nvidia.com" in agent._base_url_lower + _is_kimi = ( + base_url_host_matches(agent.base_url, "api.kimi.com") + or base_url_host_matches(agent.base_url, "moonshot.ai") + or base_url_host_matches(agent.base_url, "moonshot.cn") + ) + _is_tokenhub = base_url_host_matches(agent._base_url_lower, "tokenhub.tencentmaas.com") + _is_lmstudio = (agent.provider or "").strip().lower() == "lmstudio" + + # Temperature: _fixed_temperature_for_model may return OMIT_TEMPERATURE + # sentinel (temperature omitted entirely), a numeric override, or None. + try: + from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE + _ft = _fixed_temperature_for_model(agent.model, agent.base_url) + _omit_temp = _ft is OMIT_TEMPERATURE + _fixed_temp = _ft if not _omit_temp else None + except Exception: + _omit_temp = False + _fixed_temp = None + + # Provider preferences (OpenRouter-style) + _prefs: Dict[str, Any] = {} + if agent.providers_allowed: + _prefs["only"] = agent.providers_allowed + if agent.providers_ignored: + _prefs["ignore"] = agent.providers_ignored + if agent.providers_order: + _prefs["order"] = agent.providers_order + if agent.provider_sort: + _prefs["sort"] = agent.provider_sort + if agent.provider_require_parameters: + _prefs["require_parameters"] = True + if agent.provider_data_collection: + _prefs["data_collection"] = agent.provider_data_collection + + # Claude max-output override on aggregators + _ant_max = None + if (_is_or or _is_nous) and "claude" in (agent.model or "").lower(): + try: + from agent.anthropic_adapter import _get_anthropic_max_output + _ant_max = _get_anthropic_max_output(agent.model) + except Exception: + pass + + # Qwen session metadata + _qwen_meta = None + if _is_qwen: + _qwen_meta = { + "sessionId": agent.session_id or "hermes", + "promptId": str(uuid.uuid4()), + } + + # ── Provider profile path (registered providers) ─────────────────── + # Profiles handle per-provider quirks via hooks. When a profile is + # found, delegate fully; otherwise fall through to the legacy flag path. + try: + from providers import get_provider_profile + _profile = get_provider_profile(agent.provider) + except Exception: + _profile = None + + if _profile: + _ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None) + if _ephemeral_out is not None: + agent._ephemeral_max_output_tokens = None + + return _ct.build_kwargs( + model=agent.model, + messages=api_messages, + tools=tools_for_api, + base_url=agent.base_url, + timeout=agent._resolved_api_call_timeout(), + max_tokens=agent.max_tokens, + ephemeral_max_output_tokens=_ephemeral_out, + max_tokens_param_fn=agent._max_tokens_param, + reasoning_config=agent.reasoning_config, + request_overrides=agent.request_overrides, + session_id=getattr(agent, "session_id", None), + provider_profile=_profile, + ollama_num_ctx=agent._ollama_num_ctx, + # Context forwarded to profile hooks: + provider_preferences=_prefs or None, + openrouter_min_coding_score=agent.openrouter_min_coding_score, + anthropic_max_output=_ant_max, + supports_reasoning=agent._supports_reasoning_extra_body(), + qwen_session_metadata=_qwen_meta, + ) + + # ── Legacy flag path ──────────────────────────────────────────── + # Reached only when get_provider_profile() returns None — i.e. a + # completely unknown provider not in providers/ registry. + _ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None) + if _ephemeral_out is not None: + agent._ephemeral_max_output_tokens = None + + # Strip image parts for non-vision models (no-op when vision-capable). + _msgs_for_chat = agent._prepare_messages_for_non_vision_model(api_messages) + + return _ct.build_kwargs( + model=agent.model, + messages=_msgs_for_chat, + tools=tools_for_api, + base_url=agent.base_url, + timeout=agent._resolved_api_call_timeout(), + max_tokens=agent.max_tokens, + ephemeral_max_output_tokens=_ephemeral_out, + max_tokens_param_fn=agent._max_tokens_param, + reasoning_config=agent.reasoning_config, + request_overrides=agent.request_overrides, + session_id=getattr(agent, "session_id", None), + model_lower=(agent.model or "").lower(), + is_openrouter=_is_or, + is_nous=_is_nous, + is_qwen_portal=_is_qwen, + is_github_models=_is_gh, + is_nvidia_nim=_is_nvidia, + is_kimi=_is_kimi, + is_tokenhub=_is_tokenhub, + is_lmstudio=_is_lmstudio, + is_custom_provider=agent.provider == "custom", + ollama_num_ctx=agent._ollama_num_ctx, + provider_preferences=_prefs or None, + openrouter_min_coding_score=agent.openrouter_min_coding_score, + qwen_prepare_fn=agent._qwen_prepare_chat_messages if _is_qwen else None, + qwen_prepare_inplace_fn=agent._qwen_prepare_chat_messages_inplace if _is_qwen else None, + qwen_session_metadata=_qwen_meta, + fixed_temperature=_fixed_temp, + omit_temperature=_omit_temp, + supports_reasoning=agent._supports_reasoning_extra_body(), + github_reasoning_extra=agent._github_models_reasoning_extra_body() if _is_gh else None, + lmstudio_reasoning_options=agent._lmstudio_reasoning_options_cached() if _is_lmstudio else None, + anthropic_max_output=_ant_max, + provider_name=agent.provider, + ) + + + +def build_assistant_message(agent, assistant_message, finish_reason: str) -> dict: + """Build a normalized assistant message dict from an API response message. + + Handles reasoning extraction, reasoning_details, and optional tool_calls + so both the tool-call path and the final-response path share one builder. + """ + assistant_tool_calls = getattr(assistant_message, "tool_calls", None) + reasoning_text = agent._extract_reasoning(assistant_message) + _from_structured = bool(reasoning_text) + + # Fallback: extract inline blocks from content when no structured + # reasoning fields are present (some models/providers embed thinking + # directly in the content rather than returning separate API fields). + if not reasoning_text: + content = assistant_message.content or "" + think_blocks = re.findall(r'(.*?)', content, flags=re.DOTALL) + if think_blocks: + combined = "\n\n".join(b.strip() for b in think_blocks if b.strip()) + reasoning_text = combined or None + + if reasoning_text and agent.verbose_logging: + logging.debug(f"Captured reasoning ({len(reasoning_text)} chars): {reasoning_text}") + + if reasoning_text and agent.reasoning_callback: + # Skip callback when streaming is active — reasoning was already + # displayed during the stream via one of two paths: + # (a) _fire_reasoning_delta (structured reasoning_content deltas) + # (b) _stream_delta tag extraction (/) + # When streaming is NOT active, always fire so non-streaming modes + # (gateway, batch, quiet) still get reasoning. + # Any reasoning that wasn't shown during streaming is caught by the + # CLI post-response display fallback (cli.py _reasoning_shown_this_turn). + if not agent.stream_delta_callback and not agent._stream_callback: + try: + agent.reasoning_callback(reasoning_text) + except Exception: + pass + + # Sanitize surrogates from API response — some models (e.g. Kimi/GLM via Ollama) + # can return invalid surrogate code points that crash json.dumps() on persist. + _raw_content = assistant_message.content or "" + _san_content = _sanitize_surrogates(_raw_content) + if reasoning_text: + reasoning_text = _sanitize_surrogates(reasoning_text) + + # Strip inline reasoning tags (… etc.) from the stored + # assistant content. Reasoning was already captured into + # ``reasoning_text`` above (either from structured fields or the + # inline-block fallback), so the raw tags in content are redundant. + # Leaving them in place caused reasoning to leak to messaging + # platforms (#8878, #9568), inflate context on subsequent turns + # (#9306 observed 16% content-size reduction on a real MiniMax + # session), and pollute generated session titles. One strip at the + # storage boundary cleans content for every downstream consumer: + # API replay, session transcript, gateway delivery, CLI display, + # compression, title generation. + if isinstance(_san_content, str) and _san_content: + _san_content = agent._strip_think_blocks(_san_content).strip() + + msg = { + "role": "assistant", + "content": _san_content, + "reasoning": reasoning_text, + "finish_reason": finish_reason, + } + + raw_reasoning_content = getattr(assistant_message, "reasoning_content", None) + if raw_reasoning_content is None and hasattr(assistant_message, "model_extra"): + model_extra = getattr(assistant_message, "model_extra", None) or {} + if isinstance(model_extra, dict) and "reasoning_content" in model_extra: + raw_reasoning_content = model_extra["reasoning_content"] + if raw_reasoning_content is not None: + msg["reasoning_content"] = _sanitize_surrogates(raw_reasoning_content) + elif assistant_tool_calls and agent._needs_thinking_reasoning_pad(): + # DeepSeek v4 thinking mode and Kimi / Moonshot thinking mode + # both require reasoning_content on every assistant tool-call + # message. Without it, replaying the persisted message causes + # HTTP 400 ("The reasoning_content in the thinking mode must + # be passed back to the API"). Include streamed reasoning + # text when captured; otherwise pad with a single space — + # DeepSeek V4 Pro tightened validation and rejects empty + # string ("The reasoning content in the thinking mode must + # be passed back to the API"). A space satisfies non-empty + # checks everywhere without leaking fabricated reasoning. + # Refs #15250, #17400, #17341. + msg["reasoning_content"] = reasoning_text or " " + + # Additive fallback (refs #16844, #16884). Streaming-only providers + # (glm, MiniMax, gpt-5.x via aigw, Anthropic via openai-compat shims) + # accumulate reasoning through ``delta.reasoning_content`` chunks + # but never land it on the message object as a top-level attribute, + # so neither branch above fires and the chain-of-thought is stored + # only under the internal ``reasoning`` key. When the user later + # replays that history through a DeepSeek-v4 / Kimi thinking model, + # the missing ``reasoning_content`` causes HTTP 400 ("The + # reasoning_content in the thinking mode must be passed back to the + # API."). + # + # Promote the already-sanitized streamed ``reasoning_text`` to + # ``reasoning_content`` at write time, but ONLY when no prior branch + # already set it AND we actually captured reasoning text. This + # preserves every existing behavior: + # - SDK-exposed ``reasoning_content`` (OpenAI/Moonshot/DeepSeek SDK) + # still wins. + # - DeepSeek tool-call ""-pad (#15250) still fires. + # - Non-thinking turns with no reasoning leave the field absent, + # so ``_copy_reasoning_content_for_api``'s cross-provider leak + # guard (#15748) and ``reasoning``→``reasoning_content`` + # promotion tiers still apply at replay time. + if "reasoning_content" not in msg and reasoning_text: + msg["reasoning_content"] = reasoning_text + + if hasattr(assistant_message, 'reasoning_details') and assistant_message.reasoning_details: + # Pass reasoning_details back unmodified so providers (OpenRouter, + # Anthropic, OpenAI) can maintain reasoning continuity across turns. + # Each provider may include opaque fields (signature, encrypted_content) + # that must be preserved exactly. + raw_details = assistant_message.reasoning_details + preserved = [] + for d in raw_details: + if isinstance(d, dict): + preserved.append(d) + elif hasattr(d, "__dict__"): + preserved.append(d.__dict__) + elif hasattr(d, "model_dump"): + preserved.append(d.model_dump()) + if preserved: + msg["reasoning_details"] = preserved + + # Codex Responses API: preserve encrypted reasoning items for + # multi-turn continuity. These get replayed as input on the next turn. + codex_items = getattr(assistant_message, "codex_reasoning_items", None) + if codex_items: + msg["codex_reasoning_items"] = codex_items + + # Codex Responses API: preserve exact assistant message items (with + # id/phase) so follow-up turns can replay structured items instead of + # flattening to plain text. This is required for prefix cache hits. + codex_message_items = getattr(assistant_message, "codex_message_items", None) + if codex_message_items: + msg["codex_message_items"] = codex_message_items + + if assistant_tool_calls: + tool_calls = [] + for tool_call in assistant_tool_calls: + raw_id = getattr(tool_call, "id", None) + call_id = getattr(tool_call, "call_id", None) + if not isinstance(call_id, str) or not call_id.strip(): + embedded_call_id, _ = agent._split_responses_tool_id(raw_id) + call_id = embedded_call_id + if not isinstance(call_id, str) or not call_id.strip(): + if isinstance(raw_id, str) and raw_id.strip(): + call_id = raw_id.strip() + else: + _fn = getattr(tool_call, "function", None) + _fn_name = getattr(_fn, "name", "") if _fn else "" + _fn_args = getattr(_fn, "arguments", "{}") if _fn else "{}" + call_id = agent._deterministic_call_id(_fn_name, _fn_args, len(tool_calls)) + call_id = call_id.strip() + + response_item_id = getattr(tool_call, "response_item_id", None) + if not isinstance(response_item_id, str) or not response_item_id.strip(): + _, embedded_response_item_id = agent._split_responses_tool_id(raw_id) + response_item_id = embedded_response_item_id + + response_item_id = agent._derive_responses_function_call_id( + call_id, + response_item_id if isinstance(response_item_id, str) else None, + ) + + tc_dict = { + "id": call_id, + "call_id": call_id, + "response_item_id": response_item_id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments + }, + } + # Preserve extra_content (e.g. Gemini thought_signature) so it + # is sent back on subsequent API calls. Without this, Gemini 3 + # thinking models reject the request with a 400 error. + extra = getattr(tool_call, "extra_content", None) + if extra is not None: + if hasattr(extra, "model_dump"): + extra = extra.model_dump() + tc_dict["extra_content"] = extra + tool_calls.append(tc_dict) + msg["tool_calls"] = tool_calls + + return msg + + + +def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool: + """Switch to the next fallback model/provider in the chain. + + Called when the current model is failing after retries. Swaps the + OpenAI client, model slug, and provider in-place so the retry loop + can continue with the new backend. Advances through the chain on + each call; returns False when exhausted. + + Uses the centralized provider router (resolve_provider_client) for + auth resolution and client construction — no duplicated provider→key + mappings. + """ + if reason in {FailoverReason.rate_limit, FailoverReason.billing}: + # Only start cooldown when leaving the primary provider. If we're + # already on a fallback and chain-switching, the primary wasn't the + # source of the 429 so the cooldown should not be reset/extended. + fallback_already_active = bool(getattr(agent, "_fallback_activated", False)) + current_provider = (getattr(agent, "provider", "") or "").strip().lower() + primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower() + if (not fallback_already_active) or (primary_provider and current_provider == primary_provider): + agent._rate_limited_until = time.monotonic() + 60 + if agent._fallback_index >= len(agent._fallback_chain): + return False + + fb = agent._fallback_chain[agent._fallback_index] + agent._fallback_index += 1 + fb_provider = (fb.get("provider") or "").strip().lower() + fb_model = (fb.get("model") or "").strip() + if not fb_provider or not fb_model: + return agent._try_activate_fallback() # skip invalid, try next + + # Skip entries that resolve to the current (provider, model) — falling + # back to the same backend that just failed loops the failure. Compare + # base_url too so two distinct custom_providers entries pointing at the + # same shim/proxy URL also dedup. See issue #22548. + current_provider = (getattr(agent, "provider", "") or "").strip().lower() + current_model = (getattr(agent, "model", "") or "").strip() + current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower() + fb_base_url_for_dedup = (fb.get("base_url") or "").strip().rstrip("/").lower() + if fb_provider == current_provider and fb_model == current_model: + logging.warning( + "Fallback skip: chain entry %s/%s matches current provider/model", + fb_provider, fb_model, + ) + return agent._try_activate_fallback() + if ( + fb_base_url_for_dedup + and current_base_url + and fb_base_url_for_dedup == current_base_url + and fb_model == current_model + ): + logging.warning( + "Fallback skip: chain entry base_url %s matches current backend", + fb_base_url_for_dedup, + ) + return agent._try_activate_fallback() + + # Use centralized router for client construction. + # raw_codex=True because the main agent needs direct responses.stream() + # access for Codex providers. + try: + from agent.auxiliary_client import resolve_provider_client + # Pass base_url and api_key from fallback config so custom + # endpoints (e.g. Ollama Cloud) resolve correctly instead of + # falling through to OpenRouter defaults. + fb_base_url_hint = (fb.get("base_url") or "").strip() or None + fb_api_key_hint = (fb.get("api_key") or "").strip() or None + if not fb_api_key_hint: + # key_env and api_key_env are both documented aliases (see + # _normalize_custom_provider_entry in hermes_cli/config.py). + fb_key_env = (fb.get("key_env") or fb.get("api_key_env") or "").strip() + if fb_key_env: + fb_api_key_hint = os.getenv(fb_key_env, "").strip() or None + # For Ollama Cloud endpoints, pull OLLAMA_API_KEY from env + # when no explicit key is in the fallback config. Host match + # (not substring) — see GHSA-76xc-57q6-vm5m. + if fb_base_url_hint and base_url_host_matches(fb_base_url_hint, "ollama.com") and not fb_api_key_hint: + fb_api_key_hint = os.getenv("OLLAMA_API_KEY") or None + fb_client, _resolved_fb_model = resolve_provider_client( + fb_provider, model=fb_model, raw_codex=True, + explicit_base_url=fb_base_url_hint, + explicit_api_key=fb_api_key_hint) + if fb_client is None: + logging.warning( + "Fallback to %s failed: provider not configured", + fb_provider) + return agent._try_activate_fallback() # try next in chain + try: + from hermes_cli.model_normalize import normalize_model_for_provider + + fb_model = normalize_model_for_provider(fb_model, fb_provider) + except Exception: + pass + + # Determine api_mode from provider / base URL / model + fb_api_mode = "chat_completions" + fb_base_url = str(fb_client.base_url) + _fb_is_azure = agent._is_azure_openai_url(fb_base_url) + if fb_provider == "openai-codex": + fb_api_mode = "codex_responses" + elif fb_provider == "anthropic" or fb_base_url.rstrip("/").lower().endswith("/anthropic"): + fb_api_mode = "anthropic_messages" + elif _fb_is_azure: + # Azure OpenAI serves gpt-5.x on /chat/completions — does NOT + # support the Responses API. Stay on chat_completions. + fb_api_mode = "chat_completions" + elif agent._is_direct_openai_url(fb_base_url): + fb_api_mode = "codex_responses" + elif agent._provider_model_requires_responses_api( + fb_model, + provider=fb_provider, + ): + # GPT-5.x models usually need Responses API, but keep + # provider-specific exceptions like Copilot gpt-5-mini on + # chat completions. + fb_api_mode = "codex_responses" + elif fb_provider == "bedrock" or ( + base_url_hostname(fb_base_url).startswith("bedrock-runtime.") + and base_url_host_matches(fb_base_url, "amazonaws.com") + ): + fb_api_mode = "bedrock_converse" + + old_model = agent.model + + # Clear the per-config context_length override so the fallback + # model's actual context window is resolved instead of inheriting + # the stale value from the previous model. See #22387. + agent._config_context_length = None + agent.model = fb_model + agent.provider = fb_provider + agent.base_url = fb_base_url + agent.api_mode = fb_api_mode + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + agent._fallback_activated = True + + # Honor per-provider / per-model request_timeout_seconds for the + # fallback target (same knob the primary client uses). None = use + # SDK default. + _fb_timeout = get_provider_request_timeout(fb_provider, fb_model) + + if fb_api_mode == "anthropic_messages": + # Build native Anthropic client instead of using OpenAI client + from agent.anthropic_adapter import build_anthropic_client, resolve_anthropic_token, _is_oauth_token + effective_key = (fb_client.api_key or resolve_anthropic_token() or "") if fb_provider == "anthropic" else (fb_client.api_key or "") + agent.api_key = effective_key + agent._anthropic_api_key = effective_key + agent._anthropic_base_url = fb_base_url + agent._anthropic_client = build_anthropic_client( + effective_key, agent._anthropic_base_url, timeout=_fb_timeout, + ) + agent._is_anthropic_oauth = _is_oauth_token(effective_key) if fb_provider == "anthropic" else False + agent.client = None + agent._client_kwargs = {} + else: + # Swap OpenAI client and config in-place + agent.api_key = fb_client.api_key + agent.client = fb_client + # Preserve provider-specific headers that + # resolve_provider_client() may have baked into + # fb_client via the default_headers kwarg. The OpenAI + # SDK stores these in _custom_headers. Without this, + # subsequent request-client rebuilds (via + # _create_request_openai_client) drop the headers, + # causing 403s from providers like Kimi Coding that + # require a User-Agent sentinel. + fb_headers = getattr(fb_client, "_custom_headers", None) + if not fb_headers: + fb_headers = getattr(fb_client, "default_headers", None) + agent._client_kwargs = { + "api_key": fb_client.api_key, + "base_url": fb_base_url, + **({"default_headers": dict(fb_headers)} if fb_headers else {}), + } + if _fb_timeout is not None: + agent._client_kwargs["timeout"] = _fb_timeout + # Rebuild the shared OpenAI client so the configured + # timeout takes effect on the very next fallback request, + # not only after a later credential-rotation rebuild. + agent._replace_primary_openai_client(reason="fallback_timeout_apply") + + # Re-evaluate prompt caching for the new provider/model + agent._use_prompt_caching, agent._use_native_cache_layout = ( + agent._anthropic_prompt_cache_policy( + provider=fb_provider, + base_url=fb_base_url, + api_mode=fb_api_mode, + model=fb_model, + ) + ) + + # LM Studio: preload before probing the fallback's context length. + agent._ensure_lmstudio_runtime_loaded() + + # Update context compressor limits for the fallback model. + # Without this, compression decisions use the primary model's + # context window (e.g. 200K) instead of the fallback's (e.g. 32K), + # causing oversized sessions to overflow the fallback. + # Also pass _config_context_length so the explicit config override + # (model.context_length in config.yaml) is respected — without this, + # the fallback activation drops to 128K even when config says 204800. + if hasattr(agent, 'context_compressor') and agent.context_compressor: + from agent.model_metadata import get_model_context_length + fb_context_length = get_model_context_length( + agent.model, base_url=agent.base_url, + api_key=agent.api_key, provider=agent.provider, + config_context_length=getattr(agent, "_config_context_length", None), + ) + agent.context_compressor.update_model( + model=agent.model, + context_length=fb_context_length, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + ) + + agent._emit_status( + f"🔄 Primary model failed — switching to fallback: " + f"{fb_model} via {fb_provider}" + ) + logging.info( + "Fallback activated: %s → %s (%s)", + old_model, fb_model, fb_provider, + ) + return True + except Exception as e: + logging.error("Failed to activate fallback %s: %s", fb_model, e) + return agent._try_activate_fallback() # try next in chain + + + +def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: + """Request a summary when max iterations are reached. Returns the final response text.""" + print(f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary...") + + summary_request = ( + "You've reached the maximum number of tool-calling iterations allowed. " + "Please provide a final response summarizing what you've found and accomplished so far, " + "without calling any more tools." + ) + messages.append({"role": "user", "content": summary_request}) + + try: + # Build API messages, stripping internal-only fields + # (finish_reason, reasoning) that strict APIs like Mistral reject with 422 + _needs_sanitize = agent._should_sanitize_tool_calls() + api_messages = [] + for msg in messages: + api_msg = msg.copy() + agent._copy_reasoning_content_for_api(msg, api_msg) + for internal_field in ("reasoning", "finish_reason", "_thinking_prefill"): + api_msg.pop(internal_field, None) + if _needs_sanitize: + agent._sanitize_tool_calls_for_strict_api(api_msg) + api_messages.append(api_msg) + + effective_system = agent._cached_system_prompt or "" + if agent.ephemeral_system_prompt: + effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip() + if effective_system: + api_messages = [{"role": "system", "content": effective_system}] + api_messages + if agent.prefill_messages: + sys_offset = 1 if effective_system else 0 + for idx, pfm in enumerate(agent.prefill_messages): + api_messages.insert(sys_offset + idx, pfm.copy()) + + # Same safety net as the main loop: repair tool-call/result + # pairing before asking for a final summary. Compression and + # session resume can leave a tool result whose parent assistant + # tool_call was summarized away; Responses API rejects that as + # "No tool call found for function call output". + api_messages = agent._sanitize_api_messages(api_messages) + + # Same safety net as the main loop: drop thinking-only assistant + # turns so Anthropic-family providers don't 400 the summary call. + api_messages = agent._drop_thinking_only_and_merge_users(api_messages) + + summary_extra_body = {} + try: + from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE as _OMIT_TEMP + except Exception: + _fixed_temperature_for_model = None + _OMIT_TEMP = None + _raw_summary_temp = ( + _fixed_temperature_for_model(agent.model, agent.base_url) + if _fixed_temperature_for_model is not None + else None + ) + _omit_summary_temperature = _raw_summary_temp is _OMIT_TEMP + _summary_temperature = None if _omit_summary_temperature else _raw_summary_temp + _is_nous = "nousresearch" in agent._base_url_lower + # LM Studio uses top-level `reasoning_effort` (not extra_body.reasoning). + # Mirror ChatCompletionsTransport.build_kwargs() so the summary path + # — which calls chat.completions.create() directly without going + # through the transport — sends the same shape the transport does. + _is_lmstudio_summary = ( + (agent.provider or "").strip().lower() == "lmstudio" + and agent._supports_reasoning_extra_body() + ) + _lm_reasoning_effort: str | None = ( + agent._resolve_lmstudio_summary_reasoning_effort() + if _is_lmstudio_summary else None + ) + if not _is_lmstudio_summary and agent._supports_reasoning_extra_body(): + if agent.reasoning_config is not None: + summary_extra_body["reasoning"] = agent.reasoning_config + else: + summary_extra_body["reasoning"] = { + "enabled": True, + "effort": "medium" + } + if _is_nous: + from agent.portal_tags import nous_portal_tags as _portal_tags + summary_extra_body["tags"] = _portal_tags() + + if agent.api_mode == "codex_responses": + codex_kwargs = agent._build_api_kwargs(api_messages) + codex_kwargs.pop("tools", None) + summary_response = agent._run_codex_stream(codex_kwargs) + _ct_sum = agent._get_transport() + _cnr_sum = _ct_sum.normalize_response(summary_response) + final_response = (_cnr_sum.content or "").strip() + else: + summary_kwargs = { + "model": agent.model, + "messages": api_messages, + } + if _summary_temperature is not None: + summary_kwargs["temperature"] = _summary_temperature + if agent.max_tokens is not None: + summary_kwargs.update(agent._max_tokens_param(agent.max_tokens)) + if _lm_reasoning_effort is not None: + summary_kwargs["reasoning_effort"] = _lm_reasoning_effort + + # Include provider routing preferences + provider_preferences = {} + if agent.providers_allowed: + provider_preferences["only"] = agent.providers_allowed + if agent.providers_ignored: + provider_preferences["ignore"] = agent.providers_ignored + if agent.providers_order: + provider_preferences["order"] = agent.providers_order + if agent.provider_sort: + provider_preferences["sort"] = agent.provider_sort + if provider_preferences and ( + (agent.provider or "").strip().lower() == "openrouter" + or agent._is_openrouter_url() + ): + summary_extra_body["provider"] = provider_preferences + + # Pareto Code router plugin — model-gated. Same shape as + # the main-loop emission so summary calls on + # openrouter/pareto-code respect the user's coding-score floor. + if ( + agent.model == "openrouter/pareto-code" + and ( + (agent.provider or "").strip().lower() == "openrouter" + or agent._is_openrouter_url() + ) + and agent.openrouter_min_coding_score is not None + and agent.openrouter_min_coding_score != "" + ): + try: + _ps = float(agent.openrouter_min_coding_score) + except (TypeError, ValueError): + _ps = None + if _ps is not None and 0.0 <= _ps <= 1.0: + summary_extra_body["plugins"] = [ + {"id": "pareto-router", "min_coding_score": _ps} + ] + + if summary_extra_body: + summary_kwargs["extra_body"] = summary_extra_body + + if agent.api_mode == "anthropic_messages": + _tsum = agent._get_transport() + _ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None, + max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, + is_oauth=agent._is_anthropic_oauth, + preserve_dots=agent._anthropic_preserve_dots()) + summary_response = agent._anthropic_messages_create(_ant_kw) + _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) + final_response = (_summary_result.content or "").strip() + else: + summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) + _summary_result = agent._get_transport().normalize_response(summary_response) + final_response = (_summary_result.content or "").strip() + + if final_response: + if "" in final_response: + final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() + if final_response: + messages.append({"role": "assistant", "content": final_response}) + else: + final_response = "I reached the iteration limit and couldn't generate a summary." + else: + # Retry summary generation + if agent.api_mode == "codex_responses": + codex_kwargs = agent._build_api_kwargs(api_messages) + codex_kwargs.pop("tools", None) + retry_response = agent._run_codex_stream(codex_kwargs) + _ct_retry = agent._get_transport() + _cnr_retry = _ct_retry.normalize_response(retry_response) + final_response = (_cnr_retry.content or "").strip() + elif agent.api_mode == "anthropic_messages": + _tretry = agent._get_transport() + _ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None, + is_oauth=agent._is_anthropic_oauth, + max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, + preserve_dots=agent._anthropic_preserve_dots()) + retry_response = agent._anthropic_messages_create(_ant_kw2) + _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) + final_response = (_retry_result.content or "").strip() + else: + summary_kwargs = { + "model": agent.model, + "messages": api_messages, + } + if _summary_temperature is not None: + summary_kwargs["temperature"] = _summary_temperature + if agent.max_tokens is not None: + summary_kwargs.update(agent._max_tokens_param(agent.max_tokens)) + if _lm_reasoning_effort is not None: + summary_kwargs["reasoning_effort"] = _lm_reasoning_effort + if summary_extra_body: + summary_kwargs["extra_body"] = summary_extra_body + + summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) + _retry_result = agent._get_transport().normalize_response(summary_response) + final_response = (_retry_result.content or "").strip() + + if final_response: + if "" in final_response: + final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() + if final_response: + messages.append({"role": "assistant", "content": final_response}) + else: + final_response = "I reached the iteration limit and couldn't generate a summary." + else: + final_response = "I reached the iteration limit and couldn't generate a summary." + + except Exception as e: + logging.warning(f"Failed to get summary response: {e}") + final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" + + return final_response + + + +def cleanup_task_resources(agent, task_id: str) -> None: + """Clean up VM and browser resources for a given task. + + Skips ``cleanup_vm`` when the active terminal environment is marked + persistent (``persistent_filesystem=True``) so that long-lived sandbox + containers survive between turns. The idle reaper in + ``terminal_tool._cleanup_inactive_envs`` still tears them down once + ``terminal.lifetime_seconds`` is exceeded. Non-persistent backends are + torn down per-turn as before to prevent resource leakage (the original + intent of this hook for the Morph backend, see commit fbd3a2fd). + """ + try: + if is_persistent_env(task_id): + if agent.verbose_logging: + logging.debug( + f"Skipping per-turn cleanup_vm for persistent env {task_id}; " + f"idle reaper will handle it." + ) + else: + _ra().cleanup_vm(task_id) + except Exception as e: + if agent.verbose_logging: + logging.warning(f"Failed to cleanup VM for task {task_id}: {e}") + try: + _ra().cleanup_browser(task_id) + except Exception as e: + if agent.verbose_logging: + logging.warning(f"Failed to cleanup browser for task {task_id}: {e}") + + + +__all__ = [ + "interruptible_api_call", + "build_api_kwargs", + "build_assistant_message", + "try_activate_fallback", + "handle_max_iterations", + "cleanup_task_resources", +] diff --git a/run_agent.py b/run_agent.py index 234a322a480..9ee4a0b7bbb 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2772,33 +2772,9 @@ class AIAgent: return None def _cleanup_task_resources(self, task_id: str) -> None: - """Clean up VM and browser resources for a given task. - - Skips ``cleanup_vm`` when the active terminal environment is marked - persistent (``persistent_filesystem=True``) so that long-lived sandbox - containers survive between turns. The idle reaper in - ``terminal_tool._cleanup_inactive_envs`` still tears them down once - ``terminal.lifetime_seconds`` is exceeded. Non-persistent backends are - torn down per-turn as before to prevent resource leakage (the original - intent of this hook for the Morph backend, see commit fbd3a2fd). - """ - try: - if is_persistent_env(task_id): - if self.verbose_logging: - logging.debug( - f"Skipping per-turn cleanup_vm for persistent env {task_id}; " - f"idle reaper will handle it." - ) - else: - cleanup_vm(task_id) - except Exception as e: - if self.verbose_logging: - logging.warning(f"Failed to cleanup VM for task {task_id}: {e}") - try: - cleanup_browser(task_id) - except Exception as e: - if self.verbose_logging: - logging.warning(f"Failed to cleanup browser for task {task_id}: {e}") + """Forwarder — see ``agent.chat_completion_helpers.cleanup_task_resources``.""" + from agent.chat_completion_helpers import cleanup_task_resources + return cleanup_task_resources(self, task_id) # ------------------------------------------------------------------ # Background memory/skill review @@ -5804,156 +5780,9 @@ class AIAgent: ) def _interruptible_api_call(self, api_kwargs: dict): - """ - Run the API call in a background thread so the main conversation loop - can detect interrupts without waiting for the full HTTP round-trip. - - Each worker thread gets its own OpenAI client instance. Interrupts only - close that worker-local client, so retries and other requests never - inherit a closed transport. - - Includes a stale-call detector: if no response arrives within the - configured timeout, the connection is killed and an error raised so - the main retry loop can try again with backoff / credential rotation / - provider fallback. - """ - result = {"response": None, "error": None} - request_client_holder = {"client": None} - - def _call(): - try: - if self.api_mode == "codex_responses": - request_client_holder["client"] = self._create_request_openai_client( - reason="codex_stream_request", - api_kwargs=api_kwargs, - ) - result["response"] = self._run_codex_stream( - api_kwargs, - client=request_client_holder["client"], - on_first_delta=getattr(self, "_codex_on_first_delta", None), - ) - elif self.api_mode == "anthropic_messages": - result["response"] = self._anthropic_messages_create(api_kwargs) - elif self.api_mode == "bedrock_converse": - # Bedrock uses boto3 directly — no OpenAI client needed. - # normalize_converse_response produces an OpenAI-compatible - # SimpleNamespace so the rest of the agent loop can treat - # bedrock responses like chat_completions responses. - from agent.bedrock_adapter import ( - _get_bedrock_runtime_client, - invalidate_runtime_client, - is_stale_connection_error, - normalize_converse_response, - ) - region = api_kwargs.pop("__bedrock_region__", "us-east-1") - api_kwargs.pop("__bedrock_converse__", None) - client = _get_bedrock_runtime_client(region) - try: - raw_response = client.converse(**api_kwargs) - except Exception as _bedrock_exc: - # Evict the cached client on stale-connection failures - # so the outer retry loop builds a fresh client/pool. - if is_stale_connection_error(_bedrock_exc): - invalidate_runtime_client(region) - raise - result["response"] = normalize_converse_response(raw_response) - else: - request_client_holder["client"] = self._create_request_openai_client( - reason="chat_completion_request", - api_kwargs=api_kwargs, - ) - result["response"] = request_client_holder["client"].chat.completions.create(**api_kwargs) - except Exception as e: - result["error"] = e - finally: - request_client = request_client_holder.get("client") - if request_client is not None: - self._close_request_openai_client(request_client, reason="request_complete") - - # ── Stale-call timeout (mirrors streaming stale detector) ──────── - # Non-streaming calls return nothing until the full response is - # ready. Without this, a hung provider can block for the full - # httpx timeout (default 1800s) with zero feedback. The stale - # detector kills the connection early so the main retry loop can - # apply richer recovery (credential rotation, provider fallback). - _stale_timeout = self._compute_non_stream_stale_timeout( - api_kwargs.get("messages", []) - ) - - _call_start = time.time() - self._touch_activity("waiting for non-streaming API response") - - t = threading.Thread(target=_call, daemon=True) - t.start() - _poll_count = 0 - while t.is_alive(): - t.join(timeout=0.3) - _poll_count += 1 - - # Touch activity every ~30s so the gateway's inactivity - # monitor knows we're alive while waiting for the response. - if _poll_count % 100 == 0: # 100 × 0.3s = 30s - _elapsed = time.time() - _call_start - self._touch_activity( - f"waiting for non-streaming response ({int(_elapsed)}s elapsed)" - ) - - # Stale-call detector: kill the connection if no response - # arrives within the configured timeout. - _elapsed = time.time() - _call_start - if _elapsed > _stale_timeout: - _est_ctx = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 - logger.warning( - "Non-streaming API call stale for %.0fs (threshold %.0fs). " - "model=%s context=~%s tokens. Killing connection.", - _elapsed, _stale_timeout, - api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", - ) - self._emit_status( - f"⚠️ No response from provider for {int(_elapsed)}s " - f"(non-streaming, model: {api_kwargs.get('model', 'unknown')}). " - f"Aborting call." - ) - try: - if self.api_mode == "anthropic_messages": - self._anthropic_client.close() - self._rebuild_anthropic_client() - else: - rc = request_client_holder.get("client") - if rc is not None: - self._close_request_openai_client(rc, reason="stale_call_kill") - except Exception: - pass - self._touch_activity( - f"stale non-streaming call killed after {int(_elapsed)}s" - ) - # Wait briefly for the thread to notice the closed connection. - t.join(timeout=2.0) - if result["error"] is None and result["response"] is None: - result["error"] = TimeoutError( - f"Non-streaming API call timed out after {int(_elapsed)}s " - f"with no response (threshold: {int(_stale_timeout)}s)" - ) - break - - if self._interrupt_requested: - # Force-close the in-flight worker-local HTTP connection to stop - # token generation without poisoning the shared client used to - # seed future retries. - try: - if self.api_mode == "anthropic_messages": - self._anthropic_client.close() - self._rebuild_anthropic_client() - else: - request_client = request_client_holder.get("client") - if request_client is not None: - self._close_request_openai_client(request_client, reason="interrupt_abort") - except Exception: - pass - raise InterruptedError("Agent interrupted during API call") - if result["error"] is not None: - raise result["error"] - return result["response"] + """Forwarder — see ``agent.chat_completion_helpers.interruptible_api_call``.""" + from agent.chat_completion_helpers import interruptible_api_call + return interruptible_api_call(self, api_kwargs) # ── Unified streaming API call ───────────────────────────────────────── @@ -7015,233 +6844,9 @@ class AIAgent: # ── Provider fallback ────────────────────────────────────────────────── def _try_activate_fallback(self, reason: "FailoverReason | None" = None) -> bool: - """Switch to the next fallback model/provider in the chain. - - Called when the current model is failing after retries. Swaps the - OpenAI client, model slug, and provider in-place so the retry loop - can continue with the new backend. Advances through the chain on - each call; returns False when exhausted. - - Uses the centralized provider router (resolve_provider_client) for - auth resolution and client construction — no duplicated provider→key - mappings. - """ - if reason in {FailoverReason.rate_limit, FailoverReason.billing}: - # Only start cooldown when leaving the primary provider. If we're - # already on a fallback and chain-switching, the primary wasn't the - # source of the 429 so the cooldown should not be reset/extended. - fallback_already_active = bool(getattr(self, "_fallback_activated", False)) - current_provider = (getattr(self, "provider", "") or "").strip().lower() - primary_provider = ((self._primary_runtime or {}).get("provider") or "").strip().lower() - if (not fallback_already_active) or (primary_provider and current_provider == primary_provider): - self._rate_limited_until = time.monotonic() + 60 - if self._fallback_index >= len(self._fallback_chain): - return False - - fb = self._fallback_chain[self._fallback_index] - self._fallback_index += 1 - fb_provider = (fb.get("provider") or "").strip().lower() - fb_model = (fb.get("model") or "").strip() - if not fb_provider or not fb_model: - return self._try_activate_fallback() # skip invalid, try next - - # Skip entries that resolve to the current (provider, model) — falling - # back to the same backend that just failed loops the failure. Compare - # base_url too so two distinct custom_providers entries pointing at the - # same shim/proxy URL also dedup. See issue #22548. - current_provider = (getattr(self, "provider", "") or "").strip().lower() - current_model = (getattr(self, "model", "") or "").strip() - current_base_url = str(getattr(self, "base_url", "") or "").rstrip("/").lower() - fb_base_url_for_dedup = (fb.get("base_url") or "").strip().rstrip("/").lower() - if fb_provider == current_provider and fb_model == current_model: - logging.warning( - "Fallback skip: chain entry %s/%s matches current provider/model", - fb_provider, fb_model, - ) - return self._try_activate_fallback() - if ( - fb_base_url_for_dedup - and current_base_url - and fb_base_url_for_dedup == current_base_url - and fb_model == current_model - ): - logging.warning( - "Fallback skip: chain entry base_url %s matches current backend", - fb_base_url_for_dedup, - ) - return self._try_activate_fallback() - - # Use centralized router for client construction. - # raw_codex=True because the main agent needs direct responses.stream() - # access for Codex providers. - try: - from agent.auxiliary_client import resolve_provider_client - # Pass base_url and api_key from fallback config so custom - # endpoints (e.g. Ollama Cloud) resolve correctly instead of - # falling through to OpenRouter defaults. - fb_base_url_hint = (fb.get("base_url") or "").strip() or None - fb_api_key_hint = (fb.get("api_key") or "").strip() or None - if not fb_api_key_hint: - # key_env and api_key_env are both documented aliases (see - # _normalize_custom_provider_entry in hermes_cli/config.py). - fb_key_env = (fb.get("key_env") or fb.get("api_key_env") or "").strip() - if fb_key_env: - fb_api_key_hint = os.getenv(fb_key_env, "").strip() or None - # For Ollama Cloud endpoints, pull OLLAMA_API_KEY from env - # when no explicit key is in the fallback config. Host match - # (not substring) — see GHSA-76xc-57q6-vm5m. - if fb_base_url_hint and base_url_host_matches(fb_base_url_hint, "ollama.com") and not fb_api_key_hint: - fb_api_key_hint = os.getenv("OLLAMA_API_KEY") or None - fb_client, _resolved_fb_model = resolve_provider_client( - fb_provider, model=fb_model, raw_codex=True, - explicit_base_url=fb_base_url_hint, - explicit_api_key=fb_api_key_hint) - if fb_client is None: - logging.warning( - "Fallback to %s failed: provider not configured", - fb_provider) - return self._try_activate_fallback() # try next in chain - try: - from hermes_cli.model_normalize import normalize_model_for_provider - - fb_model = normalize_model_for_provider(fb_model, fb_provider) - except Exception: - pass - - # Determine api_mode from provider / base URL / model - fb_api_mode = "chat_completions" - fb_base_url = str(fb_client.base_url) - _fb_is_azure = self._is_azure_openai_url(fb_base_url) - if fb_provider == "openai-codex": - fb_api_mode = "codex_responses" - elif fb_provider == "anthropic" or fb_base_url.rstrip("/").lower().endswith("/anthropic"): - fb_api_mode = "anthropic_messages" - elif _fb_is_azure: - # Azure OpenAI serves gpt-5.x on /chat/completions — does NOT - # support the Responses API. Stay on chat_completions. - fb_api_mode = "chat_completions" - elif self._is_direct_openai_url(fb_base_url): - fb_api_mode = "codex_responses" - elif self._provider_model_requires_responses_api( - fb_model, - provider=fb_provider, - ): - # GPT-5.x models usually need Responses API, but keep - # provider-specific exceptions like Copilot gpt-5-mini on - # chat completions. - fb_api_mode = "codex_responses" - elif fb_provider == "bedrock" or ( - base_url_hostname(fb_base_url).startswith("bedrock-runtime.") - and base_url_host_matches(fb_base_url, "amazonaws.com") - ): - fb_api_mode = "bedrock_converse" - - old_model = self.model - - # Clear the per-config context_length override so the fallback - # model's actual context window is resolved instead of inheriting - # the stale value from the previous model. See #22387. - self._config_context_length = None - self.model = fb_model - self.provider = fb_provider - self.base_url = fb_base_url - self.api_mode = fb_api_mode - if hasattr(self, "_transport_cache"): - self._transport_cache.clear() - self._fallback_activated = True - - # Honor per-provider / per-model request_timeout_seconds for the - # fallback target (same knob the primary client uses). None = use - # SDK default. - _fb_timeout = get_provider_request_timeout(fb_provider, fb_model) - - if fb_api_mode == "anthropic_messages": - # Build native Anthropic client instead of using OpenAI client - from agent.anthropic_adapter import build_anthropic_client, resolve_anthropic_token, _is_oauth_token - effective_key = (fb_client.api_key or resolve_anthropic_token() or "") if fb_provider == "anthropic" else (fb_client.api_key or "") - self.api_key = effective_key - self._anthropic_api_key = effective_key - self._anthropic_base_url = fb_base_url - self._anthropic_client = build_anthropic_client( - effective_key, self._anthropic_base_url, timeout=_fb_timeout, - ) - self._is_anthropic_oauth = _is_oauth_token(effective_key) if fb_provider == "anthropic" else False - self.client = None - self._client_kwargs = {} - else: - # Swap OpenAI client and config in-place - self.api_key = fb_client.api_key - self.client = fb_client - # Preserve provider-specific headers that - # resolve_provider_client() may have baked into - # fb_client via the default_headers kwarg. The OpenAI - # SDK stores these in _custom_headers. Without this, - # subsequent request-client rebuilds (via - # _create_request_openai_client) drop the headers, - # causing 403s from providers like Kimi Coding that - # require a User-Agent sentinel. - fb_headers = getattr(fb_client, "_custom_headers", None) - if not fb_headers: - fb_headers = getattr(fb_client, "default_headers", None) - self._client_kwargs = { - "api_key": fb_client.api_key, - "base_url": fb_base_url, - **({"default_headers": dict(fb_headers)} if fb_headers else {}), - } - if _fb_timeout is not None: - self._client_kwargs["timeout"] = _fb_timeout - # Rebuild the shared OpenAI client so the configured - # timeout takes effect on the very next fallback request, - # not only after a later credential-rotation rebuild. - self._replace_primary_openai_client(reason="fallback_timeout_apply") - - # Re-evaluate prompt caching for the new provider/model - self._use_prompt_caching, self._use_native_cache_layout = ( - self._anthropic_prompt_cache_policy( - provider=fb_provider, - base_url=fb_base_url, - api_mode=fb_api_mode, - model=fb_model, - ) - ) - - # LM Studio: preload before probing the fallback's context length. - self._ensure_lmstudio_runtime_loaded() - - # Update context compressor limits for the fallback model. - # Without this, compression decisions use the primary model's - # context window (e.g. 200K) instead of the fallback's (e.g. 32K), - # causing oversized sessions to overflow the fallback. - # Also pass _config_context_length so the explicit config override - # (model.context_length in config.yaml) is respected — without this, - # the fallback activation drops to 128K even when config says 204800. - if hasattr(self, 'context_compressor') and self.context_compressor: - from agent.model_metadata import get_model_context_length - fb_context_length = get_model_context_length( - self.model, base_url=self.base_url, - api_key=self.api_key, provider=self.provider, - config_context_length=getattr(self, "_config_context_length", None), - ) - self.context_compressor.update_model( - model=self.model, - context_length=fb_context_length, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - ) - - self._emit_status( - f"🔄 Primary model failed — switching to fallback: " - f"{fb_model} via {fb_provider}" - ) - logging.info( - "Fallback activated: %s → %s (%s)", - old_model, fb_model, fb_provider, - ) - return True - except Exception as e: - logging.error("Failed to activate fallback %s: %s", fb_model, e) - return self._try_activate_fallback() # try next in chain + """Forwarder — see ``agent.chat_completion_helpers.try_activate_fallback``.""" + from agent.chat_completion_helpers import try_activate_fallback + return try_activate_fallback(self, reason) # ── Per-turn primary restoration ───────────────────────────────────── @@ -7789,220 +7394,9 @@ class AIAgent: break def _build_api_kwargs(self, api_messages: list) -> dict: - """Build the keyword arguments dict for the active API mode.""" - tools_for_api = self.tools - - if self.api_mode == "anthropic_messages": - _transport = self._get_transport() - anthropic_messages = self._prepare_anthropic_messages_for_api(api_messages) - ctx_len = getattr(self, "context_compressor", None) - ctx_len = ctx_len.context_length if ctx_len else None - ephemeral_out = getattr(self, "_ephemeral_max_output_tokens", None) - if ephemeral_out is not None: - self._ephemeral_max_output_tokens = None # consume immediately - return _transport.build_kwargs( - model=self.model, - messages=anthropic_messages, - tools=tools_for_api, - max_tokens=ephemeral_out if ephemeral_out is not None else self.max_tokens, - reasoning_config=self.reasoning_config, - is_oauth=self._is_anthropic_oauth, - preserve_dots=self._anthropic_preserve_dots(), - context_length=ctx_len, - base_url=getattr(self, "_anthropic_base_url", None), - fast_mode=(self.request_overrides or {}).get("speed") == "fast", - drop_context_1m_beta=bool(getattr(self, "_oauth_1m_beta_disabled", False)), - ) - - # AWS Bedrock native Converse API — bypasses the OpenAI client entirely. - # The adapter handles message/tool conversion and boto3 calls directly. - if self.api_mode == "bedrock_converse": - _bt = self._get_transport() - region = getattr(self, "_bedrock_region", None) or "us-east-1" - guardrail = getattr(self, "_bedrock_guardrail_config", None) - return _bt.build_kwargs( - model=self.model, - messages=api_messages, - tools=tools_for_api, - max_tokens=self.max_tokens or 4096, - region=region, - guardrail_config=guardrail, - ) - - if self.api_mode == "codex_responses": - _ct = self._get_transport() - is_github_responses = ( - base_url_host_matches(self.base_url, "models.github.ai") - or base_url_host_matches(self.base_url, "api.githubcopilot.com") - ) - is_codex_backend = ( - self.provider == "openai-codex" - or ( - self._base_url_hostname == "chatgpt.com" - and "/backend-api/codex" in self._base_url_lower - ) - ) - is_xai_responses = self.provider == "xai" or self._base_url_hostname == "api.x.ai" - _msgs_for_codex = self._prepare_messages_for_non_vision_model(api_messages) - return _ct.build_kwargs( - model=self.model, - messages=_msgs_for_codex, - tools=tools_for_api, - reasoning_config=self.reasoning_config, - session_id=getattr(self, "session_id", None), - max_tokens=self.max_tokens, - request_overrides=self.request_overrides, - is_github_responses=is_github_responses, - is_codex_backend=is_codex_backend, - is_xai_responses=is_xai_responses, - github_reasoning_extra=self._github_models_reasoning_extra_body() if is_github_responses else None, - ) - - # ── chat_completions (default) ───────────────────────────────────── - _ct = self._get_transport() - - # Provider detection flags - _is_qwen = self._is_qwen_portal() - _is_or = self._is_openrouter_url() - _is_gh = ( - base_url_host_matches(self._base_url_lower, "models.github.ai") - or base_url_host_matches(self._base_url_lower, "api.githubcopilot.com") - ) - _is_nous = "nousresearch" in self._base_url_lower - _is_nvidia = "integrate.api.nvidia.com" in self._base_url_lower - _is_kimi = ( - base_url_host_matches(self.base_url, "api.kimi.com") - or base_url_host_matches(self.base_url, "moonshot.ai") - or base_url_host_matches(self.base_url, "moonshot.cn") - ) - _is_tokenhub = base_url_host_matches(self._base_url_lower, "tokenhub.tencentmaas.com") - _is_lmstudio = (self.provider or "").strip().lower() == "lmstudio" - - # Temperature: _fixed_temperature_for_model may return OMIT_TEMPERATURE - # sentinel (temperature omitted entirely), a numeric override, or None. - try: - from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE - _ft = _fixed_temperature_for_model(self.model, self.base_url) - _omit_temp = _ft is OMIT_TEMPERATURE - _fixed_temp = _ft if not _omit_temp else None - except Exception: - _omit_temp = False - _fixed_temp = None - - # Provider preferences (OpenRouter-style) - _prefs: Dict[str, Any] = {} - if self.providers_allowed: - _prefs["only"] = self.providers_allowed - if self.providers_ignored: - _prefs["ignore"] = self.providers_ignored - if self.providers_order: - _prefs["order"] = self.providers_order - if self.provider_sort: - _prefs["sort"] = self.provider_sort - if self.provider_require_parameters: - _prefs["require_parameters"] = True - if self.provider_data_collection: - _prefs["data_collection"] = self.provider_data_collection - - # Claude max-output override on aggregators - _ant_max = None - if (_is_or or _is_nous) and "claude" in (self.model or "").lower(): - try: - from agent.anthropic_adapter import _get_anthropic_max_output - _ant_max = _get_anthropic_max_output(self.model) - except Exception: - pass - - # Qwen session metadata - _qwen_meta = None - if _is_qwen: - _qwen_meta = { - "sessionId": self.session_id or "hermes", - "promptId": str(uuid.uuid4()), - } - - # ── Provider profile path (registered providers) ─────────────────── - # Profiles handle per-provider quirks via hooks. When a profile is - # found, delegate fully; otherwise fall through to the legacy flag path. - try: - from providers import get_provider_profile - _profile = get_provider_profile(self.provider) - except Exception: - _profile = None - - if _profile: - _ephemeral_out = getattr(self, "_ephemeral_max_output_tokens", None) - if _ephemeral_out is not None: - self._ephemeral_max_output_tokens = None - - return _ct.build_kwargs( - model=self.model, - messages=api_messages, - tools=tools_for_api, - base_url=self.base_url, - timeout=self._resolved_api_call_timeout(), - max_tokens=self.max_tokens, - ephemeral_max_output_tokens=_ephemeral_out, - max_tokens_param_fn=self._max_tokens_param, - reasoning_config=self.reasoning_config, - request_overrides=self.request_overrides, - session_id=getattr(self, "session_id", None), - provider_profile=_profile, - ollama_num_ctx=self._ollama_num_ctx, - # Context forwarded to profile hooks: - provider_preferences=_prefs or None, - openrouter_min_coding_score=self.openrouter_min_coding_score, - anthropic_max_output=_ant_max, - supports_reasoning=self._supports_reasoning_extra_body(), - qwen_session_metadata=_qwen_meta, - ) - - # ── Legacy flag path ──────────────────────────────────────────── - # Reached only when get_provider_profile() returns None — i.e. a - # completely unknown provider not in providers/ registry. - _ephemeral_out = getattr(self, "_ephemeral_max_output_tokens", None) - if _ephemeral_out is not None: - self._ephemeral_max_output_tokens = None - - # Strip image parts for non-vision models (no-op when vision-capable). - _msgs_for_chat = self._prepare_messages_for_non_vision_model(api_messages) - - return _ct.build_kwargs( - model=self.model, - messages=_msgs_for_chat, - tools=tools_for_api, - base_url=self.base_url, - timeout=self._resolved_api_call_timeout(), - max_tokens=self.max_tokens, - ephemeral_max_output_tokens=_ephemeral_out, - max_tokens_param_fn=self._max_tokens_param, - reasoning_config=self.reasoning_config, - request_overrides=self.request_overrides, - session_id=getattr(self, "session_id", None), - model_lower=(self.model or "").lower(), - is_openrouter=_is_or, - is_nous=_is_nous, - is_qwen_portal=_is_qwen, - is_github_models=_is_gh, - is_nvidia_nim=_is_nvidia, - is_kimi=_is_kimi, - is_tokenhub=_is_tokenhub, - is_lmstudio=_is_lmstudio, - is_custom_provider=self.provider == "custom", - ollama_num_ctx=self._ollama_num_ctx, - provider_preferences=_prefs or None, - openrouter_min_coding_score=self.openrouter_min_coding_score, - qwen_prepare_fn=self._qwen_prepare_chat_messages if _is_qwen else None, - qwen_prepare_inplace_fn=self._qwen_prepare_chat_messages_inplace if _is_qwen else None, - qwen_session_metadata=_qwen_meta, - fixed_temperature=_fixed_temp, - omit_temperature=_omit_temp, - supports_reasoning=self._supports_reasoning_extra_body(), - github_reasoning_extra=self._github_models_reasoning_extra_body() if _is_gh else None, - lmstudio_reasoning_options=self._lmstudio_reasoning_options_cached() if _is_lmstudio else None, - anthropic_max_output=_ant_max, - provider_name=self.provider, - ) + """Forwarder — see ``agent.chat_completion_helpers.build_api_kwargs``.""" + from agent.chat_completion_helpers import build_api_kwargs + return build_api_kwargs(self, api_messages) def _supports_reasoning_extra_body(self) -> bool: """Return True when reasoning extra_body is safe to send for this route/model. @@ -8127,197 +7521,9 @@ class AIAgent: return {"effort": requested_effort} def _build_assistant_message(self, assistant_message, finish_reason: str) -> dict: - """Build a normalized assistant message dict from an API response message. - - Handles reasoning extraction, reasoning_details, and optional tool_calls - so both the tool-call path and the final-response path share one builder. - """ - assistant_tool_calls = getattr(assistant_message, "tool_calls", None) - reasoning_text = self._extract_reasoning(assistant_message) - _from_structured = bool(reasoning_text) - - # Fallback: extract inline blocks from content when no structured - # reasoning fields are present (some models/providers embed thinking - # directly in the content rather than returning separate API fields). - if not reasoning_text: - content = assistant_message.content or "" - think_blocks = re.findall(r'(.*?)', content, flags=re.DOTALL) - if think_blocks: - combined = "\n\n".join(b.strip() for b in think_blocks if b.strip()) - reasoning_text = combined or None - - if reasoning_text and self.verbose_logging: - logging.debug(f"Captured reasoning ({len(reasoning_text)} chars): {reasoning_text}") - - if reasoning_text and self.reasoning_callback: - # Skip callback when streaming is active — reasoning was already - # displayed during the stream via one of two paths: - # (a) _fire_reasoning_delta (structured reasoning_content deltas) - # (b) _stream_delta tag extraction (/) - # When streaming is NOT active, always fire so non-streaming modes - # (gateway, batch, quiet) still get reasoning. - # Any reasoning that wasn't shown during streaming is caught by the - # CLI post-response display fallback (cli.py _reasoning_shown_this_turn). - if not self.stream_delta_callback and not self._stream_callback: - try: - self.reasoning_callback(reasoning_text) - except Exception: - pass - - # Sanitize surrogates from API response — some models (e.g. Kimi/GLM via Ollama) - # can return invalid surrogate code points that crash json.dumps() on persist. - _raw_content = assistant_message.content or "" - _san_content = _sanitize_surrogates(_raw_content) - if reasoning_text: - reasoning_text = _sanitize_surrogates(reasoning_text) - - # Strip inline reasoning tags (… etc.) from the stored - # assistant content. Reasoning was already captured into - # ``reasoning_text`` above (either from structured fields or the - # inline-block fallback), so the raw tags in content are redundant. - # Leaving them in place caused reasoning to leak to messaging - # platforms (#8878, #9568), inflate context on subsequent turns - # (#9306 observed 16% content-size reduction on a real MiniMax - # session), and pollute generated session titles. One strip at the - # storage boundary cleans content for every downstream consumer: - # API replay, session transcript, gateway delivery, CLI display, - # compression, title generation. - if isinstance(_san_content, str) and _san_content: - _san_content = self._strip_think_blocks(_san_content).strip() - - msg = { - "role": "assistant", - "content": _san_content, - "reasoning": reasoning_text, - "finish_reason": finish_reason, - } - - raw_reasoning_content = getattr(assistant_message, "reasoning_content", None) - if raw_reasoning_content is None and hasattr(assistant_message, "model_extra"): - model_extra = getattr(assistant_message, "model_extra", None) or {} - if isinstance(model_extra, dict) and "reasoning_content" in model_extra: - raw_reasoning_content = model_extra["reasoning_content"] - if raw_reasoning_content is not None: - msg["reasoning_content"] = _sanitize_surrogates(raw_reasoning_content) - elif assistant_tool_calls and self._needs_thinking_reasoning_pad(): - # DeepSeek v4 thinking mode and Kimi / Moonshot thinking mode - # both require reasoning_content on every assistant tool-call - # message. Without it, replaying the persisted message causes - # HTTP 400 ("The reasoning_content in the thinking mode must - # be passed back to the API"). Include streamed reasoning - # text when captured; otherwise pad with a single space — - # DeepSeek V4 Pro tightened validation and rejects empty - # string ("The reasoning content in the thinking mode must - # be passed back to the API"). A space satisfies non-empty - # checks everywhere without leaking fabricated reasoning. - # Refs #15250, #17400, #17341. - msg["reasoning_content"] = reasoning_text or " " - - # Additive fallback (refs #16844, #16884). Streaming-only providers - # (glm, MiniMax, gpt-5.x via aigw, Anthropic via openai-compat shims) - # accumulate reasoning through ``delta.reasoning_content`` chunks - # but never land it on the message object as a top-level attribute, - # so neither branch above fires and the chain-of-thought is stored - # only under the internal ``reasoning`` key. When the user later - # replays that history through a DeepSeek-v4 / Kimi thinking model, - # the missing ``reasoning_content`` causes HTTP 400 ("The - # reasoning_content in the thinking mode must be passed back to the - # API."). - # - # Promote the already-sanitized streamed ``reasoning_text`` to - # ``reasoning_content`` at write time, but ONLY when no prior branch - # already set it AND we actually captured reasoning text. This - # preserves every existing behavior: - # - SDK-exposed ``reasoning_content`` (OpenAI/Moonshot/DeepSeek SDK) - # still wins. - # - DeepSeek tool-call ""-pad (#15250) still fires. - # - Non-thinking turns with no reasoning leave the field absent, - # so ``_copy_reasoning_content_for_api``'s cross-provider leak - # guard (#15748) and ``reasoning``→``reasoning_content`` - # promotion tiers still apply at replay time. - if "reasoning_content" not in msg and reasoning_text: - msg["reasoning_content"] = reasoning_text - - if hasattr(assistant_message, 'reasoning_details') and assistant_message.reasoning_details: - # Pass reasoning_details back unmodified so providers (OpenRouter, - # Anthropic, OpenAI) can maintain reasoning continuity across turns. - # Each provider may include opaque fields (signature, encrypted_content) - # that must be preserved exactly. - raw_details = assistant_message.reasoning_details - preserved = [] - for d in raw_details: - if isinstance(d, dict): - preserved.append(d) - elif hasattr(d, "__dict__"): - preserved.append(d.__dict__) - elif hasattr(d, "model_dump"): - preserved.append(d.model_dump()) - if preserved: - msg["reasoning_details"] = preserved - - # Codex Responses API: preserve encrypted reasoning items for - # multi-turn continuity. These get replayed as input on the next turn. - codex_items = getattr(assistant_message, "codex_reasoning_items", None) - if codex_items: - msg["codex_reasoning_items"] = codex_items - - # Codex Responses API: preserve exact assistant message items (with - # id/phase) so follow-up turns can replay structured items instead of - # flattening to plain text. This is required for prefix cache hits. - codex_message_items = getattr(assistant_message, "codex_message_items", None) - if codex_message_items: - msg["codex_message_items"] = codex_message_items - - if assistant_tool_calls: - tool_calls = [] - for tool_call in assistant_tool_calls: - raw_id = getattr(tool_call, "id", None) - call_id = getattr(tool_call, "call_id", None) - if not isinstance(call_id, str) or not call_id.strip(): - embedded_call_id, _ = self._split_responses_tool_id(raw_id) - call_id = embedded_call_id - if not isinstance(call_id, str) or not call_id.strip(): - if isinstance(raw_id, str) and raw_id.strip(): - call_id = raw_id.strip() - else: - _fn = getattr(tool_call, "function", None) - _fn_name = getattr(_fn, "name", "") if _fn else "" - _fn_args = getattr(_fn, "arguments", "{}") if _fn else "{}" - call_id = self._deterministic_call_id(_fn_name, _fn_args, len(tool_calls)) - call_id = call_id.strip() - - response_item_id = getattr(tool_call, "response_item_id", None) - if not isinstance(response_item_id, str) or not response_item_id.strip(): - _, embedded_response_item_id = self._split_responses_tool_id(raw_id) - response_item_id = embedded_response_item_id - - response_item_id = self._derive_responses_function_call_id( - call_id, - response_item_id if isinstance(response_item_id, str) else None, - ) - - tc_dict = { - "id": call_id, - "call_id": call_id, - "response_item_id": response_item_id, - "type": tool_call.type, - "function": { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments - }, - } - # Preserve extra_content (e.g. Gemini thought_signature) so it - # is sent back on subsequent API calls. Without this, Gemini 3 - # thinking models reject the request with a 400 error. - extra = getattr(tool_call, "extra_content", None) - if extra is not None: - if hasattr(extra, "model_dump"): - extra = extra.model_dump() - tc_dict["extra_content"] = extra - tool_calls.append(tc_dict) - msg["tool_calls"] = tool_calls - - return msg + """Forwarder — see ``agent.chat_completion_helpers.build_assistant_message``.""" + from agent.chat_completion_helpers import build_assistant_message + return build_assistant_message(self, assistant_message, finish_reason) def _needs_thinking_reasoning_pad(self) -> bool: """Return True when the active provider enforces reasoning_content echo-back. @@ -8809,220 +8015,9 @@ class AIAgent: return execute_tool_calls_sequential(self, assistant_message, messages, effective_task_id, api_call_count) def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: - """Request a summary when max iterations are reached. Returns the final response text.""" - print(f"⚠️ Reached maximum iterations ({self.max_iterations}). Requesting summary...") - - summary_request = ( - "You've reached the maximum number of tool-calling iterations allowed. " - "Please provide a final response summarizing what you've found and accomplished so far, " - "without calling any more tools." - ) - messages.append({"role": "user", "content": summary_request}) - - try: - # Build API messages, stripping internal-only fields - # (finish_reason, reasoning) that strict APIs like Mistral reject with 422 - _needs_sanitize = self._should_sanitize_tool_calls() - api_messages = [] - for msg in messages: - api_msg = msg.copy() - self._copy_reasoning_content_for_api(msg, api_msg) - for internal_field in ("reasoning", "finish_reason", "_thinking_prefill"): - api_msg.pop(internal_field, None) - if _needs_sanitize: - self._sanitize_tool_calls_for_strict_api(api_msg) - api_messages.append(api_msg) - - effective_system = self._cached_system_prompt or "" - if self.ephemeral_system_prompt: - effective_system = (effective_system + "\n\n" + self.ephemeral_system_prompt).strip() - if effective_system: - api_messages = [{"role": "system", "content": effective_system}] + api_messages - if self.prefill_messages: - sys_offset = 1 if effective_system else 0 - for idx, pfm in enumerate(self.prefill_messages): - api_messages.insert(sys_offset + idx, pfm.copy()) - - # Same safety net as the main loop: repair tool-call/result - # pairing before asking for a final summary. Compression and - # session resume can leave a tool result whose parent assistant - # tool_call was summarized away; Responses API rejects that as - # "No tool call found for function call output". - api_messages = self._sanitize_api_messages(api_messages) - - # Same safety net as the main loop: drop thinking-only assistant - # turns so Anthropic-family providers don't 400 the summary call. - api_messages = self._drop_thinking_only_and_merge_users(api_messages) - - summary_extra_body = {} - try: - from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE as _OMIT_TEMP - except Exception: - _fixed_temperature_for_model = None - _OMIT_TEMP = None - _raw_summary_temp = ( - _fixed_temperature_for_model(self.model, self.base_url) - if _fixed_temperature_for_model is not None - else None - ) - _omit_summary_temperature = _raw_summary_temp is _OMIT_TEMP - _summary_temperature = None if _omit_summary_temperature else _raw_summary_temp - _is_nous = "nousresearch" in self._base_url_lower - # LM Studio uses top-level `reasoning_effort` (not extra_body.reasoning). - # Mirror ChatCompletionsTransport.build_kwargs() so the summary path - # — which calls chat.completions.create() directly without going - # through the transport — sends the same shape the transport does. - _is_lmstudio_summary = ( - (self.provider or "").strip().lower() == "lmstudio" - and self._supports_reasoning_extra_body() - ) - _lm_reasoning_effort: str | None = ( - self._resolve_lmstudio_summary_reasoning_effort() - if _is_lmstudio_summary else None - ) - if not _is_lmstudio_summary and self._supports_reasoning_extra_body(): - if self.reasoning_config is not None: - summary_extra_body["reasoning"] = self.reasoning_config - else: - summary_extra_body["reasoning"] = { - "enabled": True, - "effort": "medium" - } - if _is_nous: - from agent.portal_tags import nous_portal_tags as _portal_tags - summary_extra_body["tags"] = _portal_tags() - - if self.api_mode == "codex_responses": - codex_kwargs = self._build_api_kwargs(api_messages) - codex_kwargs.pop("tools", None) - summary_response = self._run_codex_stream(codex_kwargs) - _ct_sum = self._get_transport() - _cnr_sum = _ct_sum.normalize_response(summary_response) - final_response = (_cnr_sum.content or "").strip() - else: - summary_kwargs = { - "model": self.model, - "messages": api_messages, - } - if _summary_temperature is not None: - summary_kwargs["temperature"] = _summary_temperature - if self.max_tokens is not None: - summary_kwargs.update(self._max_tokens_param(self.max_tokens)) - if _lm_reasoning_effort is not None: - summary_kwargs["reasoning_effort"] = _lm_reasoning_effort - - # Include provider routing preferences - provider_preferences = {} - if self.providers_allowed: - provider_preferences["only"] = self.providers_allowed - if self.providers_ignored: - provider_preferences["ignore"] = self.providers_ignored - if self.providers_order: - provider_preferences["order"] = self.providers_order - if self.provider_sort: - provider_preferences["sort"] = self.provider_sort - if provider_preferences and ( - (self.provider or "").strip().lower() == "openrouter" - or self._is_openrouter_url() - ): - summary_extra_body["provider"] = provider_preferences - - # Pareto Code router plugin — model-gated. Same shape as - # the main-loop emission so summary calls on - # openrouter/pareto-code respect the user's coding-score floor. - if ( - self.model == "openrouter/pareto-code" - and ( - (self.provider or "").strip().lower() == "openrouter" - or self._is_openrouter_url() - ) - and self.openrouter_min_coding_score is not None - and self.openrouter_min_coding_score != "" - ): - try: - _ps = float(self.openrouter_min_coding_score) - except (TypeError, ValueError): - _ps = None - if _ps is not None and 0.0 <= _ps <= 1.0: - summary_extra_body["plugins"] = [ - {"id": "pareto-router", "min_coding_score": _ps} - ] - - if summary_extra_body: - summary_kwargs["extra_body"] = summary_extra_body - - if self.api_mode == "anthropic_messages": - _tsum = self._get_transport() - _ant_kw = _tsum.build_kwargs(model=self.model, messages=api_messages, tools=None, - max_tokens=self.max_tokens, reasoning_config=self.reasoning_config, - is_oauth=self._is_anthropic_oauth, - preserve_dots=self._anthropic_preserve_dots()) - summary_response = self._anthropic_messages_create(_ant_kw) - _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=self._is_anthropic_oauth) - final_response = (_summary_result.content or "").strip() - else: - summary_response = self._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) - _summary_result = self._get_transport().normalize_response(summary_response) - final_response = (_summary_result.content or "").strip() - - if final_response: - if "" in final_response: - final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() - if final_response: - messages.append({"role": "assistant", "content": final_response}) - else: - final_response = "I reached the iteration limit and couldn't generate a summary." - else: - # Retry summary generation - if self.api_mode == "codex_responses": - codex_kwargs = self._build_api_kwargs(api_messages) - codex_kwargs.pop("tools", None) - retry_response = self._run_codex_stream(codex_kwargs) - _ct_retry = self._get_transport() - _cnr_retry = _ct_retry.normalize_response(retry_response) - final_response = (_cnr_retry.content or "").strip() - elif self.api_mode == "anthropic_messages": - _tretry = self._get_transport() - _ant_kw2 = _tretry.build_kwargs(model=self.model, messages=api_messages, tools=None, - is_oauth=self._is_anthropic_oauth, - max_tokens=self.max_tokens, reasoning_config=self.reasoning_config, - preserve_dots=self._anthropic_preserve_dots()) - retry_response = self._anthropic_messages_create(_ant_kw2) - _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=self._is_anthropic_oauth) - final_response = (_retry_result.content or "").strip() - else: - summary_kwargs = { - "model": self.model, - "messages": api_messages, - } - if _summary_temperature is not None: - summary_kwargs["temperature"] = _summary_temperature - if self.max_tokens is not None: - summary_kwargs.update(self._max_tokens_param(self.max_tokens)) - if _lm_reasoning_effort is not None: - summary_kwargs["reasoning_effort"] = _lm_reasoning_effort - if summary_extra_body: - summary_kwargs["extra_body"] = summary_extra_body - - summary_response = self._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) - _retry_result = self._get_transport().normalize_response(summary_response) - final_response = (_retry_result.content or "").strip() - - if final_response: - if "" in final_response: - final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() - if final_response: - messages.append({"role": "assistant", "content": final_response}) - else: - final_response = "I reached the iteration limit and couldn't generate a summary." - else: - final_response = "I reached the iteration limit and couldn't generate a summary." - - except Exception as e: - logging.warning(f"Failed to get summary response: {e}") - final_response = f"I reached the maximum iterations ({self.max_iterations}) but couldn't summarize. Error: {str(e)}" - - return final_response + """Forwarder — see ``agent.chat_completion_helpers.handle_max_iterations``.""" + from agent.chat_completion_helpers import handle_max_iterations + return handle_max_iterations(self, messages, api_call_count) def run_conversation( self, diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index dadb7b31cce..722de089628 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -4777,16 +4777,18 @@ class TestAnthropicInterruptHandler: def test_interruptible_has_anthropic_branch(self): """The interrupt handler must check api_mode == 'anthropic_messages'.""" import inspect - source = inspect.getsource(AIAgent._interruptible_api_call) + from agent.chat_completion_helpers import interruptible_api_call + source = inspect.getsource(interruptible_api_call) assert "anthropic_messages" in source, \ - "_interruptible_api_call must handle Anthropic interrupt (api_mode check)" + "interruptible_api_call must handle Anthropic interrupt (api_mode check)" def test_interruptible_rebuilds_anthropic_client(self): """After interrupting, the Anthropic client should be rebuilt.""" import inspect - source = inspect.getsource(AIAgent._interruptible_api_call) + from agent.chat_completion_helpers import interruptible_api_call + source = inspect.getsource(interruptible_api_call) assert "build_anthropic_client" in source, \ - "_interruptible_api_call must rebuild Anthropic client after interrupt" + "interruptible_api_call must rebuild Anthropic client after interrupt" def test_streaming_has_anthropic_branch(self): """_streaming_api_call must also handle Anthropic interrupt.""" From 0430e71ec971d673a6d2a32fd54c1847683a16d2 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 18:48:22 -0700 Subject: [PATCH 010/142] refactor(run_agent): extract streaming API caller (893 LOC) to agent/chat_completion_helpers.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move _interruptible_streaming_api_call out of run_agent.py — the biggest single method in the file. Body lives next to interruptible_api_call in agent/chat_completion_helpers.py so streaming + non-streaming code share one home. Nested closures (_call_chat_completions, _call_anthropic, the codex stream branch) all come along with the body and still capture the parent function's locals as expected. AIAgent keeps a thin forwarder method. is_local_endpoint added to the import block (used by the stream stale-timeout disable logic). One source-introspection test in TestAnthropicInterruptHandler is updated to scan agent.chat_completion_helpers.interruptible_streaming_api_call instead of AIAgent._interruptible_streaming_api_call. tests/run_agent/ + tests/agent/: 4312 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 12277 -> 11385 lines (-892). --- agent/chat_completion_helpers.py | 896 ++++++++++++++++++++++++++++++ run_agent.py | 892 +---------------------------- tests/run_agent/test_run_agent.py | 5 +- 3 files changed, 902 insertions(+), 891 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index fd1eb22ae4e..9616fefe0e4 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -35,6 +35,7 @@ from urllib.parse import urlparse, parse_qs, urlunparse from hermes_cli.timeouts import get_provider_request_timeout from agent.error_classifier import classify_api_error, FailoverReason +from agent.model_metadata import is_local_endpoint from agent.message_sanitization import ( _sanitize_surrogates, _sanitize_messages_surrogates, @@ -1122,6 +1123,900 @@ def cleanup_task_resources(agent, task_id: str) -> None: + +def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=None): + """Streaming variant of _interruptible_api_call for real-time token delivery. + + Handles all three api_modes: + - chat_completions: stream=True on OpenAI-compatible endpoints + - anthropic_messages: client.messages.stream() via Anthropic SDK + - codex_responses: delegates to _run_codex_stream (already streaming) + + Fires stream_delta_callback and _stream_callback for each text token. + Tool-call turns suppress the callback — only text-only final responses + stream to the consumer. Returns a SimpleNamespace that mimics the + non-streaming response shape so the rest of the agent loop is unchanged. + + Falls back to _interruptible_api_call on provider errors indicating + streaming is not supported. + """ + if agent._interrupt_requested: + raise InterruptedError("Agent interrupted before streaming API call") + + if agent.api_mode == "codex_responses": + # Codex streams internally via _run_codex_stream. The main dispatch + # in _interruptible_api_call already calls it; we just need to + # ensure on_first_delta reaches it. Store it on the instance + # temporarily so _run_codex_stream can pick it up. + agent._codex_on_first_delta = on_first_delta + try: + return agent._interruptible_api_call(api_kwargs) + finally: + agent._codex_on_first_delta = None + + # Bedrock Converse uses boto3's converse_stream() with real-time delta + # callbacks — same UX as Anthropic and chat_completions streaming. + if agent.api_mode == "bedrock_converse": + result = {"response": None, "error": None} + first_delta_fired = {"done": False} + deltas_were_sent = {"yes": False} + + def _fire_first(): + if not first_delta_fired["done"] and on_first_delta: + first_delta_fired["done"] = True + try: + on_first_delta() + except Exception: + pass + + def _bedrock_call(): + try: + from agent.bedrock_adapter import ( + _get_bedrock_runtime_client, + invalidate_runtime_client, + is_stale_connection_error, + stream_converse_with_callbacks, + ) + region = api_kwargs.pop("__bedrock_region__", "us-east-1") + api_kwargs.pop("__bedrock_converse__", None) + client = _get_bedrock_runtime_client(region) + try: + raw_response = client.converse_stream(**api_kwargs) + except Exception as _bedrock_exc: + # Evict the cached client on stale-connection failures + # so the outer retry loop builds a fresh client/pool. + if is_stale_connection_error(_bedrock_exc): + invalidate_runtime_client(region) + raise + + def _on_text(text): + _fire_first() + agent._fire_stream_delta(text) + deltas_were_sent["yes"] = True + + def _on_tool(name): + _fire_first() + agent._fire_tool_gen_started(name) + + def _on_reasoning(text): + _fire_first() + agent._fire_reasoning_delta(text) + + result["response"] = stream_converse_with_callbacks( + raw_response, + on_text_delta=_on_text if agent._has_stream_consumers() else None, + on_tool_start=_on_tool, + on_reasoning_delta=_on_reasoning if agent.reasoning_callback or agent.stream_delta_callback else None, + on_interrupt_check=lambda: agent._interrupt_requested, + ) + except Exception as e: + result["error"] = e + + t = threading.Thread(target=_bedrock_call, daemon=True) + t.start() + while t.is_alive(): + t.join(timeout=0.3) + if agent._interrupt_requested: + raise InterruptedError("Agent interrupted during Bedrock API call") + if result["error"] is not None: + raise result["error"] + return result["response"] + + result = {"response": None, "error": None, "partial_tool_names": []} + request_client_holder = {"client": None, "diag": None} + first_delta_fired = {"done": False} + deltas_were_sent = {"yes": False} # Track if any deltas were fired (for fallback) + # Wall-clock timestamp of the last real streaming chunk. The outer + # poll loop uses this to detect stale connections that keep receiving + # SSE keep-alive pings but no actual data. + last_chunk_time = {"t": time.time()} + + def _fire_first_delta(): + if not first_delta_fired["done"] and on_first_delta: + first_delta_fired["done"] = True + try: + on_first_delta() + except Exception: + pass + + def _call_chat_completions(): + """Stream a chat completions response.""" + import httpx as _httpx + # Per-provider / per-model request_timeout_seconds (from config.yaml) + # wins over the HERMES_API_TIMEOUT env default if the user set it. + _provider_timeout_cfg = get_provider_request_timeout(agent.provider, agent.model) + _base_timeout = ( + _provider_timeout_cfg + if _provider_timeout_cfg is not None + else float(os.getenv("HERMES_API_TIMEOUT", 1800.0)) + ) + # Read timeout: config wins here too. Otherwise use + # HERMES_STREAM_READ_TIMEOUT (default 120s) for cloud providers. + if _provider_timeout_cfg is not None: + _stream_read_timeout = _provider_timeout_cfg + else: + _stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0)) + # Local providers (Ollama, llama.cpp, vLLM) can take minutes for + # prefill on large contexts before producing the first token. + # Auto-increase the httpx read timeout unless the user explicitly + # overrode HERMES_STREAM_READ_TIMEOUT. + if _stream_read_timeout == 120.0 and agent.base_url and is_local_endpoint(agent.base_url): + _stream_read_timeout = _base_timeout + logger.debug( + "Local provider detected (%s) — stream read timeout raised to %.0fs", + agent.base_url, _stream_read_timeout, + ) + stream_kwargs = { + **api_kwargs, + "stream": True, + "stream_options": {"include_usage": True}, + "timeout": _httpx.Timeout( + connect=30.0, + read=_stream_read_timeout, + write=_base_timeout, + pool=30.0, + ), + } + request_client_holder["client"] = agent._create_request_openai_client( + reason="chat_completion_stream_request", + api_kwargs=stream_kwargs, + ) + # Reset stale-stream timer so the detector measures from this + # attempt's start, not a previous attempt's last chunk. + last_chunk_time["t"] = time.time() + agent._touch_activity("waiting for provider response (streaming)") + # Initialize per-attempt stream diagnostics so the retry block can + # reach for them after the stream dies. Lives on + # ``request_client_holder["diag"]`` for closure access. + _diag = agent._stream_diag_init() + request_client_holder["diag"] = _diag + stream = request_client_holder["client"].chat.completions.create(**stream_kwargs) + + # Capture rate limit headers from the initial HTTP response. + # The OpenAI SDK Stream object exposes the underlying httpx + # response via .response before any chunks are consumed. + agent._capture_rate_limits(getattr(stream, "response", None)) + # Snapshot diagnostic headers (cf-ray, x-openrouter-provider, etc.) + # so they survive even when the stream dies before any chunk + # arrives. Best-effort; never raises. + agent._stream_diag_capture_response(_diag, getattr(stream, "response", None)) + + # Log OpenRouter response cache status when present. + agent._check_openrouter_cache_status(getattr(stream, "response", None)) + + content_parts: list = [] + tool_calls_acc: dict = {} + tool_gen_notified: set = set() + # Ollama-compatible endpoints reuse index 0 for every tool call + # in a parallel batch, distinguishing them only by id. Track + # the last seen id per raw index so we can detect a new tool + # call starting at the same index and redirect it to a fresh slot. + _last_id_at_idx: dict = {} # raw_index -> last seen non-empty id + _active_slot_by_idx: dict = {} # raw_index -> current slot in tool_calls_acc + finish_reason = None + model_name = None + role = "assistant" + reasoning_parts: list = [] + usage_obj = None + for chunk in stream: + last_chunk_time["t"] = time.time() + agent._touch_activity("receiving stream response") + + # Update per-attempt diagnostic counters. Best-effort — + # failures are swallowed so the streaming hot path is never + # interrupted by diagnostic accounting. + try: + _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 + if _diag.get("first_chunk_at") is None: + _diag["first_chunk_at"] = last_chunk_time["t"] + # Approximate byte size from the chunk's repr — exact wire + # bytes aren't exposed by the SDK, but len(repr(chunk)) is + # a stable proxy for "how much content arrived" that + # survives stub provider differences. + try: + _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(chunk)) + except Exception: + pass + except Exception: + pass + + if agent._interrupt_requested: + break + + if not chunk.choices: + if hasattr(chunk, "model") and chunk.model: + model_name = chunk.model + # Usage comes in the final chunk with empty choices + if hasattr(chunk, "usage") and chunk.usage: + usage_obj = chunk.usage + continue + + delta = chunk.choices[0].delta + if hasattr(chunk, "model") and chunk.model: + model_name = chunk.model + + # Accumulate reasoning content + reasoning_text = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None) + if reasoning_text: + reasoning_parts.append(reasoning_text) + _fire_first_delta() + agent._fire_reasoning_delta(reasoning_text) + + # Accumulate text content — fire callback only when no tool calls + if delta and delta.content: + content_parts.append(delta.content) + if not tool_calls_acc: + _fire_first_delta() + agent._fire_stream_delta(delta.content) + deltas_were_sent["yes"] = True + # Tool calls suppress regular content streaming (avoids + # displaying chatty "I'll use the tool..." text alongside + # tool calls). But reasoning tags embedded in suppressed + # content should still reach the display — otherwise the + # reasoning box only appears as a post-response fallback, + # rendering it confusingly after the already-streamed + # response. Route suppressed content through the stream + # delta callback so its tag extraction can fire the + # reasoning display. Non-reasoning text is harmlessly + # suppressed by the CLI's _stream_delta when the stream + # box is already closed (tool boundary flush). + elif agent.stream_delta_callback: + try: + agent.stream_delta_callback(delta.content) + agent._record_streamed_assistant_text(delta.content) + except Exception: + pass + + # Accumulate tool call deltas — notify display on first name + if delta and delta.tool_calls: + for tc_delta in delta.tool_calls: + raw_idx = tc_delta.index if tc_delta.index is not None else 0 + delta_id = tc_delta.id or "" + + # Ollama fix: detect a new tool call reusing the same + # raw index (different id) and redirect to a fresh slot. + if raw_idx not in _active_slot_by_idx: + _active_slot_by_idx[raw_idx] = raw_idx + if ( + delta_id + and raw_idx in _last_id_at_idx + and delta_id != _last_id_at_idx[raw_idx] + ): + new_slot = max(tool_calls_acc, default=-1) + 1 + _active_slot_by_idx[raw_idx] = new_slot + if delta_id: + _last_id_at_idx[raw_idx] = delta_id + idx = _active_slot_by_idx[raw_idx] + + if idx not in tool_calls_acc: + tool_calls_acc[idx] = { + "id": tc_delta.id or "", + "type": "function", + "function": {"name": "", "arguments": ""}, + "extra_content": None, + } + entry = tool_calls_acc[idx] + if tc_delta.id: + entry["id"] = tc_delta.id + if tc_delta.function: + if tc_delta.function.name: + # Use assignment, not +=. Function names are + # atomic identifiers delivered complete in the + # first chunk (OpenAI spec). Some providers + # (MiniMax M2.7 via NVIDIA NIM) resend the full + # name in every chunk; concatenation would + # produce "read_fileread_file". Assignment + # (matching the OpenAI Node SDK / LiteLLM / + # Vercel AI patterns) is immune to this. + entry["function"]["name"] = tc_delta.function.name + if tc_delta.function.arguments: + entry["function"]["arguments"] += tc_delta.function.arguments + extra = getattr(tc_delta, "extra_content", None) + if extra is None and hasattr(tc_delta, "model_extra"): + extra = (tc_delta.model_extra or {}).get("extra_content") + if extra is not None: + if hasattr(extra, "model_dump"): + extra = extra.model_dump() + entry["extra_content"] = extra + # Fire once per tool when the full name is available + name = entry["function"]["name"] + if name and idx not in tool_gen_notified: + tool_gen_notified.add(idx) + _fire_first_delta() + agent._fire_tool_gen_started(name) + # Record the partial tool-call name so the outer + # stub-builder can surface a user-visible warning + # if streaming dies before this tool's arguments + # are fully delivered. Without this, a stall + # during tool-call JSON generation lets the stub + # at line ~6107 return `tool_calls=None`, silently + # discarding the attempted action. + result["partial_tool_names"].append(name) + + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + # Usage in the final chunk + if hasattr(chunk, "usage") and chunk.usage: + usage_obj = chunk.usage + + # Build mock response matching non-streaming shape + full_content = "".join(content_parts) or None + mock_tool_calls = None + has_truncated_tool_args = False + if tool_calls_acc: + mock_tool_calls = [] + for idx in sorted(tool_calls_acc): + tc = tool_calls_acc[idx] + arguments = tc["function"]["arguments"] + tool_name = tc["function"]["name"] or "?" + if arguments and arguments.strip(): + try: + json.loads(arguments) + except json.JSONDecodeError: + # Attempt repair before flagging as truncated. + # Models like GLM-5.1 via Ollama produce trailing + # commas, unclosed brackets, Python None, etc. + # Without repair, these hit the truncation handler + # and kill the session. _repair_tool_call_arguments + # returns "{}" for unrepairable args, which is far + # better than a crashed session. + repaired = _repair_tool_call_arguments(arguments, tool_name) + if repaired != "{}": + # Successfully repaired — use the fixed args + arguments = repaired + else: + # Unrepairable — flag for truncation handling + has_truncated_tool_args = True + mock_tool_calls.append(SimpleNamespace( + id=tc["id"], + type=tc["type"], + extra_content=tc.get("extra_content"), + function=SimpleNamespace( + name=tc["function"]["name"], + arguments=arguments, + ), + )) + + effective_finish_reason = finish_reason or "stop" + if has_truncated_tool_args: + effective_finish_reason = "length" + + full_reasoning = "".join(reasoning_parts) or None + mock_message = SimpleNamespace( + role=role, + content=full_content, + tool_calls=mock_tool_calls, + reasoning_content=full_reasoning, + ) + mock_choice = SimpleNamespace( + index=0, + message=mock_message, + finish_reason=effective_finish_reason, + ) + return SimpleNamespace( + id="stream-" + str(uuid.uuid4()), + model=model_name, + choices=[mock_choice], + usage=usage_obj, + ) + + def _call_anthropic(): + """Stream an Anthropic Messages API response. + + Fires delta callbacks for real-time token delivery, but returns + the native Anthropic Message object from get_final_message() so + the rest of the agent loop (validation, tool extraction, etc.) + works unchanged. + """ + has_tool_use = False + + # Reset stale-stream timer for this attempt + last_chunk_time["t"] = time.time() + # Per-attempt diagnostic dict for the retry block to consume. + _diag = agent._stream_diag_init() + request_client_holder["diag"] = _diag + # Use the Anthropic SDK's streaming context manager + with agent._anthropic_client.messages.stream(**api_kwargs) as stream: + # The Anthropic SDK exposes the raw httpx response on + # ``stream.response``. Snapshot diagnostic headers + # immediately so they survive a stream that dies before the + # first event. + try: + agent._stream_diag_capture_response( + _diag, getattr(stream, "response", None) + ) + except Exception: + pass + for event in stream: + # Update stale-stream timer on every event so the + # outer poll loop knows data is flowing. Without + # this, the detector kills healthy long-running + # Opus streams after 180 s even when events are + # actively arriving (the chat_completions path + # already does this at the top of its chunk loop). + last_chunk_time["t"] = time.time() + agent._touch_activity("receiving stream response") + + # Update per-attempt diagnostic counters (best-effort). + try: + _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 + if _diag.get("first_chunk_at") is None: + _diag["first_chunk_at"] = last_chunk_time["t"] + try: + _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) + except Exception: + pass + except Exception: + pass + + if agent._interrupt_requested: + break + + event_type = getattr(event, "type", None) + + if event_type == "content_block_start": + block = getattr(event, "content_block", None) + if block and getattr(block, "type", None) == "tool_use": + has_tool_use = True + tool_name = getattr(block, "name", None) + if tool_name: + _fire_first_delta() + agent._fire_tool_gen_started(tool_name) + + elif event_type == "content_block_delta": + delta = getattr(event, "delta", None) + if delta: + delta_type = getattr(delta, "type", None) + if delta_type == "text_delta": + text = getattr(delta, "text", "") + if text and not has_tool_use: + _fire_first_delta() + agent._fire_stream_delta(text) + deltas_were_sent["yes"] = True + elif delta_type == "thinking_delta": + thinking_text = getattr(delta, "thinking", "") + if thinking_text: + _fire_first_delta() + agent._fire_reasoning_delta(thinking_text) + + # Return the native Anthropic Message for downstream processing + return stream.get_final_message() + + def _call(): + import httpx as _httpx + + _max_stream_retries = int(os.getenv("HERMES_STREAM_RETRIES", 2)) + + try: + for _stream_attempt in range(_max_stream_retries + 1): + # Check for interrupt before each retry attempt. Without + # this, /stop closes the HTTP connection (outer poll loop), + # but the retry loop opens a FRESH connection — negating the + # interrupt entirely. On slow providers (ollama-cloud) each + # retry can block for the full stream-read timeout (120s+), + # causing multi-minute delays between /stop and response. + if agent._interrupt_requested: + raise InterruptedError("Agent interrupted before stream retry") + try: + if agent.api_mode == "anthropic_messages": + agent._try_refresh_anthropic_client_credentials() + result["response"] = _call_anthropic() + else: + result["response"] = _call_chat_completions() + return # success + except Exception as e: + _is_timeout = isinstance( + e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout) + ) + _is_conn_err = isinstance( + e, (_httpx.ConnectError, _httpx.RemoteProtocolError, ConnectionError) + ) + + # If the stream died AFTER some tokens were delivered: + # normally we don't retry (the user already saw text, + # retrying would duplicate it). BUT: if a tool call + # was in-flight when the stream died, silently aborting + # discards the tool call entirely. In that case we + # prefer to retry — the user sees a brief + # "reconnecting" marker + duplicated preamble text, + # which is strictly better than a failed action with + # a "retry manually" message. Limit this to transient + # connection errors (Clawdbot-style narrow gate): no + # tool has executed yet within this API call, so + # silent retry is safe wrt side-effects. + if deltas_were_sent["yes"]: + _partial_tool_in_flight = bool( + result.get("partial_tool_names") + ) + _is_sse_conn_err_preview = False + if not _is_timeout and not _is_conn_err: + from openai import APIError as _APIError + if isinstance(e, _APIError) and not getattr(e, "status_code", None): + _err_lower_preview = str(e).lower() + _SSE_PREVIEW_PHRASES = ( + "connection lost", + "connection reset", + "connection closed", + "connection terminated", + "network error", + "network connection", + "terminated", + "peer closed", + "broken pipe", + "upstream connect error", + ) + _is_sse_conn_err_preview = any( + phrase in _err_lower_preview + for phrase in _SSE_PREVIEW_PHRASES + ) + _is_transient = ( + _is_timeout or _is_conn_err or _is_sse_conn_err_preview + ) + _can_silent_retry = ( + _partial_tool_in_flight + and _is_transient + and _stream_attempt < _max_stream_retries + ) + if not _can_silent_retry: + # Either no tool call was in-flight (so the + # turn was a pure text response — current + # stub-with-recovered-text behaviour is + # correct), or retries are exhausted, or the + # error isn't transient. Fall through to the + # stub path. + logger.warning( + "Streaming failed after partial delivery, not retrying: %s", e + ) + result["error"] = e + return + # Tool call was in-flight AND error is transient: + # retry silently. Clear per-attempt state so the + # next stream starts clean. Fire a "reconnecting" + # marker so the user sees why the preamble is + # about to be re-streamed. Structured WARNING is + # emitted by ``_emit_stream_drop`` below; no + # additional INFO line needed. + try: + agent._fire_stream_delta( + "\n\n⚠ Connection dropped mid tool-call; " + "reconnecting…\n\n" + ) + except Exception: + pass + # Reset the streamed-text buffer so the retry's + # fresh preamble doesn't get double-recorded in + # _current_streamed_assistant_text (which would + # pollute the interim-visible-text comparison). + try: + agent._reset_stream_delivery_tracking() + except Exception: + pass + # Reset in-memory accumulators so the next + # attempt's chunks don't concat onto the dead + # stream's partial JSON. + result["partial_tool_names"] = [] + deltas_were_sent["yes"] = False + first_delta_fired["done"] = False + agent._emit_stream_drop( + error=e, + attempt=_stream_attempt + 2, + max_attempts=_max_stream_retries + 1, + mid_tool_call=True, + diag=request_client_holder.get("diag"), + ) + stale = request_client_holder.get("client") + if stale is not None: + agent._close_request_openai_client( + stale, reason="stream_mid_tool_retry_cleanup" + ) + request_client_holder["client"] = None + try: + agent._replace_primary_openai_client( + reason="stream_mid_tool_retry_pool_cleanup" + ) + except Exception: + pass + continue + + # SSE error events from proxies (e.g. OpenRouter sends + # {"error":{"message":"Network connection lost."}}) are + # raised as APIError by the OpenAI SDK. These are + # semantically identical to httpx connection drops — + # the upstream stream died — and should be retried with + # a fresh connection. Distinguish from HTTP errors: + # APIError from SSE has no status_code, while + # APIStatusError (4xx/5xx) always has one. + _is_sse_conn_err = False + if not _is_timeout and not _is_conn_err: + from openai import APIError as _APIError + if isinstance(e, _APIError) and not getattr(e, "status_code", None): + _err_lower_sse = str(e).lower() + _SSE_CONN_PHRASES = ( + "connection lost", + "connection reset", + "connection closed", + "connection terminated", + "network error", + "network connection", + "terminated", + "peer closed", + "broken pipe", + "upstream connect error", + ) + _is_sse_conn_err = any( + phrase in _err_lower_sse + for phrase in _SSE_CONN_PHRASES + ) + + if _is_timeout or _is_conn_err or _is_sse_conn_err: + # Transient network / timeout error. Retry the + # streaming request with a fresh connection first. + if _stream_attempt < _max_stream_retries: + agent._emit_stream_drop( + error=e, + attempt=_stream_attempt + 2, + max_attempts=_max_stream_retries + 1, + mid_tool_call=False, + diag=request_client_holder.get("diag"), + ) + # Close the stale request client before retry + stale = request_client_holder.get("client") + if stale is not None: + agent._close_request_openai_client( + stale, reason="stream_retry_cleanup" + ) + request_client_holder["client"] = None + # Also rebuild the primary client to purge + # any dead connections from the pool. + try: + agent._replace_primary_openai_client( + reason="stream_retry_pool_cleanup" + ) + except Exception: + pass + continue + # Retries exhausted. Log the final failure with + # full diagnostic detail (chain, headers, + # bytes/elapsed) via the same helper used for + # mid-flight retries — subagent lines get the + # ``[subagent-N]`` log_prefix so the parent can + # attribute them. + agent._log_stream_retry( + kind="exhausted", + error=e, + attempt=_max_stream_retries + 1, + max_attempts=_max_stream_retries + 1, + mid_tool_call=False, + diag=request_client_holder.get("diag"), + ) + agent._emit_status( + "❌ Connection to provider failed after " + f"{_max_stream_retries + 1} attempts. " + "The provider may be experiencing issues — " + "try again in a moment." + ) + else: + _err_lower = str(e).lower() + _is_stream_unsupported = ( + "stream" in _err_lower + and "not supported" in _err_lower + ) + if _is_stream_unsupported: + agent._disable_streaming = True + agent._safe_print( + "\n⚠ Streaming is not supported for this " + "model/provider. Switching to non-streaming.\n" + " To avoid this delay, set display.streaming: false " + "in config.yaml\n" + ) + logger.info( + "Streaming failed before delivery: %s", + e, + ) + + # Propagate the error to the main retry loop instead of + # falling back to non-streaming inline. The main loop has + # richer recovery: credential rotation, provider fallback, + # backoff, and — for "stream not supported" — will switch + # to non-streaming on the next attempt via _disable_streaming. + result["error"] = e + return + except InterruptedError as e: + # The interrupt may be noticed inside the worker thread before + # the polling loop sees it. Surface it through the normal result + # channel so callers never miss a fast pre-retry interrupt. + result["error"] = e + return + finally: + request_client = request_client_holder.get("client") + if request_client is not None: + agent._close_request_openai_client(request_client, reason="stream_request_complete") + + _stream_stale_timeout_base = float(os.getenv("HERMES_STREAM_STALE_TIMEOUT", 180.0)) + # Local providers (Ollama, oMLX, llama-cpp) can take 300+ seconds + # for prefill on large contexts. Disable the stale detector unless + # the user explicitly set HERMES_STREAM_STALE_TIMEOUT. + if _stream_stale_timeout_base == 180.0 and agent.base_url and is_local_endpoint(agent.base_url): + _stream_stale_timeout = float("inf") + logger.debug("Local provider detected (%s) — stale stream timeout disabled", agent.base_url) + else: + # Scale the stale timeout for large contexts: slow models (like Opus) + # can legitimately think for minutes before producing the first token + # when the context is large. Without this, the stale detector kills + # healthy connections during the model's thinking phase, producing + # spurious RemoteProtocolError ("peer closed connection"). + _est_tokens = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 + if _est_tokens > 100_000: + _stream_stale_timeout = max(_stream_stale_timeout_base, 300.0) + elif _est_tokens > 50_000: + _stream_stale_timeout = max(_stream_stale_timeout_base, 240.0) + else: + _stream_stale_timeout = _stream_stale_timeout_base + + t = threading.Thread(target=_call, daemon=True) + t.start() + _last_heartbeat = time.time() + _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches + while t.is_alive(): + t.join(timeout=0.3) + + # Periodic heartbeat: touch the agent's activity tracker so the + # gateway's inactivity monitor knows we're alive while waiting + # for stream chunks. Without this, long thinking pauses (e.g. + # reasoning models) or slow prefill on local providers (Ollama) + # trigger false inactivity timeouts. The _call thread touches + # activity on each chunk, but the gap between API call start + # and first chunk can exceed the gateway timeout — especially + # when the stale-stream timeout is disabled (local providers). + _hb_now = time.time() + if _hb_now - _last_heartbeat >= _HEARTBEAT_INTERVAL: + _last_heartbeat = _hb_now + _waiting_secs = int(_hb_now - last_chunk_time["t"]) + agent._touch_activity( + f"waiting for stream response ({_waiting_secs}s, no chunks yet)" + ) + + # Detect stale streams: connections kept alive by SSE pings + # but delivering no real chunks. Kill the client so the + # inner retry loop can start a fresh connection. + _stale_elapsed = time.time() - last_chunk_time["t"] + if _stale_elapsed > _stream_stale_timeout: + _est_ctx = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 + logger.warning( + "Stream stale for %.0fs (threshold %.0fs) — no chunks received. " + "model=%s context=~%s tokens. Killing connection.", + _stale_elapsed, _stream_stale_timeout, + api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", + ) + agent._emit_status( + f"⚠️ No response from provider for {int(_stale_elapsed)}s " + f"(model: {api_kwargs.get('model', 'unknown')}, " + f"context: ~{_est_ctx:,} tokens). " + f"Reconnecting..." + ) + try: + rc = request_client_holder.get("client") + if rc is not None: + agent._close_request_openai_client(rc, reason="stale_stream_kill") + except Exception: + pass + # Rebuild the primary client too — its connection pool + # may hold dead sockets from the same provider outage. + try: + agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup") + except Exception: + pass + # Reset the timer so we don't kill repeatedly while + # the inner thread processes the closure. + last_chunk_time["t"] = time.time() + agent._touch_activity( + f"stale stream detected after {int(_stale_elapsed)}s, reconnecting" + ) + + if agent._interrupt_requested: + try: + if agent.api_mode == "anthropic_messages": + agent._anthropic_client.close() + agent._rebuild_anthropic_client() + else: + request_client = request_client_holder.get("client") + if request_client is not None: + agent._close_request_openai_client(request_client, reason="stream_interrupt_abort") + except Exception: + pass + raise InterruptedError("Agent interrupted during streaming API call") + if result["error"] is not None: + if deltas_were_sent["yes"]: + # Streaming failed AFTER some tokens were already delivered to + # the platform. Re-raising would let the outer retry loop make + # a new API call, creating a duplicate message. Return a + # partial "stop" response instead so the outer loop treats this + # turn as complete (no retry, no fallback). + # Recover whatever content was already streamed to the user. + # _current_streamed_assistant_text accumulates text fired + # through _fire_stream_delta, so it has exactly what the + # user saw before the connection died. + _partial_text = ( + getattr(agent, "_current_streamed_assistant_text", "") or "" + ).strip() or None + + # If the stream died while the model was emitting a tool call, + # the stub below will silently set `tool_calls=None` and the + # agent loop will treat the turn as complete — the attempted + # action is lost with no user-facing signal. Append a + # human-visible warning to the stub content so (a) the user + # knows something failed, and (b) the next turn's model sees + # in conversation history what was attempted and can retry. + _partial_names = list(result.get("partial_tool_names") or []) + if _partial_names: + _name_str = ", ".join(_partial_names[:3]) + if len(_partial_names) > 3: + _name_str += f", +{len(_partial_names) - 3} more" + _warn = ( + f"\n\n⚠ Stream stalled mid tool-call " + f"({_name_str}); the action was not executed. " + f"Ask me to retry if you want to continue." + ) + _partial_text = (_partial_text or "") + _warn + # Also fire as a streaming delta so the user sees it now + # instead of only in the persisted transcript. + try: + agent._fire_stream_delta(_warn) + except Exception: + pass + logger.warning( + "Partial stream dropped tool call(s) %s after %s chars " + "of text; surfaced warning to user: %s", + _partial_names, len(_partial_text or ""), result["error"], + ) + else: + logger.warning( + "Partial stream delivered before error; returning stub " + "response with %s chars of recovered content to prevent " + "duplicate messages: %s", + len(_partial_text or ""), + result["error"], + ) + _stub_msg = SimpleNamespace( + role="assistant", content=_partial_text, tool_calls=None, + reasoning_content=None, + ) + return SimpleNamespace( + id="partial-stream-stub", + model=getattr(agent, "model", "unknown"), + choices=[SimpleNamespace( + index=0, message=_stub_msg, finish_reason="stop", + )], + usage=None, + ) + raise result["error"] + return result["response"] + +# ── Provider fallback ────────────────────────────────────────────────── + + + __all__ = [ "interruptible_api_call", "build_api_kwargs", @@ -1129,4 +2024,5 @@ __all__ = [ "try_activate_fallback", "handle_max_iterations", "cleanup_task_resources", + "interruptible_streaming_api_call", ] diff --git a/run_agent.py b/run_agent.py index 9ee4a0b7bbb..4b5e405018e 100644 --- a/run_agent.py +++ b/run_agent.py @@ -5953,895 +5953,9 @@ class AIAgent: def _interruptible_streaming_api_call( self, api_kwargs: dict, *, on_first_delta: callable = None ): - """Streaming variant of _interruptible_api_call for real-time token delivery. - - Handles all three api_modes: - - chat_completions: stream=True on OpenAI-compatible endpoints - - anthropic_messages: client.messages.stream() via Anthropic SDK - - codex_responses: delegates to _run_codex_stream (already streaming) - - Fires stream_delta_callback and _stream_callback for each text token. - Tool-call turns suppress the callback — only text-only final responses - stream to the consumer. Returns a SimpleNamespace that mimics the - non-streaming response shape so the rest of the agent loop is unchanged. - - Falls back to _interruptible_api_call on provider errors indicating - streaming is not supported. - """ - if self._interrupt_requested: - raise InterruptedError("Agent interrupted before streaming API call") - - if self.api_mode == "codex_responses": - # Codex streams internally via _run_codex_stream. The main dispatch - # in _interruptible_api_call already calls it; we just need to - # ensure on_first_delta reaches it. Store it on the instance - # temporarily so _run_codex_stream can pick it up. - self._codex_on_first_delta = on_first_delta - try: - return self._interruptible_api_call(api_kwargs) - finally: - self._codex_on_first_delta = None - - # Bedrock Converse uses boto3's converse_stream() with real-time delta - # callbacks — same UX as Anthropic and chat_completions streaming. - if self.api_mode == "bedrock_converse": - result = {"response": None, "error": None} - first_delta_fired = {"done": False} - deltas_were_sent = {"yes": False} - - def _fire_first(): - if not first_delta_fired["done"] and on_first_delta: - first_delta_fired["done"] = True - try: - on_first_delta() - except Exception: - pass - - def _bedrock_call(): - try: - from agent.bedrock_adapter import ( - _get_bedrock_runtime_client, - invalidate_runtime_client, - is_stale_connection_error, - stream_converse_with_callbacks, - ) - region = api_kwargs.pop("__bedrock_region__", "us-east-1") - api_kwargs.pop("__bedrock_converse__", None) - client = _get_bedrock_runtime_client(region) - try: - raw_response = client.converse_stream(**api_kwargs) - except Exception as _bedrock_exc: - # Evict the cached client on stale-connection failures - # so the outer retry loop builds a fresh client/pool. - if is_stale_connection_error(_bedrock_exc): - invalidate_runtime_client(region) - raise - - def _on_text(text): - _fire_first() - self._fire_stream_delta(text) - deltas_were_sent["yes"] = True - - def _on_tool(name): - _fire_first() - self._fire_tool_gen_started(name) - - def _on_reasoning(text): - _fire_first() - self._fire_reasoning_delta(text) - - result["response"] = stream_converse_with_callbacks( - raw_response, - on_text_delta=_on_text if self._has_stream_consumers() else None, - on_tool_start=_on_tool, - on_reasoning_delta=_on_reasoning if self.reasoning_callback or self.stream_delta_callback else None, - on_interrupt_check=lambda: self._interrupt_requested, - ) - except Exception as e: - result["error"] = e - - t = threading.Thread(target=_bedrock_call, daemon=True) - t.start() - while t.is_alive(): - t.join(timeout=0.3) - if self._interrupt_requested: - raise InterruptedError("Agent interrupted during Bedrock API call") - if result["error"] is not None: - raise result["error"] - return result["response"] - - result = {"response": None, "error": None, "partial_tool_names": []} - request_client_holder = {"client": None, "diag": None} - first_delta_fired = {"done": False} - deltas_were_sent = {"yes": False} # Track if any deltas were fired (for fallback) - # Wall-clock timestamp of the last real streaming chunk. The outer - # poll loop uses this to detect stale connections that keep receiving - # SSE keep-alive pings but no actual data. - last_chunk_time = {"t": time.time()} - - def _fire_first_delta(): - if not first_delta_fired["done"] and on_first_delta: - first_delta_fired["done"] = True - try: - on_first_delta() - except Exception: - pass - - def _call_chat_completions(): - """Stream a chat completions response.""" - import httpx as _httpx - # Per-provider / per-model request_timeout_seconds (from config.yaml) - # wins over the HERMES_API_TIMEOUT env default if the user set it. - _provider_timeout_cfg = get_provider_request_timeout(self.provider, self.model) - _base_timeout = ( - _provider_timeout_cfg - if _provider_timeout_cfg is not None - else float(os.getenv("HERMES_API_TIMEOUT", 1800.0)) - ) - # Read timeout: config wins here too. Otherwise use - # HERMES_STREAM_READ_TIMEOUT (default 120s) for cloud providers. - if _provider_timeout_cfg is not None: - _stream_read_timeout = _provider_timeout_cfg - else: - _stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0)) - # Local providers (Ollama, llama.cpp, vLLM) can take minutes for - # prefill on large contexts before producing the first token. - # Auto-increase the httpx read timeout unless the user explicitly - # overrode HERMES_STREAM_READ_TIMEOUT. - if _stream_read_timeout == 120.0 and self.base_url and is_local_endpoint(self.base_url): - _stream_read_timeout = _base_timeout - logger.debug( - "Local provider detected (%s) — stream read timeout raised to %.0fs", - self.base_url, _stream_read_timeout, - ) - stream_kwargs = { - **api_kwargs, - "stream": True, - "stream_options": {"include_usage": True}, - "timeout": _httpx.Timeout( - connect=30.0, - read=_stream_read_timeout, - write=_base_timeout, - pool=30.0, - ), - } - request_client_holder["client"] = self._create_request_openai_client( - reason="chat_completion_stream_request", - api_kwargs=stream_kwargs, - ) - # Reset stale-stream timer so the detector measures from this - # attempt's start, not a previous attempt's last chunk. - last_chunk_time["t"] = time.time() - self._touch_activity("waiting for provider response (streaming)") - # Initialize per-attempt stream diagnostics so the retry block can - # reach for them after the stream dies. Lives on - # ``request_client_holder["diag"]`` for closure access. - _diag = self._stream_diag_init() - request_client_holder["diag"] = _diag - stream = request_client_holder["client"].chat.completions.create(**stream_kwargs) - - # Capture rate limit headers from the initial HTTP response. - # The OpenAI SDK Stream object exposes the underlying httpx - # response via .response before any chunks are consumed. - self._capture_rate_limits(getattr(stream, "response", None)) - # Snapshot diagnostic headers (cf-ray, x-openrouter-provider, etc.) - # so they survive even when the stream dies before any chunk - # arrives. Best-effort; never raises. - self._stream_diag_capture_response(_diag, getattr(stream, "response", None)) - - # Log OpenRouter response cache status when present. - self._check_openrouter_cache_status(getattr(stream, "response", None)) - - content_parts: list = [] - tool_calls_acc: dict = {} - tool_gen_notified: set = set() - # Ollama-compatible endpoints reuse index 0 for every tool call - # in a parallel batch, distinguishing them only by id. Track - # the last seen id per raw index so we can detect a new tool - # call starting at the same index and redirect it to a fresh slot. - _last_id_at_idx: dict = {} # raw_index -> last seen non-empty id - _active_slot_by_idx: dict = {} # raw_index -> current slot in tool_calls_acc - finish_reason = None - model_name = None - role = "assistant" - reasoning_parts: list = [] - usage_obj = None - for chunk in stream: - last_chunk_time["t"] = time.time() - self._touch_activity("receiving stream response") - - # Update per-attempt diagnostic counters. Best-effort — - # failures are swallowed so the streaming hot path is never - # interrupted by diagnostic accounting. - try: - _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 - if _diag.get("first_chunk_at") is None: - _diag["first_chunk_at"] = last_chunk_time["t"] - # Approximate byte size from the chunk's repr — exact wire - # bytes aren't exposed by the SDK, but len(repr(chunk)) is - # a stable proxy for "how much content arrived" that - # survives stub provider differences. - try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(chunk)) - except Exception: - pass - except Exception: - pass - - if self._interrupt_requested: - break - - if not chunk.choices: - if hasattr(chunk, "model") and chunk.model: - model_name = chunk.model - # Usage comes in the final chunk with empty choices - if hasattr(chunk, "usage") and chunk.usage: - usage_obj = chunk.usage - continue - - delta = chunk.choices[0].delta - if hasattr(chunk, "model") and chunk.model: - model_name = chunk.model - - # Accumulate reasoning content - reasoning_text = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None) - if reasoning_text: - reasoning_parts.append(reasoning_text) - _fire_first_delta() - self._fire_reasoning_delta(reasoning_text) - - # Accumulate text content — fire callback only when no tool calls - if delta and delta.content: - content_parts.append(delta.content) - if not tool_calls_acc: - _fire_first_delta() - self._fire_stream_delta(delta.content) - deltas_were_sent["yes"] = True - # Tool calls suppress regular content streaming (avoids - # displaying chatty "I'll use the tool..." text alongside - # tool calls). But reasoning tags embedded in suppressed - # content should still reach the display — otherwise the - # reasoning box only appears as a post-response fallback, - # rendering it confusingly after the already-streamed - # response. Route suppressed content through the stream - # delta callback so its tag extraction can fire the - # reasoning display. Non-reasoning text is harmlessly - # suppressed by the CLI's _stream_delta when the stream - # box is already closed (tool boundary flush). - elif self.stream_delta_callback: - try: - self.stream_delta_callback(delta.content) - self._record_streamed_assistant_text(delta.content) - except Exception: - pass - - # Accumulate tool call deltas — notify display on first name - if delta and delta.tool_calls: - for tc_delta in delta.tool_calls: - raw_idx = tc_delta.index if tc_delta.index is not None else 0 - delta_id = tc_delta.id or "" - - # Ollama fix: detect a new tool call reusing the same - # raw index (different id) and redirect to a fresh slot. - if raw_idx not in _active_slot_by_idx: - _active_slot_by_idx[raw_idx] = raw_idx - if ( - delta_id - and raw_idx in _last_id_at_idx - and delta_id != _last_id_at_idx[raw_idx] - ): - new_slot = max(tool_calls_acc, default=-1) + 1 - _active_slot_by_idx[raw_idx] = new_slot - if delta_id: - _last_id_at_idx[raw_idx] = delta_id - idx = _active_slot_by_idx[raw_idx] - - if idx not in tool_calls_acc: - tool_calls_acc[idx] = { - "id": tc_delta.id or "", - "type": "function", - "function": {"name": "", "arguments": ""}, - "extra_content": None, - } - entry = tool_calls_acc[idx] - if tc_delta.id: - entry["id"] = tc_delta.id - if tc_delta.function: - if tc_delta.function.name: - # Use assignment, not +=. Function names are - # atomic identifiers delivered complete in the - # first chunk (OpenAI spec). Some providers - # (MiniMax M2.7 via NVIDIA NIM) resend the full - # name in every chunk; concatenation would - # produce "read_fileread_file". Assignment - # (matching the OpenAI Node SDK / LiteLLM / - # Vercel AI patterns) is immune to this. - entry["function"]["name"] = tc_delta.function.name - if tc_delta.function.arguments: - entry["function"]["arguments"] += tc_delta.function.arguments - extra = getattr(tc_delta, "extra_content", None) - if extra is None and hasattr(tc_delta, "model_extra"): - extra = (tc_delta.model_extra or {}).get("extra_content") - if extra is not None: - if hasattr(extra, "model_dump"): - extra = extra.model_dump() - entry["extra_content"] = extra - # Fire once per tool when the full name is available - name = entry["function"]["name"] - if name and idx not in tool_gen_notified: - tool_gen_notified.add(idx) - _fire_first_delta() - self._fire_tool_gen_started(name) - # Record the partial tool-call name so the outer - # stub-builder can surface a user-visible warning - # if streaming dies before this tool's arguments - # are fully delivered. Without this, a stall - # during tool-call JSON generation lets the stub - # at line ~6107 return `tool_calls=None`, silently - # discarding the attempted action. - result["partial_tool_names"].append(name) - - if chunk.choices[0].finish_reason: - finish_reason = chunk.choices[0].finish_reason - - # Usage in the final chunk - if hasattr(chunk, "usage") and chunk.usage: - usage_obj = chunk.usage - - # Build mock response matching non-streaming shape - full_content = "".join(content_parts) or None - mock_tool_calls = None - has_truncated_tool_args = False - if tool_calls_acc: - mock_tool_calls = [] - for idx in sorted(tool_calls_acc): - tc = tool_calls_acc[idx] - arguments = tc["function"]["arguments"] - tool_name = tc["function"]["name"] or "?" - if arguments and arguments.strip(): - try: - json.loads(arguments) - except json.JSONDecodeError: - # Attempt repair before flagging as truncated. - # Models like GLM-5.1 via Ollama produce trailing - # commas, unclosed brackets, Python None, etc. - # Without repair, these hit the truncation handler - # and kill the session. _repair_tool_call_arguments - # returns "{}" for unrepairable args, which is far - # better than a crashed session. - repaired = _repair_tool_call_arguments(arguments, tool_name) - if repaired != "{}": - # Successfully repaired — use the fixed args - arguments = repaired - else: - # Unrepairable — flag for truncation handling - has_truncated_tool_args = True - mock_tool_calls.append(SimpleNamespace( - id=tc["id"], - type=tc["type"], - extra_content=tc.get("extra_content"), - function=SimpleNamespace( - name=tc["function"]["name"], - arguments=arguments, - ), - )) - - effective_finish_reason = finish_reason or "stop" - if has_truncated_tool_args: - effective_finish_reason = "length" - - full_reasoning = "".join(reasoning_parts) or None - mock_message = SimpleNamespace( - role=role, - content=full_content, - tool_calls=mock_tool_calls, - reasoning_content=full_reasoning, - ) - mock_choice = SimpleNamespace( - index=0, - message=mock_message, - finish_reason=effective_finish_reason, - ) - return SimpleNamespace( - id="stream-" + str(uuid.uuid4()), - model=model_name, - choices=[mock_choice], - usage=usage_obj, - ) - - def _call_anthropic(): - """Stream an Anthropic Messages API response. - - Fires delta callbacks for real-time token delivery, but returns - the native Anthropic Message object from get_final_message() so - the rest of the agent loop (validation, tool extraction, etc.) - works unchanged. - """ - has_tool_use = False - - # Reset stale-stream timer for this attempt - last_chunk_time["t"] = time.time() - # Per-attempt diagnostic dict for the retry block to consume. - _diag = self._stream_diag_init() - request_client_holder["diag"] = _diag - # Use the Anthropic SDK's streaming context manager - with self._anthropic_client.messages.stream(**api_kwargs) as stream: - # The Anthropic SDK exposes the raw httpx response on - # ``stream.response``. Snapshot diagnostic headers - # immediately so they survive a stream that dies before the - # first event. - try: - self._stream_diag_capture_response( - _diag, getattr(stream, "response", None) - ) - except Exception: - pass - for event in stream: - # Update stale-stream timer on every event so the - # outer poll loop knows data is flowing. Without - # this, the detector kills healthy long-running - # Opus streams after 180 s even when events are - # actively arriving (the chat_completions path - # already does this at the top of its chunk loop). - last_chunk_time["t"] = time.time() - self._touch_activity("receiving stream response") - - # Update per-attempt diagnostic counters (best-effort). - try: - _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 - if _diag.get("first_chunk_at") is None: - _diag["first_chunk_at"] = last_chunk_time["t"] - try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) - except Exception: - pass - except Exception: - pass - - if self._interrupt_requested: - break - - event_type = getattr(event, "type", None) - - if event_type == "content_block_start": - block = getattr(event, "content_block", None) - if block and getattr(block, "type", None) == "tool_use": - has_tool_use = True - tool_name = getattr(block, "name", None) - if tool_name: - _fire_first_delta() - self._fire_tool_gen_started(tool_name) - - elif event_type == "content_block_delta": - delta = getattr(event, "delta", None) - if delta: - delta_type = getattr(delta, "type", None) - if delta_type == "text_delta": - text = getattr(delta, "text", "") - if text and not has_tool_use: - _fire_first_delta() - self._fire_stream_delta(text) - deltas_were_sent["yes"] = True - elif delta_type == "thinking_delta": - thinking_text = getattr(delta, "thinking", "") - if thinking_text: - _fire_first_delta() - self._fire_reasoning_delta(thinking_text) - - # Return the native Anthropic Message for downstream processing - return stream.get_final_message() - - def _call(): - import httpx as _httpx - - _max_stream_retries = int(os.getenv("HERMES_STREAM_RETRIES", 2)) - - try: - for _stream_attempt in range(_max_stream_retries + 1): - # Check for interrupt before each retry attempt. Without - # this, /stop closes the HTTP connection (outer poll loop), - # but the retry loop opens a FRESH connection — negating the - # interrupt entirely. On slow providers (ollama-cloud) each - # retry can block for the full stream-read timeout (120s+), - # causing multi-minute delays between /stop and response. - if self._interrupt_requested: - raise InterruptedError("Agent interrupted before stream retry") - try: - if self.api_mode == "anthropic_messages": - self._try_refresh_anthropic_client_credentials() - result["response"] = _call_anthropic() - else: - result["response"] = _call_chat_completions() - return # success - except Exception as e: - _is_timeout = isinstance( - e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout) - ) - _is_conn_err = isinstance( - e, (_httpx.ConnectError, _httpx.RemoteProtocolError, ConnectionError) - ) - - # If the stream died AFTER some tokens were delivered: - # normally we don't retry (the user already saw text, - # retrying would duplicate it). BUT: if a tool call - # was in-flight when the stream died, silently aborting - # discards the tool call entirely. In that case we - # prefer to retry — the user sees a brief - # "reconnecting" marker + duplicated preamble text, - # which is strictly better than a failed action with - # a "retry manually" message. Limit this to transient - # connection errors (Clawdbot-style narrow gate): no - # tool has executed yet within this API call, so - # silent retry is safe wrt side-effects. - if deltas_were_sent["yes"]: - _partial_tool_in_flight = bool( - result.get("partial_tool_names") - ) - _is_sse_conn_err_preview = False - if not _is_timeout and not _is_conn_err: - from openai import APIError as _APIError - if isinstance(e, _APIError) and not getattr(e, "status_code", None): - _err_lower_preview = str(e).lower() - _SSE_PREVIEW_PHRASES = ( - "connection lost", - "connection reset", - "connection closed", - "connection terminated", - "network error", - "network connection", - "terminated", - "peer closed", - "broken pipe", - "upstream connect error", - ) - _is_sse_conn_err_preview = any( - phrase in _err_lower_preview - for phrase in _SSE_PREVIEW_PHRASES - ) - _is_transient = ( - _is_timeout or _is_conn_err or _is_sse_conn_err_preview - ) - _can_silent_retry = ( - _partial_tool_in_flight - and _is_transient - and _stream_attempt < _max_stream_retries - ) - if not _can_silent_retry: - # Either no tool call was in-flight (so the - # turn was a pure text response — current - # stub-with-recovered-text behaviour is - # correct), or retries are exhausted, or the - # error isn't transient. Fall through to the - # stub path. - logger.warning( - "Streaming failed after partial delivery, not retrying: %s", e - ) - result["error"] = e - return - # Tool call was in-flight AND error is transient: - # retry silently. Clear per-attempt state so the - # next stream starts clean. Fire a "reconnecting" - # marker so the user sees why the preamble is - # about to be re-streamed. Structured WARNING is - # emitted by ``_emit_stream_drop`` below; no - # additional INFO line needed. - try: - self._fire_stream_delta( - "\n\n⚠ Connection dropped mid tool-call; " - "reconnecting…\n\n" - ) - except Exception: - pass - # Reset the streamed-text buffer so the retry's - # fresh preamble doesn't get double-recorded in - # _current_streamed_assistant_text (which would - # pollute the interim-visible-text comparison). - try: - self._reset_stream_delivery_tracking() - except Exception: - pass - # Reset in-memory accumulators so the next - # attempt's chunks don't concat onto the dead - # stream's partial JSON. - result["partial_tool_names"] = [] - deltas_were_sent["yes"] = False - first_delta_fired["done"] = False - self._emit_stream_drop( - error=e, - attempt=_stream_attempt + 2, - max_attempts=_max_stream_retries + 1, - mid_tool_call=True, - diag=request_client_holder.get("diag"), - ) - stale = request_client_holder.get("client") - if stale is not None: - self._close_request_openai_client( - stale, reason="stream_mid_tool_retry_cleanup" - ) - request_client_holder["client"] = None - try: - self._replace_primary_openai_client( - reason="stream_mid_tool_retry_pool_cleanup" - ) - except Exception: - pass - continue - - # SSE error events from proxies (e.g. OpenRouter sends - # {"error":{"message":"Network connection lost."}}) are - # raised as APIError by the OpenAI SDK. These are - # semantically identical to httpx connection drops — - # the upstream stream died — and should be retried with - # a fresh connection. Distinguish from HTTP errors: - # APIError from SSE has no status_code, while - # APIStatusError (4xx/5xx) always has one. - _is_sse_conn_err = False - if not _is_timeout and not _is_conn_err: - from openai import APIError as _APIError - if isinstance(e, _APIError) and not getattr(e, "status_code", None): - _err_lower_sse = str(e).lower() - _SSE_CONN_PHRASES = ( - "connection lost", - "connection reset", - "connection closed", - "connection terminated", - "network error", - "network connection", - "terminated", - "peer closed", - "broken pipe", - "upstream connect error", - ) - _is_sse_conn_err = any( - phrase in _err_lower_sse - for phrase in _SSE_CONN_PHRASES - ) - - if _is_timeout or _is_conn_err or _is_sse_conn_err: - # Transient network / timeout error. Retry the - # streaming request with a fresh connection first. - if _stream_attempt < _max_stream_retries: - self._emit_stream_drop( - error=e, - attempt=_stream_attempt + 2, - max_attempts=_max_stream_retries + 1, - mid_tool_call=False, - diag=request_client_holder.get("diag"), - ) - # Close the stale request client before retry - stale = request_client_holder.get("client") - if stale is not None: - self._close_request_openai_client( - stale, reason="stream_retry_cleanup" - ) - request_client_holder["client"] = None - # Also rebuild the primary client to purge - # any dead connections from the pool. - try: - self._replace_primary_openai_client( - reason="stream_retry_pool_cleanup" - ) - except Exception: - pass - continue - # Retries exhausted. Log the final failure with - # full diagnostic detail (chain, headers, - # bytes/elapsed) via the same helper used for - # mid-flight retries — subagent lines get the - # ``[subagent-N]`` log_prefix so the parent can - # attribute them. - self._log_stream_retry( - kind="exhausted", - error=e, - attempt=_max_stream_retries + 1, - max_attempts=_max_stream_retries + 1, - mid_tool_call=False, - diag=request_client_holder.get("diag"), - ) - self._emit_status( - "❌ Connection to provider failed after " - f"{_max_stream_retries + 1} attempts. " - "The provider may be experiencing issues — " - "try again in a moment." - ) - else: - _err_lower = str(e).lower() - _is_stream_unsupported = ( - "stream" in _err_lower - and "not supported" in _err_lower - ) - if _is_stream_unsupported: - self._disable_streaming = True - self._safe_print( - "\n⚠ Streaming is not supported for this " - "model/provider. Switching to non-streaming.\n" - " To avoid this delay, set display.streaming: false " - "in config.yaml\n" - ) - logger.info( - "Streaming failed before delivery: %s", - e, - ) - - # Propagate the error to the main retry loop instead of - # falling back to non-streaming inline. The main loop has - # richer recovery: credential rotation, provider fallback, - # backoff, and — for "stream not supported" — will switch - # to non-streaming on the next attempt via _disable_streaming. - result["error"] = e - return - except InterruptedError as e: - # The interrupt may be noticed inside the worker thread before - # the polling loop sees it. Surface it through the normal result - # channel so callers never miss a fast pre-retry interrupt. - result["error"] = e - return - finally: - request_client = request_client_holder.get("client") - if request_client is not None: - self._close_request_openai_client(request_client, reason="stream_request_complete") - - _stream_stale_timeout_base = float(os.getenv("HERMES_STREAM_STALE_TIMEOUT", 180.0)) - # Local providers (Ollama, oMLX, llama-cpp) can take 300+ seconds - # for prefill on large contexts. Disable the stale detector unless - # the user explicitly set HERMES_STREAM_STALE_TIMEOUT. - if _stream_stale_timeout_base == 180.0 and self.base_url and is_local_endpoint(self.base_url): - _stream_stale_timeout = float("inf") - logger.debug("Local provider detected (%s) — stale stream timeout disabled", self.base_url) - else: - # Scale the stale timeout for large contexts: slow models (like Opus) - # can legitimately think for minutes before producing the first token - # when the context is large. Without this, the stale detector kills - # healthy connections during the model's thinking phase, producing - # spurious RemoteProtocolError ("peer closed connection"). - _est_tokens = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 - if _est_tokens > 100_000: - _stream_stale_timeout = max(_stream_stale_timeout_base, 300.0) - elif _est_tokens > 50_000: - _stream_stale_timeout = max(_stream_stale_timeout_base, 240.0) - else: - _stream_stale_timeout = _stream_stale_timeout_base - - t = threading.Thread(target=_call, daemon=True) - t.start() - _last_heartbeat = time.time() - _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches - while t.is_alive(): - t.join(timeout=0.3) - - # Periodic heartbeat: touch the agent's activity tracker so the - # gateway's inactivity monitor knows we're alive while waiting - # for stream chunks. Without this, long thinking pauses (e.g. - # reasoning models) or slow prefill on local providers (Ollama) - # trigger false inactivity timeouts. The _call thread touches - # activity on each chunk, but the gap between API call start - # and first chunk can exceed the gateway timeout — especially - # when the stale-stream timeout is disabled (local providers). - _hb_now = time.time() - if _hb_now - _last_heartbeat >= _HEARTBEAT_INTERVAL: - _last_heartbeat = _hb_now - _waiting_secs = int(_hb_now - last_chunk_time["t"]) - self._touch_activity( - f"waiting for stream response ({_waiting_secs}s, no chunks yet)" - ) - - # Detect stale streams: connections kept alive by SSE pings - # but delivering no real chunks. Kill the client so the - # inner retry loop can start a fresh connection. - _stale_elapsed = time.time() - last_chunk_time["t"] - if _stale_elapsed > _stream_stale_timeout: - _est_ctx = sum(len(str(v)) for v in api_kwargs.get("messages", [])) // 4 - logger.warning( - "Stream stale for %.0fs (threshold %.0fs) — no chunks received. " - "model=%s context=~%s tokens. Killing connection.", - _stale_elapsed, _stream_stale_timeout, - api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", - ) - self._emit_status( - f"⚠️ No response from provider for {int(_stale_elapsed)}s " - f"(model: {api_kwargs.get('model', 'unknown')}, " - f"context: ~{_est_ctx:,} tokens). " - f"Reconnecting..." - ) - try: - rc = request_client_holder.get("client") - if rc is not None: - self._close_request_openai_client(rc, reason="stale_stream_kill") - except Exception: - pass - # Rebuild the primary client too — its connection pool - # may hold dead sockets from the same provider outage. - try: - self._replace_primary_openai_client(reason="stale_stream_pool_cleanup") - except Exception: - pass - # Reset the timer so we don't kill repeatedly while - # the inner thread processes the closure. - last_chunk_time["t"] = time.time() - self._touch_activity( - f"stale stream detected after {int(_stale_elapsed)}s, reconnecting" - ) - - if self._interrupt_requested: - try: - if self.api_mode == "anthropic_messages": - self._anthropic_client.close() - self._rebuild_anthropic_client() - else: - request_client = request_client_holder.get("client") - if request_client is not None: - self._close_request_openai_client(request_client, reason="stream_interrupt_abort") - except Exception: - pass - raise InterruptedError("Agent interrupted during streaming API call") - if result["error"] is not None: - if deltas_were_sent["yes"]: - # Streaming failed AFTER some tokens were already delivered to - # the platform. Re-raising would let the outer retry loop make - # a new API call, creating a duplicate message. Return a - # partial "stop" response instead so the outer loop treats this - # turn as complete (no retry, no fallback). - # Recover whatever content was already streamed to the user. - # _current_streamed_assistant_text accumulates text fired - # through _fire_stream_delta, so it has exactly what the - # user saw before the connection died. - _partial_text = ( - getattr(self, "_current_streamed_assistant_text", "") or "" - ).strip() or None - - # If the stream died while the model was emitting a tool call, - # the stub below will silently set `tool_calls=None` and the - # agent loop will treat the turn as complete — the attempted - # action is lost with no user-facing signal. Append a - # human-visible warning to the stub content so (a) the user - # knows something failed, and (b) the next turn's model sees - # in conversation history what was attempted and can retry. - _partial_names = list(result.get("partial_tool_names") or []) - if _partial_names: - _name_str = ", ".join(_partial_names[:3]) - if len(_partial_names) > 3: - _name_str += f", +{len(_partial_names) - 3} more" - _warn = ( - f"\n\n⚠ Stream stalled mid tool-call " - f"({_name_str}); the action was not executed. " - f"Ask me to retry if you want to continue." - ) - _partial_text = (_partial_text or "") + _warn - # Also fire as a streaming delta so the user sees it now - # instead of only in the persisted transcript. - try: - self._fire_stream_delta(_warn) - except Exception: - pass - logger.warning( - "Partial stream dropped tool call(s) %s after %s chars " - "of text; surfaced warning to user: %s", - _partial_names, len(_partial_text or ""), result["error"], - ) - else: - logger.warning( - "Partial stream delivered before error; returning stub " - "response with %s chars of recovered content to prevent " - "duplicate messages: %s", - len(_partial_text or ""), - result["error"], - ) - _stub_msg = SimpleNamespace( - role="assistant", content=_partial_text, tool_calls=None, - reasoning_content=None, - ) - return SimpleNamespace( - id="partial-stream-stub", - model=getattr(self, "model", "unknown"), - choices=[SimpleNamespace( - index=0, message=_stub_msg, finish_reason="stop", - )], - usage=None, - ) - raise result["error"] - return result["response"] - - # ── Provider fallback ────────────────────────────────────────────────── + """Forwarder — see ``agent.chat_completion_helpers.interruptible_streaming_api_call``.""" + from agent.chat_completion_helpers import interruptible_streaming_api_call + return interruptible_streaming_api_call(self, api_kwargs, on_first_delta=on_first_delta) def _try_activate_fallback(self, reason: "FailoverReason | None" = None) -> bool: """Forwarder — see ``agent.chat_completion_helpers.try_activate_fallback``.""" diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 722de089628..eb5efcafca7 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -4793,9 +4793,10 @@ class TestAnthropicInterruptHandler: def test_streaming_has_anthropic_branch(self): """_streaming_api_call must also handle Anthropic interrupt.""" import inspect - source = inspect.getsource(AIAgent._interruptible_streaming_api_call) + from agent.chat_completion_helpers import interruptible_streaming_api_call + source = inspect.getsource(interruptible_streaming_api_call) assert "anthropic_messages" in source, \ - "_streaming_api_call must handle Anthropic interrupt" + "interruptible_streaming_api_call must handle Anthropic interrupt" # --------------------------------------------------------------------------- From c42fa94afc39c7caca15cdb7b951cc338a4587f0 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 19:03:30 -0700 Subject: [PATCH 011/142] refactor(run_agent): extract Codex runtime + assorted helpers to dedicated modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new modules: * agent/codex_runtime.py — three Codex API-mode methods - run_codex_app_server_turn (148 LOC) — Codex CLI subprocess driver - run_codex_stream (125 LOC) — Codex Responses API stream - run_codex_create_stream_fallback (78 LOC) — fallback after Responses stream=true initial create failure * agent/agent_runtime_helpers.py — twelve assorted AIAgent helpers totalling ~1,166 LOC: convert_to_trajectory_format, sanitize_tool_call_arguments (static), repair_message_sequence, strip_think_blocks, recover_with_credential_pool, try_recover_primary_transport, drop_thinking_only_and_merge_users (static), restore_primary_runtime, extract_reasoning, dump_api_request_debug, anthropic_prompt_cache_policy, create_openai_client AIAgent keeps thin forwarder methods for all 15 (preserving @staticmethod where needed). Symbols tests patch on run_agent (OpenAI, AIAgent class attrs) are routed through _ra() to honor the patch contract. The _TRANSIENT_TRANSPORT_ERRORS frozenset moves with try_recover_primary_transport and is referenced as a module-level constant in the extracted code. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 11391 -> 9887 lines (-1504). --- agent/agent_runtime_helpers.py | 1260 +++++++++++++++++++++++++++ agent/codex_runtime.py | 400 +++++++++ run_agent.py | 1495 +------------------------------- 3 files changed, 1705 insertions(+), 1450 deletions(-) create mode 100644 agent/agent_runtime_helpers.py create mode 100644 agent/codex_runtime.py diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py new file mode 100644 index 00000000000..4efe5203421 --- /dev/null +++ b/agent/agent_runtime_helpers.py @@ -0,0 +1,1260 @@ +"""Assorted AIAgent runtime helpers — moved out of run_agent.py for clarity. + +Each function takes the parent ``AIAgent`` as its first argument +(``agent``) except for the static helpers (``sanitize_tool_call_arguments``, +``drop_thinking_only_and_merge_users``) which are stateless. AIAgent +keeps thin forwarders for backward compatibility. + +Methods covered: +* ``convert_to_trajectory_format`` — internal -> trajectory-file format +* ``sanitize_tool_call_arguments`` — repair corrupted JSON in tool_calls +* ``repair_message_sequence`` — enforce alternation invariants +* ``strip_think_blocks`` — remove inline reasoning from stored content +* ``recover_with_credential_pool`` — rotate pool entries on 429 +* ``try_recover_primary_transport`` — re-create OpenAI client after rate-limit +* ``drop_thinking_only_and_merge_users`` — Anthropic-style cleanup +* ``restore_primary_runtime`` — un-do fallback activation +* ``extract_reasoning`` — pull reasoning fields out of API responses +* ``dump_api_request_debug`` — write request body for post-mortem +* ``anthropic_prompt_cache_policy`` — compute cache_control breakpoints +* ``create_openai_client`` — build the per-agent OpenAI SDK client +""" + +from __future__ import annotations + +import copy +import json +import logging +import os +import re +import threading +import time +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from hermes_cli.timeouts import get_provider_request_timeout +from agent.message_sanitization import ( + _repair_tool_call_arguments, + _sanitize_surrogates, +) +from agent.tool_dispatch_helpers import _trajectory_normalize_msg +from agent.trajectory import convert_scratchpad_to_think +from agent.error_classifier import classify_api_error, FailoverReason +from utils import base_url_host_matches, base_url_hostname, env_var_enabled, atomic_json_write + +logger = logging.getLogger(__name__) + + +def _ra(): + """Lazy ``run_agent`` reference for test-patch routing.""" + import run_agent + return run_agent + + + +def convert_to_trajectory_format(agent, messages: List[Dict[str, Any]], user_query: str, completed: bool) -> List[Dict[str, Any]]: + """ + Convert internal message format to trajectory format for saving. + + Args: + messages (List[Dict]): Internal message history + user_query (str): Original user query + completed (bool): Whether the conversation completed successfully + + Returns: + List[Dict]: Messages in trajectory format + """ + # Normalize multimodal tool results — trajectories are text-only, so + # replace image-bearing tool messages with their text_summary to avoid + # embedding ~1MB base64 blobs into every saved trajectory. + messages = [_trajectory_normalize_msg(m) for m in messages] + trajectory = [] + + # Add system message with tool definitions + system_msg = ( + "You are a function calling AI model. You are provided with function signatures within XML tags. " + "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " + "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After calling & executing the functions, you will be provided with function results within " + " XML tags. Here are the available tools:\n" + f"\n{agent._format_tools_for_system_message()}\n\n" + "For each function call return a JSON object, with the following pydantic model json schema for each:\n" + "{'title': 'FunctionCall', 'type': 'object', 'properties': {'name': {'title': 'Name', 'type': 'string'}, " + "'arguments': {'title': 'Arguments', 'type': 'object'}}, 'required': ['name', 'arguments']}\n" + "Each function call should be enclosed within XML tags.\n" + "Example:\n\n{'name': ,'arguments': }\n" + ) + + trajectory.append({ + "from": "system", + "value": system_msg + }) + + # Add the actual user prompt (from the dataset) as the first human message + trajectory.append({ + "from": "human", + "value": user_query + }) + + # Skip the first message (the user query) since we already added it above. + # Prefill messages are injected at API-call time only (not in the messages + # list), so no offset adjustment is needed here. + i = 1 + + while i < len(messages): + msg = messages[i] + + if msg["role"] == "assistant": + # Check if this message has tool calls + if "tool_calls" in msg and msg["tool_calls"]: + # Format assistant message with tool calls + # Add tags around reasoning for trajectory storage + content = "" + + # Prepend reasoning in tags if available (native thinking tokens) + if msg.get("reasoning") and msg["reasoning"].strip(): + content = f"\n{msg['reasoning']}\n\n" + + if msg.get("content") and msg["content"].strip(): + # Convert any tags to tags + # (used when native thinking is disabled and model reasons via XML) + content += convert_scratchpad_to_think(msg["content"]) + "\n" + + # Add tool calls wrapped in XML tags + for tool_call in msg["tool_calls"]: + if not tool_call or not isinstance(tool_call, dict): continue + # Parse arguments - should always succeed since we validate during conversation + # but keep try-except as safety net + try: + arguments = json.loads(tool_call["function"]["arguments"]) if isinstance(tool_call["function"]["arguments"], str) else tool_call["function"]["arguments"] + except json.JSONDecodeError: + # This shouldn't happen since we validate and retry during conversation, + # but if it does, log warning and use empty dict + logging.warning(f"Unexpected invalid JSON in trajectory conversion: {tool_call['function']['arguments'][:100]}") + arguments = {} + + tool_call_json = { + "name": tool_call["function"]["name"], + "arguments": arguments + } + content += f"\n{json.dumps(tool_call_json, ensure_ascii=False)}\n\n" + + # Ensure every gpt turn has a block (empty if no reasoning) + # so the format is consistent for training data + if "" not in content: + content = "\n\n" + content + + trajectory.append({ + "from": "gpt", + "value": content.rstrip() + }) + + # Collect all subsequent tool responses + tool_responses = [] + j = i + 1 + while j < len(messages) and messages[j]["role"] == "tool": + tool_msg = messages[j] + # Format tool response with XML tags + tool_response = "\n" + + # Try to parse tool content as JSON if it looks like JSON + tool_content = tool_msg["content"] + try: + if tool_content.strip().startswith(("{", "[")): + tool_content = json.loads(tool_content) + except (json.JSONDecodeError, AttributeError): + pass # Keep as string if not valid JSON + + tool_index = len(tool_responses) + tool_name = ( + msg["tool_calls"][tool_index]["function"]["name"] + if tool_index < len(msg["tool_calls"]) + else "unknown" + ) + tool_response += json.dumps({ + "tool_call_id": tool_msg.get("tool_call_id", ""), + "name": tool_name, + "content": tool_content + }, ensure_ascii=False) + tool_response += "\n" + tool_responses.append(tool_response) + j += 1 + + # Add all tool responses as a single message + if tool_responses: + trajectory.append({ + "from": "tool", + "value": "\n".join(tool_responses) + }) + i = j - 1 # Skip the tool messages we just processed + + else: + # Regular assistant message without tool calls + # Add tags around reasoning for trajectory storage + content = "" + + # Prepend reasoning in tags if available (native thinking tokens) + if msg.get("reasoning") and msg["reasoning"].strip(): + content = f"\n{msg['reasoning']}\n\n" + + # Convert any tags to tags + # (used when native thinking is disabled and model reasons via XML) + raw_content = msg["content"] or "" + content += convert_scratchpad_to_think(raw_content) + + # Ensure every gpt turn has a block (empty if no reasoning) + if "" not in content: + content = "\n\n" + content + + trajectory.append({ + "from": "gpt", + "value": content.strip() + }) + + elif msg["role"] == "user": + trajectory.append({ + "from": "human", + "value": msg["content"] + }) + + i += 1 + + return trajectory + + + +def sanitize_tool_call_arguments( + messages: list, + *, + logger=None, + session_id: str = None, +) -> int: + """Repair corrupted assistant tool-call argument JSON in-place.""" + log = logger or logging.getLogger(__name__) + if not isinstance(messages, list): + return 0 + + repaired = 0 + marker = _ra().AIAgent._TOOL_CALL_ARGUMENTS_CORRUPTION_MARKER + + def _prepend_marker(tool_msg: dict) -> None: + existing = tool_msg.get("content") + if isinstance(existing, str): + if not existing: + tool_msg["content"] = marker + elif not existing.startswith(marker): + tool_msg["content"] = f"{marker}\n{existing}" + return + if existing is None: + tool_msg["content"] = marker + return + try: + existing_text = json.dumps(existing) + except TypeError: + existing_text = str(existing) + tool_msg["content"] = f"{marker}\n{existing_text}" + + message_index = 0 + while message_index < len(messages): + msg = messages[message_index] + if not isinstance(msg, dict) or msg.get("role") != "assistant": + message_index += 1 + continue + + tool_calls = msg.get("tool_calls") + if not isinstance(tool_calls, list) or not tool_calls: + message_index += 1 + continue + + insert_at = message_index + 1 + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + continue + function = tool_call.get("function") + if not isinstance(function, dict): + continue + + arguments = function.get("arguments") + if arguments is None or arguments == "": + function["arguments"] = "{}" + continue + if isinstance(arguments, str) and not arguments.strip(): + function["arguments"] = "{}" + continue + if not isinstance(arguments, str): + continue + + try: + json.loads(arguments) + except json.JSONDecodeError: + tool_call_id = tool_call.get("id") + function_name = function.get("name", "?") + preview = arguments[:80] + log.warning( + "Corrupted tool_call arguments repaired before request " + "(session=%s, message_index=%s, tool_call_id=%s, function=%s, preview=%r)", + session_id or "-", + message_index, + tool_call_id or "-", + function_name, + preview, + ) + function["arguments"] = "{}" + + existing_tool_msg = None + scan_index = message_index + 1 + while scan_index < len(messages): + candidate = messages[scan_index] + if not isinstance(candidate, dict) or candidate.get("role") != "tool": + break + if candidate.get("tool_call_id") == tool_call_id: + existing_tool_msg = candidate + break + scan_index += 1 + + if existing_tool_msg is None: + messages.insert( + insert_at, + { + "role": "tool", + "name": function_name if function_name != "?" else "", + "tool_call_id": tool_call_id, + "content": marker, + }, + ) + insert_at += 1 + else: + _prepend_marker(existing_tool_msg) + + repaired += 1 + + message_index += 1 + + return repaired + + + +def repair_message_sequence(agent, messages: List[Dict]) -> int: + """Collapse malformed role-alternation left in the live history. + + Providers (OpenAI, OpenRouter, Anthropic) expect strict alternation: + after the system message, user/tool alternates with assistant, with + no two consecutive user messages and no tool-result that doesn't + follow an assistant-with-tool_calls. Violations cause silent empty + responses on most providers, which triggers the empty-retry loop. + + This runs right before the API call as a defensive belt — by the + time it fires, the scaffolding strip should already have prevented + most shapes, but external callers (gateway multi-queue replay, + session resume, cron, explicit conversation_history passed in by + host code) can feed in already-broken histories. + + Repairs applied: + 1. Stray ``tool`` messages whose ``tool_call_id`` doesn't match + any preceding assistant tool_call — dropped. + 2. Consecutive ``user`` messages — merged with newline separator + so no user input is lost. + + Deliberately does NOT rewind orphan ``assistant(tool_calls)+tool`` + pairs that precede a user message — that pattern IS valid when the + previous turn completed normally and the user jumped in to redirect + before the model got a continuation turn (the ongoing dialog + pattern). The empty-response scaffolding stripper handles the + genuinely-broken variant via its flag-gated rewind. + + Returns the number of repairs made (for logging/telemetry). + """ + if not messages: + return 0 + + repairs = 0 + + # Pass 1: drop stray tool messages that don't follow a known + # assistant tool_call_id. Uses a rolling set of known ids refreshed + # on each assistant message. + known_tool_ids: set = set() + filtered: List[Dict] = [] + for msg in messages: + if not isinstance(msg, dict): + filtered.append(msg) + continue + role = msg.get("role") + if role == "assistant": + known_tool_ids = set() + for tc in (msg.get("tool_calls") or []): + tc_id = tc.get("id") if isinstance(tc, dict) else None + if tc_id: + known_tool_ids.add(tc_id) + filtered.append(msg) + elif role == "tool": + tc_id = msg.get("tool_call_id") + if tc_id and tc_id in known_tool_ids: + filtered.append(msg) + else: + repairs += 1 + else: + if role == "user": + # A user turn closes the tool-result run; subsequent + # tool messages without a fresh assistant tool_call + # are orphans. + known_tool_ids = set() + filtered.append(msg) + + # Pass 2: merge consecutive user messages. Preserves all user input + # so nothing the user typed is lost. + merged: List[Dict] = [] + for msg in filtered: + if ( + merged + and isinstance(msg, dict) + and msg.get("role") == "user" + and isinstance(merged[-1], dict) + and merged[-1].get("role") == "user" + ): + prev = merged[-1] + prev_content = prev.get("content", "") + new_content = msg.get("content", "") + # Only merge plain-text content; leave multimodal (list) + # content alone — collapsing image/audio blocks risks + # mangling the attachment structure. + if isinstance(prev_content, str) and isinstance(new_content, str): + prev["content"] = ( + (prev_content + "\n\n" + new_content) + if prev_content and new_content + else (prev_content or new_content) + ) + repairs += 1 + continue + merged.append(msg) + + if repairs > 0: + # Rewrite in place so downstream paths (persistence, return + # value, session DB flush) see the repaired sequence. + messages[:] = merged + + return repairs + + + +def strip_think_blocks(agent, content: str) -> str: + """Remove reasoning/thinking blocks from content, returning only visible text. + + Handles four cases: + 1. Closed tag pairs (``…``) — the common path when + the provider emits complete reasoning blocks. + 2. Unterminated open tag at a block boundary (start of text or + after a newline) — e.g. MiniMax M2.7 / NIM endpoints where the + closing tag is dropped. Everything from the open tag to end + of string is stripped. The block-boundary check mirrors + ``gateway/stream_consumer.py``'s filter so models that mention + ```` in prose aren't over-stripped. + 3. Stray orphan open/close tags that slip through. + 4. Tag variants: ````, ````, ````, + ````, ```` (Gemma 4), all + case-insensitive. + + Additionally strips standalone tool-call XML blocks that some open + models (notably Gemma variants on OpenRouter) emit inside assistant + content instead of via the structured ``tool_calls`` field: + * ``…`` + * ``…`` + * ``…`` + * ``…`` + * ``…`` + * ``…`` (Gemma style) + Ported from openclaw/openclaw#67318. The ```` variant is + boundary-gated (only strips when the tag sits at start-of-line or + after punctuation and carries a ``name="..."`` attribute) so prose + mentions like "Use in JavaScript" are preserved. + """ + if not content: + return "" + # 1. Closed tag pairs — case-insensitive for all variants so + # mixed-case tags (, ) don't slip through to + # the unterminated-tag pass and take trailing content with them. + content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + # 1b. Tool-call XML blocks (openclaw/openclaw#67318). Handle the + # generic tag names first — they have no attribute gating since + # a literal in prose is already vanishingly rare. + for _tc_name in ("tool_call", "tool_calls", "tool_result", + "function_call", "function_calls"): + content = re.sub( + rf'<{_tc_name}\b[^>]*>.*?', + '', + content, + flags=re.DOTALL | re.IGNORECASE, + ) + # 1c. ... — Gemma-style standalone + # tool call. Only strip when the tag sits at a block boundary + # (start of text, after a newline, or after sentence-ending + # punctuation) AND carries a name="..." attribute. This keeps + # prose mentions like "Use to declare" safe. + content = re.sub( + r'(?:(?<=^)|(?<=[\n\r.!?:]))[ \t]*' + r']*\bname\s*=[^>]*>' + r'(?:(?:(?!).)*)', + '', + content, + flags=re.DOTALL | re.IGNORECASE, + ) + # 2. Unterminated reasoning block — open tag at a block boundary + # (start of text, or after a newline) with no matching close. + # Strip from the tag to end of string. Fixes #8878 / #9568 + # (MiniMax M2.7 leaking raw reasoning into assistant content). + content = re.sub( + r'(?:^|\n)[ \t]*<(?:think|thinking|reasoning|thought|REASONING_SCRATCHPAD)\b[^>]*>.*$', + '', + content, + flags=re.DOTALL | re.IGNORECASE, + ) + # 3. Stray orphan open/close tags that slipped through. + content = re.sub( + r'\s*', + '', + content, + flags=re.IGNORECASE, + ) + # 3b. Stray tool-call closers. (We do NOT strip bare or + # unterminated because a truncated tail + # during streaming may still be valuable to the user; matches + # OpenClaw's intentional asymmetry.) + content = re.sub( + r'\s*', + '', + content, + flags=re.IGNORECASE, + ) + return content + + + +def recover_with_credential_pool( + agent, + *, + status_code: Optional[int], + has_retried_429: bool, + classified_reason: Optional[FailoverReason] = None, + error_context: Optional[Dict[str, Any]] = None, +) -> tuple[bool, bool]: + """Attempt credential recovery via pool rotation. + + Returns (recovered, has_retried_429). + On rate limits: first occurrence retries same credential (sets flag True). + second consecutive failure rotates to next credential. + On billing exhaustion: immediately rotates. + On auth failures: attempts token refresh before rotating. + + `classified_reason` lets the recovery path honor the structured error + classifier instead of relying only on raw HTTP codes. This matters for + providers that surface billing/rate-limit/auth conditions under a + different status code, such as Anthropic returning HTTP 400 for + "out of extra usage". + """ + pool = agent._credential_pool + if pool is None: + return False, has_retried_429 + + effective_reason = classified_reason + if effective_reason is None: + if status_code == 402: + effective_reason = FailoverReason.billing + elif status_code == 429: + effective_reason = FailoverReason.rate_limit + elif status_code in {401, 403}: + effective_reason = FailoverReason.auth + + if effective_reason == FailoverReason.billing: + rotate_status = status_code if status_code is not None else 402 + next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + if next_entry is not None: + logger.info( + "Credential %s (billing) — rotated to pool entry %s", + rotate_status, + getattr(next_entry, "id", "?"), + ) + agent._swap_credential(next_entry) + return True, False + return False, has_retried_429 + + if effective_reason == FailoverReason.rate_limit: + if not has_retried_429: + return False, True + rotate_status = status_code if status_code is not None else 429 + next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + if next_entry is not None: + logger.info( + "Credential %s (rate limit) — rotated to pool entry %s", + rotate_status, + getattr(next_entry, "id", "?"), + ) + agent._swap_credential(next_entry) + return True, False + return False, True + + if effective_reason == FailoverReason.auth: + refreshed = pool.try_refresh_current() + if refreshed is not None: + logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") + agent._swap_credential(refreshed) + return True, has_retried_429 + # Refresh failed — rotate to next credential instead of giving up. + # The failed entry is already marked exhausted by try_refresh_current(). + rotate_status = status_code if status_code is not None else 401 + next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + if next_entry is not None: + logger.info( + "Credential %s (auth refresh failed) — rotated to pool entry %s", + rotate_status, + getattr(next_entry, "id", "?"), + ) + agent._swap_credential(next_entry) + return True, False + + return False, has_retried_429 + + + +def try_recover_primary_transport( + agent, api_error: Exception, *, retry_count: int, max_retries: int, +) -> bool: + """Attempt one extra primary-provider recovery cycle for transient transport failures. + + After ``max_retries`` exhaust, rebuild the primary client (clearing + stale connection pools) and give it one more attempt before falling + back. This is most useful for direct endpoints (custom, Z.AI, + Anthropic, OpenAI, local models) where a TCP-level hiccup does not + mean the provider is down. + + Skipped for proxy/aggregator providers (OpenRouter, Nous) which + already manage connection pools and retries server-side — if our + retries through them are exhausted, one more rebuilt client won't help. + """ + if agent._fallback_activated: + return False + + # Only for transient transport errors + error_type = type(api_error).__name__ + if error_type not in _TRANSIENT_TRANSPORT_ERRORS: + return False + + # Skip for aggregator providers — they manage their own retry infra + if agent._is_openrouter_url(): + return False + provider_lower = (agent.provider or "").strip().lower() + if provider_lower in {"nous", "nous-research"}: + return False + + try: + # Close existing client to release stale connections + if getattr(agent, "client", None) is not None: + try: + agent._close_openai_client( + agent.client, reason="primary_recovery", shared=True, + ) + except Exception: + pass + + # Rebuild from primary snapshot + rt = agent._primary_runtime + agent._client_kwargs = dict(rt["client_kwargs"]) + agent.model = rt["model"] + agent.provider = rt["provider"] + agent.base_url = rt["base_url"] + agent.api_mode = rt["api_mode"] + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + agent.api_key = rt["api_key"] + + if agent.api_mode == "anthropic_messages": + from agent.anthropic_adapter import build_anthropic_client + agent._anthropic_api_key = rt["anthropic_api_key"] + agent._anthropic_base_url = rt["anthropic_base_url"] + agent._anthropic_client = build_anthropic_client( + rt["anthropic_api_key"], rt["anthropic_base_url"], + timeout=get_provider_request_timeout(agent.provider, agent.model), + ) + agent._is_anthropic_oauth = rt["is_anthropic_oauth"] + agent.client = None + else: + agent.client = agent._create_openai_client( + dict(rt["client_kwargs"]), + reason="primary_recovery", + shared=True, + ) + + wait_time = min(3 + retry_count, 8) + agent._vprint( + f"{agent.log_prefix}🔁 Transient {error_type} on {agent.provider} — " + f"rebuilt client, waiting {wait_time}s before one last primary attempt.", + force=True, + ) + time.sleep(wait_time) + return True + except Exception as e: + logging.warning("Primary transport recovery failed: %s", e) + return False + +# ── End provider fallback ────────────────────────────────────────────── + + + +def drop_thinking_only_and_merge_users( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Drop thinking-only assistant turns; merge any adjacent user messages left behind. + + Runs on the per-call ``api_messages`` copy only. The stored + conversation history (``agent.messages``) is never mutated, so the + user still sees the thinking block in the CLI/gateway transcript and + session persistence keeps the full trace. Only the wire copy sent to + the provider is cleaned. + + Why drop-and-merge rather than inject stub text: + - Fabricating ``"."`` / ``"(continued)"`` text lies in the history + and makes future turns see model output the model didn't emit. + - Dropping the turn preserves honesty; merging adjacent user messages + preserves the provider's role-alternation invariant. + - This is the pattern used by Claude Code's ``normalizeMessagesForAPI`` + (filterOrphanedThinkingOnlyMessages + mergeAdjacentUserMessages). + """ + if not messages: + return messages + + # Pass 1: drop thinking-only assistant turns. + kept = [m for m in messages if not _ra().AIAgent._is_thinking_only_assistant(m)] + dropped = len(messages) - len(kept) + if dropped == 0: + return messages + + # Pass 2: merge any newly-adjacent user messages. + merged: List[Dict[str, Any]] = [] + merges = 0 + for m in kept: + prev = merged[-1] if merged else None + if ( + prev is not None + and prev.get("role") == "user" + and m.get("role") == "user" + ): + prev_content = prev.get("content", "") + cur_content = m.get("content", "") + # Work on a copy of ``prev`` so the caller's input dicts are + # never mutated. ``_sanitize_api_messages`` upstream already + # hands us per-call copies, but staying pure here means we + # can be called safely from anywhere (tests, other loops). + prev_copy = dict(prev) + # Only string-content merge is meaningful for role-alternation + # purposes. If either side is a list (multimodal), append as a + # separate block rather than collapsing. + if isinstance(prev_content, str) and isinstance(cur_content, str): + sep = "\n\n" if prev_content and cur_content else "" + prev_copy["content"] = prev_content + sep + cur_content + elif isinstance(prev_content, list) and isinstance(cur_content, list): + prev_copy["content"] = list(prev_content) + list(cur_content) + elif isinstance(prev_content, list) and isinstance(cur_content, str): + if cur_content: + prev_copy["content"] = list(prev_content) + [ + {"type": "text", "text": cur_content} + ] + else: + prev_copy["content"] = list(prev_content) + elif isinstance(prev_content, str) and isinstance(cur_content, list): + new_blocks: List[Dict[str, Any]] = [] + if prev_content: + new_blocks.append({"type": "text", "text": prev_content}) + new_blocks.extend(cur_content) + prev_copy["content"] = new_blocks + else: + # Unknown content shape — fall back to appending separately + # (violates alternation, but safer than raising in a hot path). + merged.append(m) + continue + merged[-1] = prev_copy + merges += 1 + else: + merged.append(m) + + logger.debug( + "Pre-call sanitizer: dropped %d thinking-only assistant turn(s), " + "merged %d adjacent user message(s)", + dropped, + merges, + ) + return merged + + + +def restore_primary_runtime(agent) -> bool: + """Restore the primary runtime at the start of a new turn. + + In long-lived CLI sessions a single AIAgent instance spans multiple + turns. Without restoration, one transient failure pins the session + to the fallback provider for every subsequent turn. Calling this at + the top of ``run_conversation()`` makes fallback turn-scoped. + + The gateway caches agents across messages (``_agent_cache`` in + ``gateway/run.py``), so this restoration IS needed there too. + """ + if not agent._fallback_activated: + return False + + if getattr(agent, "_rate_limited_until", 0) > time.monotonic(): + return False # primary still in rate-limit cooldown, stay on fallback + + rt = agent._primary_runtime + try: + # ── Core runtime state ── + agent.model = rt["model"] + agent.provider = rt["provider"] + agent.base_url = rt["base_url"] # setter updates _base_url_lower + agent.api_mode = rt["api_mode"] + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + agent.api_key = rt["api_key"] + agent._client_kwargs = dict(rt["client_kwargs"]) + agent._use_prompt_caching = rt["use_prompt_caching"] + # Default to native layout when the restored snapshot predates the + # native-vs-proxy split (older sessions saved before this PR). + agent._use_native_cache_layout = rt.get( + "use_native_cache_layout", + agent.api_mode == "anthropic_messages" and agent.provider == "anthropic", + ) + + # ── Rebuild client for the primary provider ── + if agent.api_mode == "anthropic_messages": + from agent.anthropic_adapter import build_anthropic_client + agent._anthropic_api_key = rt["anthropic_api_key"] + agent._anthropic_base_url = rt["anthropic_base_url"] + agent._anthropic_client = build_anthropic_client( + rt["anthropic_api_key"], rt["anthropic_base_url"], + timeout=get_provider_request_timeout(agent.provider, agent.model), + ) + agent._is_anthropic_oauth = rt["is_anthropic_oauth"] + agent.client = None + else: + agent.client = agent._create_openai_client( + dict(rt["client_kwargs"]), + reason="restore_primary", + shared=True, + ) + + # ── Restore context engine state ── + cc = agent.context_compressor + cc.update_model( + model=rt["compressor_model"], + context_length=rt["compressor_context_length"], + base_url=rt["compressor_base_url"], + api_key=rt["compressor_api_key"], + provider=rt["compressor_provider"], + ) + + # ── Reset fallback chain for the new turn ── + agent._fallback_activated = False + agent._fallback_index = 0 + + logging.info( + "Primary runtime restored for new turn: %s (%s)", + agent.model, agent.provider, + ) + return True + except Exception as e: + logging.warning("Failed to restore primary runtime: %s", e) + return False + +# Which error types indicate a transient transport failure worth +# one more attempt with a rebuilt client / connection pool. +_TRANSIENT_TRANSPORT_ERRORS = frozenset({ + "ReadTimeout", "ConnectTimeout", "PoolTimeout", + "ConnectError", "RemoteProtocolError", + "APIConnectionError", "APITimeoutError", +}) + + + +def extract_reasoning(agent, assistant_message) -> Optional[str]: + """ + Extract reasoning/thinking content from an assistant message. + + OpenRouter and various providers can return reasoning in multiple formats: + 1. message.reasoning - Direct reasoning field (DeepSeek, Qwen, etc.) + 2. message.reasoning_content - Alternative field (Moonshot AI, Novita, etc.) + 3. message.reasoning_details - Array of {type, summary, ...} objects (OpenRouter unified) + + Args: + assistant_message: The assistant message object from the API response + + Returns: + Combined reasoning text, or None if no reasoning found + """ + reasoning_parts = [] + + # Check direct reasoning field + if hasattr(assistant_message, 'reasoning') and assistant_message.reasoning: + reasoning_parts.append(assistant_message.reasoning) + + # Check reasoning_content field (alternative name used by some providers) + if hasattr(assistant_message, 'reasoning_content') and assistant_message.reasoning_content: + # Don't duplicate if same as reasoning + if assistant_message.reasoning_content not in reasoning_parts: + reasoning_parts.append(assistant_message.reasoning_content) + + # Check reasoning_details array (OpenRouter unified format) + # Format: [{"type": "reasoning.summary", "summary": "...", ...}, ...] + if hasattr(assistant_message, 'reasoning_details') and assistant_message.reasoning_details: + for detail in assistant_message.reasoning_details: + if isinstance(detail, dict): + # Extract summary from reasoning detail object + summary = ( + detail.get('summary') + or detail.get('thinking') + or detail.get('content') + or detail.get('text') + ) + if summary and summary not in reasoning_parts: + reasoning_parts.append(summary) + + # Some providers embed reasoning directly inside assistant content + # instead of returning structured reasoning fields. Only fall back + # to inline extraction when no structured reasoning was found. + content = getattr(assistant_message, "content", None) + if not reasoning_parts and isinstance(content, list): + # DeepSeek V4 Pro (and compatible providers) return content as a + # list of typed blocks, e.g.: + # [{"type": "thinking", "thinking": "..."}, {"type": "output", ...}] + # Without this branch the thinking text is silently dropped and the + # next turn fails with HTTP 400 ("thinking must be passed back"). + # Refs #21944. + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + thinking_text = block.get("thinking") or block.get("text") or "" + thinking_text = thinking_text.strip() + if thinking_text and thinking_text not in reasoning_parts: + reasoning_parts.append(thinking_text) + if not reasoning_parts and isinstance(content, str) and content: + inline_patterns = ( + r"(.*?)", + r"(.*?)", + r"(.*?)", + r"(.*?)", + r"(.*?)", + ) + for pattern in inline_patterns: + flags = re.DOTALL | re.IGNORECASE + for block in re.findall(pattern, content, flags=flags): + cleaned = block.strip() + if cleaned and cleaned not in reasoning_parts: + reasoning_parts.append(cleaned) + + # Combine all reasoning parts + if reasoning_parts: + return "\n\n".join(reasoning_parts) + + return None + + + +def dump_api_request_debug( + agent, + api_kwargs: Dict[str, Any], + *, + reason: str, + error: Optional[Exception] = None, +) -> Optional[Path]: + """ + Dump a debug-friendly HTTP request record for the active inference API. + + Captures the request body from api_kwargs (excluding transport-only keys + like timeout). Intended for debugging provider-side 4xx failures where + retries are not useful. + """ + try: + body = copy.deepcopy(api_kwargs) + body.pop("timeout", None) + body = {k: v for k, v in body.items() if v is not None} + + api_key = None + try: + api_key = getattr(agent.client, "api_key", None) + except Exception as e: + logger.debug("Could not extract API key for debug dump: %s", e) + + dump_payload: Dict[str, Any] = { + "timestamp": datetime.now().isoformat(), + "session_id": agent.session_id, + "reason": reason, + "request": { + "method": "POST", + "url": f"{agent.base_url.rstrip('/')}{'/responses' if agent.api_mode == 'codex_responses' else '/chat/completions'}", + "headers": { + "Authorization": f"Bearer {agent._mask_api_key_for_logs(api_key)}", + "Content-Type": "application/json", + }, + "body": body, + }, + } + + if error is not None: + error_info: Dict[str, Any] = { + "type": type(error).__name__, + "message": str(error), + } + for attr_name in ("status_code", "request_id", "code", "param", "type"): + attr_value = getattr(error, attr_name, None) + if attr_value is not None: + error_info[attr_name] = attr_value + + body_attr = getattr(error, "body", None) + if body_attr is not None: + error_info["body"] = body_attr + + response_obj = getattr(error, "response", None) + if response_obj is not None: + try: + error_info["response_status"] = getattr(response_obj, "status_code", None) + error_info["response_text"] = response_obj.text + except Exception as e: + logger.debug("Could not extract error response details: %s", e) + + dump_payload["error"] = error_info + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + dump_file = agent.logs_dir / f"request_dump_{agent.session_id}_{timestamp}.json" + dump_file.write_text( + json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str), + encoding="utf-8", + ) + + agent._vprint(f"{agent.log_prefix}🧾 Request debug dump written to: {dump_file}") + + if env_var_enabled("HERMES_DUMP_REQUEST_STDOUT"): + print(json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str)) + + return dump_file + except Exception as dump_error: + if agent.verbose_logging: + logging.warning(f"Failed to dump API request debug payload: {dump_error}") + return None + + + +def anthropic_prompt_cache_policy( + agent, + *, + provider: Optional[str] = None, + base_url: Optional[str] = None, + api_mode: Optional[str] = None, + model: Optional[str] = None, +) -> tuple[bool, bool]: + """Decide whether to apply Anthropic prompt caching and which layout to use. + + Returns ``(should_cache, use_native_layout)``: + * ``should_cache`` — inject ``cache_control`` breakpoints for this + request (applies to OpenRouter Claude, native Anthropic, and + third-party gateways that speak the native Anthropic protocol). + * ``use_native_layout`` — place markers on the *inner* content + blocks (native Anthropic accepts and requires this layout); + when False markers go on the message envelope (OpenRouter and + OpenAI-wire proxies expect the looser layout). + + Third-party providers using the native Anthropic transport + (``api_mode == 'anthropic_messages'`` + Claude-named model) get + caching with the native layout so they benefit from the same + cost reduction as direct Anthropic callers, provided their + gateway implements the Anthropic cache_control contract + (MiniMax, Zhipu GLM, LiteLLM's Anthropic proxy mode all do). + + Qwen / Alibaba-family models on OpenCode, OpenCode Go, and direct + Alibaba (DashScope) also honour Anthropic-style ``cache_control`` + markers on OpenAI-wire chat completions. Upstream pi-mono #3392 / + pi #3393 documented this for opencode-go Qwen. Without markers + these providers serve zero cache hits, re-billing the full prompt + on every turn. + """ + eff_provider = (provider if provider is not None else agent.provider) or "" + eff_base_url = base_url if base_url is not None else (agent.base_url or "") + eff_api_mode = api_mode if api_mode is not None else (agent.api_mode or "") + eff_model = (model if model is not None else agent.model) or "" + + model_lower = eff_model.lower() + provider_lower = eff_provider.lower() + is_claude = "claude" in model_lower + is_openrouter = base_url_host_matches(eff_base_url, "openrouter.ai") + # Nous Portal proxies to OpenRouter behind the scenes — identical + # OpenAI-wire envelope cache_control semantics. Treat it as an + # OpenRouter-equivalent endpoint for caching layout purposes. + is_nous_portal = "nousresearch" in eff_base_url.lower() + is_anthropic_wire = eff_api_mode == "anthropic_messages" + is_native_anthropic = ( + is_anthropic_wire + and (eff_provider == "anthropic" or base_url_hostname(eff_base_url) == "api.anthropic.com") + ) + + if is_native_anthropic: + return True, True + if (is_openrouter or is_nous_portal) and is_claude: + return True, False + # Nous Portal Qwen (e.g. qwen3.6-plus) takes the same envelope-layout + # cache_control path as Portal Claude. Portal proxies to OpenRouter + # and the upstream Qwen route accepts cache_control markers; without + # this branch the alibaba-family check below only matches + # provider=opencode/alibaba and Portal traffic falls through to + # (False, False), serving 0% cache hits and re-billing the full + # prompt on every turn. + if is_nous_portal and "qwen" in model_lower: + return True, False + if is_anthropic_wire and is_claude: + # Third-party Anthropic-compatible gateway. + return True, True + + # MiniMax on its Anthropic-compatible endpoint serves its own + # model family (MiniMax-M2.7, M2.5, M2.1, M2) with documented + # cache_control support (0.1× read pricing, 5-minute TTL). The + # blanket is_claude gate above excludes these — opt them in + # explicitly via provider id or host match so users on + # provider=minimax / minimax-cn (or custom endpoints pointing at + # api.minimax.io/anthropic / api.minimaxi.com/anthropic) get the + # same cost reduction as Claude traffic. + # Docs: https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache + if is_anthropic_wire: + is_minimax_provider = provider_lower in {"minimax", "minimax-cn"} + is_minimax_host = ( + base_url_host_matches(eff_base_url, "api.minimax.io") + or base_url_host_matches(eff_base_url, "api.minimaxi.com") + ) + if is_minimax_provider or is_minimax_host: + return True, True + + # Qwen/Alibaba on OpenCode (Zen/Go) and native DashScope: OpenAI-wire + # transport that accepts Anthropic-style cache_control markers and + # rewards them with real cache hits. Without this branch + # qwen3.6-plus on opencode-go reports 0% cached tokens and burns + # through the subscription on every turn. + model_is_qwen = "qwen" in model_lower + provider_is_alibaba_family = provider_lower in { + "opencode", "opencode-zen", "opencode-go", "alibaba", + } + if provider_is_alibaba_family and model_is_qwen: + # Envelope layout (native_anthropic=False): markers on inner + # content parts, not top-level tool messages. Matches + # pi-mono's "alibaba" cacheControlFormat. + return True, False + + return False, False + + + +def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: bool) -> Any: + from agent.auxiliary_client import _validate_base_url, _validate_proxy_env_urls + # Treat client_kwargs as read-only. Callers pass agent._client_kwargs (or shallow + # copies of it) in; any in-place mutation leaks back into the stored dict and is + # reused on subsequent requests. #10933 hit this by injecting an httpx.Client + # transport that was torn down after the first request, so the next request + # wrapped a closed transport and raised "Cannot send a request, as the client + # has been closed" on every retry. The revert resolved that specific path; this + # copy locks the contract so future transport/keepalive work can't reintroduce + # the same class of bug. + client_kwargs = dict(client_kwargs) + _validate_proxy_env_urls() + _validate_base_url(client_kwargs.get("base_url")) + if agent.provider == "copilot-acp" or str(client_kwargs.get("base_url", "")).startswith("acp://copilot"): + from agent.copilot_acp_client import CopilotACPClient + + client = CopilotACPClient(**client_kwargs) + logger.info( + "Copilot ACP client created (%s, shared=%s) %s", + reason, + shared, + agent._client_log_context(), + ) + return client + if agent.provider == "google-gemini-cli" or str(client_kwargs.get("base_url", "")).startswith("cloudcode-pa://"): + from agent.gemini_cloudcode_adapter import GeminiCloudCodeClient + + # Strip OpenAI-specific kwargs the Gemini client doesn't accept + safe_kwargs = { + k: v for k, v in client_kwargs.items() + if k in {"api_key", "base_url", "default_headers", "project_id", "timeout"} + } + client = GeminiCloudCodeClient(**safe_kwargs) + logger.info( + "Gemini Cloud Code Assist client created (%s, shared=%s) %s", + reason, + shared, + agent._client_log_context(), + ) + return client + if agent.provider == "gemini": + from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url + + base_url = str(client_kwargs.get("base_url", "") or "") + if is_native_gemini_base_url(base_url): + safe_kwargs = { + k: v for k, v in client_kwargs.items() + if k in {"api_key", "base_url", "default_headers", "timeout", "http_client"} + } + if "http_client" not in safe_kwargs: + keepalive_http = agent._build_keepalive_http_client(base_url) + if keepalive_http is not None: + safe_kwargs["http_client"] = keepalive_http + client = GeminiNativeClient(**safe_kwargs) + logger.info( + "Gemini native client created (%s, shared=%s) %s", + reason, + shared, + agent._client_log_context(), + ) + return client + # Inject TCP keepalives so the kernel detects dead provider connections + # instead of letting them sit silently in CLOSE-WAIT (#10324). Without + # this, a peer that drops mid-stream leaves the socket in a state where + # epoll_wait never fires, ``httpx`` read timeout may not trigger, and + # the agent hangs until manually killed. Probes after 30s idle, retry + # every 10s, give up after 3 → dead peer detected within ~60s. + # + # Safety against #10933: the ``client_kwargs = dict(client_kwargs)`` + # above means this injection only lands in the local per-call copy, + # never back into ``agent._client_kwargs``. Each ``_create_openai_client`` + # invocation therefore gets its OWN fresh ``httpx.Client`` whose + # lifetime is tied to the OpenAI client it is passed to. When the + # OpenAI client is closed (rebuild, teardown, credential rotation), + # the paired ``httpx.Client`` closes with it, and the next call + # constructs a fresh one — no stale closed transport can be reused. + # Tests in ``tests/run_agent/test_create_openai_client_reuse.py`` and + # ``tests/run_agent/test_sequential_chats_live.py`` pin this invariant. + if "http_client" not in client_kwargs: + keepalive_http = agent._build_keepalive_http_client(client_kwargs.get("base_url", "")) + if keepalive_http is not None: + client_kwargs["http_client"] = keepalive_http + # Uses the module-level `OpenAI` name, resolved lazily on first + # access via __getattr__ below. Tests patch via `run_agent.OpenAI`. + client = _ra().OpenAI(**client_kwargs) + logger.info( + "OpenAI client created (%s, shared=%s) %s", + reason, + shared, + agent._client_log_context(), + ) + return client + + + +__all__ = [ + "convert_to_trajectory_format", + "sanitize_tool_call_arguments", + "repair_message_sequence", + "strip_think_blocks", + "recover_with_credential_pool", + "try_recover_primary_transport", + "drop_thinking_only_and_merge_users", + "restore_primary_runtime", + "extract_reasoning", + "dump_api_request_debug", + "anthropic_prompt_cache_policy", + "create_openai_client", +] diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py new file mode 100644 index 00000000000..73a455f6bb0 --- /dev/null +++ b/agent/codex_runtime.py @@ -0,0 +1,400 @@ +"""Codex API runtime — App Server and Responses-API streaming paths. + +Extracted from :class:`AIAgent` to keep the agent loop file focused. +Each function takes the parent ``AIAgent`` as its first argument +(``agent``). AIAgent keeps thin forwarder methods for backward +compatibility. + +* ``run_codex_app_server_turn`` — drives one turn through the + ``codex_app_server`` subprocess client (used when a Codex CLI install + is the active provider). +* ``run_codex_stream`` — streams a Codex Responses API call (the + ``codex_responses`` api_mode). +* ``run_codex_create_stream_fallback`` — recovery path when the + Responses ``stream=True`` initial create fails. +""" + +from __future__ import annotations + +import contextvars +import json +import logging +import os +import threading +import time +import uuid +from types import SimpleNamespace +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +def _ra(): + """Lazy ``run_agent`` reference for test-patch routing.""" + import run_agent + return run_agent + + + +def run_codex_app_server_turn( + agent, + *, + user_message: str, + original_user_message: Any, + messages: List[Dict[str, Any]], + effective_task_id: str, + should_review_memory: bool = False, +) -> Dict[str, Any]: + """Codex app-server runtime path. Hands the entire turn to a `codex + app-server` subprocess and projects its events back into Hermes' + messages list so memory/skill review keep working. + + Called from run_conversation() when agent.api_mode == "codex_app_server". + Returns the same dict shape as the chat_completions path. + """ + from agent.transports.codex_app_server_session import CodexAppServerSession + + # Lazy session: one CodexAppServerSession per AIAgent instance. + # Spawned on first turn, reused across turns, closed at AIAgent + # shutdown (see _cleanup hook). + if not hasattr(agent, "_codex_session") or agent._codex_session is None: + cwd = getattr(agent, "session_cwd", None) or os.getcwd() + # Approval callback: defer to Hermes' standard prompt flow if a + # CLI thread has installed one. Gateway / cron contexts get the + # codex-side fail-closed default. + try: + from tools.terminal_tool import _get_approval_callback + approval_callback = _get_approval_callback() + except Exception: + approval_callback = None + agent._codex_session = CodexAppServerSession( + cwd=cwd, + approval_callback=approval_callback, + ) + + # NOTE: the user message is ALREADY appended to messages by the + # standard run_conversation() flow (line ~11823) before the early + # return reaches us. Do NOT append again — that would duplicate. + + try: + turn = agent._codex_session.run_turn(user_input=user_message) + except Exception as exc: + logger.exception("codex app-server turn failed") + # Crash → unconditionally drop the session so the next turn + # respawns from scratch instead of reusing a dead client. + try: + agent._codex_session.close() + except Exception: + pass + agent._codex_session = None + return { + "final_response": ( + f"Codex app-server turn failed: {exc}. " + f"Fall back to default runtime with `/codex-runtime auto`." + ), + "messages": messages, + "api_calls": 0, + "completed": False, + "partial": True, + "error": str(exc), + } + + # If the turn signalled the underlying client is wedged (deadline + # blown, post-tool watchdog tripped, OAuth refresh died, subprocess + # exited), retire the session so the next turn respawns codex + # rather than riding the broken process. Mirrors openclaw beta.8's + # "retire timed-out app-server clients" fix. + if getattr(turn, "should_retire", False): + logger.warning( + "codex app-server session retired (turn error: %s)", + turn.error, + ) + try: + agent._codex_session.close() + except Exception: + pass + agent._codex_session = None + + # Splice projected messages into the conversation. The projector emits + # standard {role, content, tool_calls, tool_call_id} entries, which + # is exactly what curator.py / sessions DB expect. + if turn.projected_messages: + messages.extend(turn.projected_messages) + + # Counter ticks for the agent-improvement loop. + # _turns_since_memory and _user_turn_count are ALREADY incremented + # in the run_conversation() pre-loop block (lines ~11793-11817) so we + # do NOT touch them here — that would double-count. + # Only _iters_since_skill needs explicit increment, since the + # chat_completions loop bumps it per tool iteration (line ~12110) + # and that loop is bypassed on this path. + agent._iters_since_skill = ( + getattr(agent, "_iters_since_skill", 0) + turn.tool_iterations + ) + + # Now check the skill nudge AFTER iters were incremented — same + # pattern the chat_completions path uses (line ~15432). + should_review_skills = False + if ( + agent._skill_nudge_interval > 0 + and agent._iters_since_skill >= agent._skill_nudge_interval + and "skill_manage" in agent.valid_tool_names + ): + should_review_skills = True + agent._iters_since_skill = 0 + + # External memory provider sync (mirrors line ~15439). Skipped on + # interrupt/error to avoid feeding partial transcripts to memory. + if not turn.interrupted and turn.error is None: + try: + agent._sync_external_memory_for_turn( + original_user_message=original_user_message, + final_response=turn.final_text, + interrupted=False, + ) + except Exception: + logger.debug("external memory sync raised", exc_info=True) + + # Background review fork — same cadence + signature as the default + # path (line ~15449). Only fires when a trigger actually tripped AND + # we have a real final response. + if ( + turn.final_text + and not turn.interrupted + and (should_review_memory or should_review_skills) + ): + try: + agent._spawn_background_review( + messages_snapshot=list(messages), + review_memory=should_review_memory, + review_skills=should_review_skills, + ) + except Exception: + logger.debug("background review spawn raised", exc_info=True) + + return { + "final_response": turn.final_text, + "messages": messages, + "api_calls": 1, # one app-server "turn" maps to one logical API call + "completed": not turn.interrupted and turn.error is None, + "partial": turn.interrupted or turn.error is not None, + "error": turn.error, + "codex_thread_id": turn.thread_id, + "codex_turn_id": turn.turn_id, + } + + + + +def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta: callable = None): + """Execute one streaming Responses API request and return the final response.""" + import httpx as _httpx + + active_client = client or agent._ensure_primary_openai_client(reason="codex_stream_direct") + max_stream_retries = 1 + has_tool_calls = False + first_delta_fired = False + # Accumulate streamed text so we can recover if get_final_response() + # returns empty output (e.g. chatgpt.com backend-api sends + # response.incomplete instead of response.completed). + agent._codex_streamed_text_parts: list = [] + for attempt in range(max_stream_retries + 1): + if agent._interrupt_requested: + raise InterruptedError("Agent interrupted before Codex stream retry") + collected_output_items: list = [] + try: + with active_client.responses.stream(**api_kwargs) as stream: + for event in stream: + agent._touch_activity("receiving stream response") + if agent._interrupt_requested: + break + event_type = getattr(event, "type", "") + # Fire callbacks on text content deltas (suppress during tool calls) + if "output_text.delta" in event_type or event_type == "response.output_text.delta": + delta_text = getattr(event, "delta", "") + if delta_text: + agent._codex_streamed_text_parts.append(delta_text) + if delta_text and not has_tool_calls: + if not first_delta_fired: + first_delta_fired = True + if on_first_delta: + try: + on_first_delta() + except Exception: + pass + agent._fire_stream_delta(delta_text) + # Track tool calls to suppress text streaming + elif "function_call" in event_type: + has_tool_calls = True + # Fire reasoning callbacks + elif "reasoning" in event_type and "delta" in event_type: + reasoning_text = getattr(event, "delta", "") + if reasoning_text: + agent._fire_reasoning_delta(reasoning_text) + # Collect completed output items — some backends + # (chatgpt.com/backend-api/codex) stream valid items + # via response.output_item.done but the SDK's + # get_final_response() returns an empty output list. + elif event_type == "response.output_item.done": + done_item = getattr(event, "item", None) + if done_item is not None: + collected_output_items.append(done_item) + # Log non-completed terminal events for diagnostics + elif event_type in {"response.incomplete", "response.failed"}: + resp_obj = getattr(event, "response", None) + status = getattr(resp_obj, "status", None) if resp_obj else None + incomplete_details = getattr(resp_obj, "incomplete_details", None) if resp_obj else None + logger.warning( + "Codex Responses stream received terminal event %s " + "(status=%s, incomplete_details=%s, streamed_chars=%d). %s", + event_type, status, incomplete_details, + sum(len(p) for p in agent._codex_streamed_text_parts), + agent._client_log_context(), + ) + final_response = stream.get_final_response() + # PATCH: ChatGPT Codex backend streams valid output items + # but get_final_response() can return an empty output list. + # Backfill from collected items or synthesize from deltas. + _out = getattr(final_response, "output", None) + if isinstance(_out, list) and not _out: + if collected_output_items: + final_response.output = list(collected_output_items) + logger.debug( + "Codex stream: backfilled %d output items from stream events", + len(collected_output_items), + ) + elif agent._codex_streamed_text_parts and not has_tool_calls: + assembled = "".join(agent._codex_streamed_text_parts) + final_response.output = [SimpleNamespace( + type="message", + role="assistant", + status="completed", + content=[SimpleNamespace(type="output_text", text=assembled)], + )] + logger.debug( + "Codex stream: synthesized output from %d text deltas (%d chars)", + len(agent._codex_streamed_text_parts), len(assembled), + ) + return final_response + except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: + if attempt < max_stream_retries: + logger.debug( + "Codex Responses stream transport failed (attempt %s/%s); retrying. %s error=%s", + attempt + 1, + max_stream_retries + 1, + agent._client_log_context(), + exc, + ) + continue + logger.debug( + "Codex Responses stream transport failed; falling back to create(stream=True). %s error=%s", + agent._client_log_context(), + exc, + ) + return agent._run_codex_create_stream_fallback(api_kwargs, client=active_client) + except RuntimeError as exc: + err_text = str(exc) + missing_completed = "response.completed" in err_text + if missing_completed and attempt < max_stream_retries: + logger.debug( + "Responses stream closed before completion (attempt %s/%s); retrying. %s", + attempt + 1, + max_stream_retries + 1, + agent._client_log_context(), + ) + continue + if missing_completed: + logger.debug( + "Responses stream did not emit response.completed; falling back to create(stream=True). %s", + agent._client_log_context(), + ) + return agent._run_codex_create_stream_fallback(api_kwargs, client=active_client) + raise + + + +def run_codex_create_stream_fallback(agent, api_kwargs: dict, client: Any = None): + """Fallback path for stream completion edge cases on Codex-style Responses backends.""" + active_client = client or agent._ensure_primary_openai_client(reason="codex_create_stream_fallback") + fallback_kwargs = dict(api_kwargs) + fallback_kwargs["stream"] = True + fallback_kwargs = agent._get_transport().preflight_kwargs(fallback_kwargs, allow_stream=True) + stream_or_response = active_client.responses.create(**fallback_kwargs) + + # Compatibility shim for mocks or providers that still return a concrete response. + if hasattr(stream_or_response, "output"): + return stream_or_response + if not hasattr(stream_or_response, "__iter__"): + return stream_or_response + + terminal_response = None + collected_output_items: list = [] + collected_text_deltas: list = [] + try: + for event in stream_or_response: + agent._touch_activity("receiving stream response") + event_type = getattr(event, "type", None) + if not event_type and isinstance(event, dict): + event_type = event.get("type") + + # Collect output items and text deltas for backfill + if event_type == "response.output_item.done": + done_item = getattr(event, "item", None) + if done_item is None and isinstance(event, dict): + done_item = event.get("item") + if done_item is not None: + collected_output_items.append(done_item) + elif event_type in {"response.output_text.delta",}: + delta = getattr(event, "delta", "") + if not delta and isinstance(event, dict): + delta = event.get("delta", "") + if delta: + collected_text_deltas.append(delta) + + if event_type not in {"response.completed", "response.incomplete", "response.failed"}: + continue + + terminal_response = getattr(event, "response", None) + if terminal_response is None and isinstance(event, dict): + terminal_response = event.get("response") + if terminal_response is not None: + # Backfill empty output from collected stream events + _out = getattr(terminal_response, "output", None) + if isinstance(_out, list) and not _out: + if collected_output_items: + terminal_response.output = list(collected_output_items) + logger.debug( + "Codex fallback stream: backfilled %d output items", + len(collected_output_items), + ) + elif collected_text_deltas: + assembled = "".join(collected_text_deltas) + terminal_response.output = [SimpleNamespace( + type="message", role="assistant", + status="completed", + content=[SimpleNamespace(type="output_text", text=assembled)], + )] + logger.debug( + "Codex fallback stream: synthesized from %d deltas (%d chars)", + len(collected_text_deltas), len(assembled), + ) + return terminal_response + finally: + close_fn = getattr(stream_or_response, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + pass + + if terminal_response is not None: + return terminal_response + raise RuntimeError("Responses create(stream=True) fallback did not emit a terminal response.") + + + +__all__ = [ + "run_codex_app_server_turn", + "run_codex_stream", + "run_codex_create_stream_fallback", +] diff --git a/run_agent.py b/run_agent.py index 4b5e405018e..8d6f7c3f35c 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2303,101 +2303,9 @@ class AIAgent: api_mode: Optional[str] = None, model: Optional[str] = None, ) -> tuple[bool, bool]: - """Decide whether to apply Anthropic prompt caching and which layout to use. - - Returns ``(should_cache, use_native_layout)``: - * ``should_cache`` — inject ``cache_control`` breakpoints for this - request (applies to OpenRouter Claude, native Anthropic, and - third-party gateways that speak the native Anthropic protocol). - * ``use_native_layout`` — place markers on the *inner* content - blocks (native Anthropic accepts and requires this layout); - when False markers go on the message envelope (OpenRouter and - OpenAI-wire proxies expect the looser layout). - - Third-party providers using the native Anthropic transport - (``api_mode == 'anthropic_messages'`` + Claude-named model) get - caching with the native layout so they benefit from the same - cost reduction as direct Anthropic callers, provided their - gateway implements the Anthropic cache_control contract - (MiniMax, Zhipu GLM, LiteLLM's Anthropic proxy mode all do). - - Qwen / Alibaba-family models on OpenCode, OpenCode Go, and direct - Alibaba (DashScope) also honour Anthropic-style ``cache_control`` - markers on OpenAI-wire chat completions. Upstream pi-mono #3392 / - pi #3393 documented this for opencode-go Qwen. Without markers - these providers serve zero cache hits, re-billing the full prompt - on every turn. - """ - eff_provider = (provider if provider is not None else self.provider) or "" - eff_base_url = base_url if base_url is not None else (self.base_url or "") - eff_api_mode = api_mode if api_mode is not None else (self.api_mode or "") - eff_model = (model if model is not None else self.model) or "" - - model_lower = eff_model.lower() - provider_lower = eff_provider.lower() - is_claude = "claude" in model_lower - is_openrouter = base_url_host_matches(eff_base_url, "openrouter.ai") - # Nous Portal proxies to OpenRouter behind the scenes — identical - # OpenAI-wire envelope cache_control semantics. Treat it as an - # OpenRouter-equivalent endpoint for caching layout purposes. - is_nous_portal = "nousresearch" in eff_base_url.lower() - is_anthropic_wire = eff_api_mode == "anthropic_messages" - is_native_anthropic = ( - is_anthropic_wire - and (eff_provider == "anthropic" or base_url_hostname(eff_base_url) == "api.anthropic.com") - ) - - if is_native_anthropic: - return True, True - if (is_openrouter or is_nous_portal) and is_claude: - return True, False - # Nous Portal Qwen (e.g. qwen3.6-plus) takes the same envelope-layout - # cache_control path as Portal Claude. Portal proxies to OpenRouter - # and the upstream Qwen route accepts cache_control markers; without - # this branch the alibaba-family check below only matches - # provider=opencode/alibaba and Portal traffic falls through to - # (False, False), serving 0% cache hits and re-billing the full - # prompt on every turn. - if is_nous_portal and "qwen" in model_lower: - return True, False - if is_anthropic_wire and is_claude: - # Third-party Anthropic-compatible gateway. - return True, True - - # MiniMax on its Anthropic-compatible endpoint serves its own - # model family (MiniMax-M2.7, M2.5, M2.1, M2) with documented - # cache_control support (0.1× read pricing, 5-minute TTL). The - # blanket is_claude gate above excludes these — opt them in - # explicitly via provider id or host match so users on - # provider=minimax / minimax-cn (or custom endpoints pointing at - # api.minimax.io/anthropic / api.minimaxi.com/anthropic) get the - # same cost reduction as Claude traffic. - # Docs: https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache - if is_anthropic_wire: - is_minimax_provider = provider_lower in {"minimax", "minimax-cn"} - is_minimax_host = ( - base_url_host_matches(eff_base_url, "api.minimax.io") - or base_url_host_matches(eff_base_url, "api.minimaxi.com") - ) - if is_minimax_provider or is_minimax_host: - return True, True - - # Qwen/Alibaba on OpenCode (Zen/Go) and native DashScope: OpenAI-wire - # transport that accepts Anthropic-style cache_control markers and - # rewards them with real cache hits. Without this branch - # qwen3.6-plus on opencode-go reports 0% cached tokens and burns - # through the subscription on every turn. - model_is_qwen = "qwen" in model_lower - provider_is_alibaba_family = provider_lower in { - "opencode", "opencode-zen", "opencode-go", "alibaba", - } - if provider_is_alibaba_family and model_is_qwen: - # Envelope layout (native_anthropic=False): markers on inner - # content parts, not top-level tool messages. Matches - # pi-mono's "alibaba" cacheControlFormat. - return True, False - - return False, False + """Forwarder — see ``agent.agent_runtime_helpers.anthropic_prompt_cache_policy``.""" + from agent.agent_runtime_helpers import anthropic_prompt_cache_policy + return anthropic_prompt_cache_policy(self, provider=provider, base_url=base_url, api_mode=api_mode, model=model) @staticmethod def _model_requires_responses_api(model: str) -> bool: @@ -2473,98 +2381,9 @@ class AIAgent: return bool(cleaned.strip()) def _strip_think_blocks(self, content: str) -> str: - """Remove reasoning/thinking blocks from content, returning only visible text. - - Handles four cases: - 1. Closed tag pairs (``…``) — the common path when - the provider emits complete reasoning blocks. - 2. Unterminated open tag at a block boundary (start of text or - after a newline) — e.g. MiniMax M2.7 / NIM endpoints where the - closing tag is dropped. Everything from the open tag to end - of string is stripped. The block-boundary check mirrors - ``gateway/stream_consumer.py``'s filter so models that mention - ```` in prose aren't over-stripped. - 3. Stray orphan open/close tags that slip through. - 4. Tag variants: ````, ````, ````, - ````, ```` (Gemma 4), all - case-insensitive. - - Additionally strips standalone tool-call XML blocks that some open - models (notably Gemma variants on OpenRouter) emit inside assistant - content instead of via the structured ``tool_calls`` field: - * ``…`` - * ``…`` - * ``…`` - * ``…`` - * ``…`` - * ``…`` (Gemma style) - Ported from openclaw/openclaw#67318. The ```` variant is - boundary-gated (only strips when the tag sits at start-of-line or - after punctuation and carries a ``name="..."`` attribute) so prose - mentions like "Use in JavaScript" are preserved. - """ - if not content: - return "" - # 1. Closed tag pairs — case-insensitive for all variants so - # mixed-case tags (, ) don't slip through to - # the unterminated-tag pass and take trailing content with them. - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - # 1b. Tool-call XML blocks (openclaw/openclaw#67318). Handle the - # generic tag names first — they have no attribute gating since - # a literal in prose is already vanishingly rare. - for _tc_name in ("tool_call", "tool_calls", "tool_result", - "function_call", "function_calls"): - content = re.sub( - rf'<{_tc_name}\b[^>]*>.*?', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) - # 1c. ... — Gemma-style standalone - # tool call. Only strip when the tag sits at a block boundary - # (start of text, after a newline, or after sentence-ending - # punctuation) AND carries a name="..." attribute. This keeps - # prose mentions like "Use to declare" safe. - content = re.sub( - r'(?:(?<=^)|(?<=[\n\r.!?:]))[ \t]*' - r']*\bname\s*=[^>]*>' - r'(?:(?:(?!).)*)', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) - # 2. Unterminated reasoning block — open tag at a block boundary - # (start of text, or after a newline) with no matching close. - # Strip from the tag to end of string. Fixes #8878 / #9568 - # (MiniMax M2.7 leaking raw reasoning into assistant content). - content = re.sub( - r'(?:^|\n)[ \t]*<(?:think|thinking|reasoning|thought|REASONING_SCRATCHPAD)\b[^>]*>.*$', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) - # 3. Stray orphan open/close tags that slipped through. - content = re.sub( - r'\s*', - '', - content, - flags=re.IGNORECASE, - ) - # 3b. Stray tool-call closers. (We do NOT strip bare or - # unterminated because a truncated tail - # during streaming may still be valuable to the user; matches - # OpenClaw's intentional asymmetry.) - content = re.sub( - r'\s*', - '', - content, - flags=re.IGNORECASE, - ) - return content + """Forwarder — see ``agent.agent_runtime_helpers.strip_think_blocks``.""" + from agent.agent_runtime_helpers import strip_think_blocks + return strip_think_blocks(self, content) @staticmethod def _has_natural_response_ending(content: str) -> bool: @@ -2692,84 +2511,9 @@ class AIAgent: def _extract_reasoning(self, assistant_message) -> Optional[str]: - """ - Extract reasoning/thinking content from an assistant message. - - OpenRouter and various providers can return reasoning in multiple formats: - 1. message.reasoning - Direct reasoning field (DeepSeek, Qwen, etc.) - 2. message.reasoning_content - Alternative field (Moonshot AI, Novita, etc.) - 3. message.reasoning_details - Array of {type, summary, ...} objects (OpenRouter unified) - - Args: - assistant_message: The assistant message object from the API response - - Returns: - Combined reasoning text, or None if no reasoning found - """ - reasoning_parts = [] - - # Check direct reasoning field - if hasattr(assistant_message, 'reasoning') and assistant_message.reasoning: - reasoning_parts.append(assistant_message.reasoning) - - # Check reasoning_content field (alternative name used by some providers) - if hasattr(assistant_message, 'reasoning_content') and assistant_message.reasoning_content: - # Don't duplicate if same as reasoning - if assistant_message.reasoning_content not in reasoning_parts: - reasoning_parts.append(assistant_message.reasoning_content) - - # Check reasoning_details array (OpenRouter unified format) - # Format: [{"type": "reasoning.summary", "summary": "...", ...}, ...] - if hasattr(assistant_message, 'reasoning_details') and assistant_message.reasoning_details: - for detail in assistant_message.reasoning_details: - if isinstance(detail, dict): - # Extract summary from reasoning detail object - summary = ( - detail.get('summary') - or detail.get('thinking') - or detail.get('content') - or detail.get('text') - ) - if summary and summary not in reasoning_parts: - reasoning_parts.append(summary) - - # Some providers embed reasoning directly inside assistant content - # instead of returning structured reasoning fields. Only fall back - # to inline extraction when no structured reasoning was found. - content = getattr(assistant_message, "content", None) - if not reasoning_parts and isinstance(content, list): - # DeepSeek V4 Pro (and compatible providers) return content as a - # list of typed blocks, e.g.: - # [{"type": "thinking", "thinking": "..."}, {"type": "output", ...}] - # Without this branch the thinking text is silently dropped and the - # next turn fails with HTTP 400 ("thinking must be passed back"). - # Refs #21944. - for block in content: - if isinstance(block, dict) and block.get("type") == "thinking": - thinking_text = block.get("thinking") or block.get("text") or "" - thinking_text = thinking_text.strip() - if thinking_text and thinking_text not in reasoning_parts: - reasoning_parts.append(thinking_text) - if not reasoning_parts and isinstance(content, str) and content: - inline_patterns = ( - r"(.*?)", - r"(.*?)", - r"(.*?)", - r"(.*?)", - r"(.*?)", - ) - for pattern in inline_patterns: - flags = re.DOTALL | re.IGNORECASE - for block in re.findall(pattern, content, flags=flags): - cleaned = block.strip() - if cleaned and cleaned not in reasoning_parts: - reasoning_parts.append(cleaned) - - # Combine all reasoning parts - if reasoning_parts: - return "\n\n".join(reasoning_parts) - - return None + """Forwarder — see ``agent.agent_runtime_helpers.extract_reasoning``.""" + from agent.agent_runtime_helpers import extract_reasoning + return extract_reasoning(self, assistant_message) def _cleanup_task_resources(self, task_id: str) -> None: """Forwarder — see ``agent.chat_completion_helpers.cleanup_task_resources``.""" @@ -3097,104 +2841,9 @@ class AIAgent: messages.pop() def _repair_message_sequence(self, messages: List[Dict]) -> int: - """Collapse malformed role-alternation left in the live history. - - Providers (OpenAI, OpenRouter, Anthropic) expect strict alternation: - after the system message, user/tool alternates with assistant, with - no two consecutive user messages and no tool-result that doesn't - follow an assistant-with-tool_calls. Violations cause silent empty - responses on most providers, which triggers the empty-retry loop. - - This runs right before the API call as a defensive belt — by the - time it fires, the scaffolding strip should already have prevented - most shapes, but external callers (gateway multi-queue replay, - session resume, cron, explicit conversation_history passed in by - host code) can feed in already-broken histories. - - Repairs applied: - 1. Stray ``tool`` messages whose ``tool_call_id`` doesn't match - any preceding assistant tool_call — dropped. - 2. Consecutive ``user`` messages — merged with newline separator - so no user input is lost. - - Deliberately does NOT rewind orphan ``assistant(tool_calls)+tool`` - pairs that precede a user message — that pattern IS valid when the - previous turn completed normally and the user jumped in to redirect - before the model got a continuation turn (the ongoing dialog - pattern). The empty-response scaffolding stripper handles the - genuinely-broken variant via its flag-gated rewind. - - Returns the number of repairs made (for logging/telemetry). - """ - if not messages: - return 0 - - repairs = 0 - - # Pass 1: drop stray tool messages that don't follow a known - # assistant tool_call_id. Uses a rolling set of known ids refreshed - # on each assistant message. - known_tool_ids: set = set() - filtered: List[Dict] = [] - for msg in messages: - if not isinstance(msg, dict): - filtered.append(msg) - continue - role = msg.get("role") - if role == "assistant": - known_tool_ids = set() - for tc in (msg.get("tool_calls") or []): - tc_id = tc.get("id") if isinstance(tc, dict) else None - if tc_id: - known_tool_ids.add(tc_id) - filtered.append(msg) - elif role == "tool": - tc_id = msg.get("tool_call_id") - if tc_id and tc_id in known_tool_ids: - filtered.append(msg) - else: - repairs += 1 - else: - if role == "user": - # A user turn closes the tool-result run; subsequent - # tool messages without a fresh assistant tool_call - # are orphans. - known_tool_ids = set() - filtered.append(msg) - - # Pass 2: merge consecutive user messages. Preserves all user input - # so nothing the user typed is lost. - merged: List[Dict] = [] - for msg in filtered: - if ( - merged - and isinstance(msg, dict) - and msg.get("role") == "user" - and isinstance(merged[-1], dict) - and merged[-1].get("role") == "user" - ): - prev = merged[-1] - prev_content = prev.get("content", "") - new_content = msg.get("content", "") - # Only merge plain-text content; leave multimodal (list) - # content alone — collapsing image/audio blocks risks - # mangling the attachment structure. - if isinstance(prev_content, str) and isinstance(new_content, str): - prev["content"] = ( - (prev_content + "\n\n" + new_content) - if prev_content and new_content - else (prev_content or new_content) - ) - repairs += 1 - continue - merged.append(msg) - - if repairs > 0: - # Rewrite in place so downstream paths (persistence, return - # value, session DB flush) see the repaired sequence. - messages[:] = merged - - return repairs + """Forwarder — see ``agent.agent_runtime_helpers.repair_message_sequence``.""" + from agent.agent_runtime_helpers import repair_message_sequence + return repair_message_sequence(self, messages) def _flush_messages_to_session_db(self, messages: List[Dict], conversation_history: List[Dict] = None): """Persist any un-flushed messages to the SQLite session store. @@ -3292,173 +2941,9 @@ class AIAgent: return format_tools_for_system_message(self) def _convert_to_trajectory_format(self, messages: List[Dict[str, Any]], user_query: str, completed: bool) -> List[Dict[str, Any]]: - """ - Convert internal message format to trajectory format for saving. - - Args: - messages (List[Dict]): Internal message history - user_query (str): Original user query - completed (bool): Whether the conversation completed successfully - - Returns: - List[Dict]: Messages in trajectory format - """ - # Normalize multimodal tool results — trajectories are text-only, so - # replace image-bearing tool messages with their text_summary to avoid - # embedding ~1MB base64 blobs into every saved trajectory. - messages = [_trajectory_normalize_msg(m) for m in messages] - trajectory = [] - - # Add system message with tool definitions - system_msg = ( - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " - " XML tags. Here are the available tools:\n" - f"\n{self._format_tools_for_system_message()}\n\n" - "For each function call return a JSON object, with the following pydantic model json schema for each:\n" - "{'title': 'FunctionCall', 'type': 'object', 'properties': {'name': {'title': 'Name', 'type': 'string'}, " - "'arguments': {'title': 'Arguments', 'type': 'object'}}, 'required': ['name', 'arguments']}\n" - "Each function call should be enclosed within XML tags.\n" - "Example:\n\n{'name': ,'arguments': }\n" - ) - - trajectory.append({ - "from": "system", - "value": system_msg - }) - - # Add the actual user prompt (from the dataset) as the first human message - trajectory.append({ - "from": "human", - "value": user_query - }) - - # Skip the first message (the user query) since we already added it above. - # Prefill messages are injected at API-call time only (not in the messages - # list), so no offset adjustment is needed here. - i = 1 - - while i < len(messages): - msg = messages[i] - - if msg["role"] == "assistant": - # Check if this message has tool calls - if "tool_calls" in msg and msg["tool_calls"]: - # Format assistant message with tool calls - # Add tags around reasoning for trajectory storage - content = "" - - # Prepend reasoning in tags if available (native thinking tokens) - if msg.get("reasoning") and msg["reasoning"].strip(): - content = f"\n{msg['reasoning']}\n\n" - - if msg.get("content") and msg["content"].strip(): - # Convert any tags to tags - # (used when native thinking is disabled and model reasons via XML) - content += convert_scratchpad_to_think(msg["content"]) + "\n" - - # Add tool calls wrapped in XML tags - for tool_call in msg["tool_calls"]: - if not tool_call or not isinstance(tool_call, dict): continue - # Parse arguments - should always succeed since we validate during conversation - # but keep try-except as safety net - try: - arguments = json.loads(tool_call["function"]["arguments"]) if isinstance(tool_call["function"]["arguments"], str) else tool_call["function"]["arguments"] - except json.JSONDecodeError: - # This shouldn't happen since we validate and retry during conversation, - # but if it does, log warning and use empty dict - logging.warning(f"Unexpected invalid JSON in trajectory conversion: {tool_call['function']['arguments'][:100]}") - arguments = {} - - tool_call_json = { - "name": tool_call["function"]["name"], - "arguments": arguments - } - content += f"\n{json.dumps(tool_call_json, ensure_ascii=False)}\n\n" - - # Ensure every gpt turn has a block (empty if no reasoning) - # so the format is consistent for training data - if "" not in content: - content = "\n\n" + content - - trajectory.append({ - "from": "gpt", - "value": content.rstrip() - }) - - # Collect all subsequent tool responses - tool_responses = [] - j = i + 1 - while j < len(messages) and messages[j]["role"] == "tool": - tool_msg = messages[j] - # Format tool response with XML tags - tool_response = "\n" - - # Try to parse tool content as JSON if it looks like JSON - tool_content = tool_msg["content"] - try: - if tool_content.strip().startswith(("{", "[")): - tool_content = json.loads(tool_content) - except (json.JSONDecodeError, AttributeError): - pass # Keep as string if not valid JSON - - tool_index = len(tool_responses) - tool_name = ( - msg["tool_calls"][tool_index]["function"]["name"] - if tool_index < len(msg["tool_calls"]) - else "unknown" - ) - tool_response += json.dumps({ - "tool_call_id": tool_msg.get("tool_call_id", ""), - "name": tool_name, - "content": tool_content - }, ensure_ascii=False) - tool_response += "\n" - tool_responses.append(tool_response) - j += 1 - - # Add all tool responses as a single message - if tool_responses: - trajectory.append({ - "from": "tool", - "value": "\n".join(tool_responses) - }) - i = j - 1 # Skip the tool messages we just processed - - else: - # Regular assistant message without tool calls - # Add tags around reasoning for trajectory storage - content = "" - - # Prepend reasoning in tags if available (native thinking tokens) - if msg.get("reasoning") and msg["reasoning"].strip(): - content = f"\n{msg['reasoning']}\n\n" - - # Convert any tags to tags - # (used when native thinking is disabled and model reasons via XML) - raw_content = msg["content"] or "" - content += convert_scratchpad_to_think(raw_content) - - # Ensure every gpt turn has a block (empty if no reasoning) - if "" not in content: - content = "\n\n" + content - - trajectory.append({ - "from": "gpt", - "value": content.strip() - }) - - elif msg["role"] == "user": - trajectory.append({ - "from": "human", - "value": msg["content"] - }) - - i += 1 - - return trajectory + """Forwarder — see ``agent.agent_runtime_helpers.convert_to_trajectory_format``.""" + from agent.agent_runtime_helpers import convert_to_trajectory_format + return convert_to_trajectory_format(self, messages, user_query, completed) def _save_trajectory(self, messages: List[Dict[str, Any]], user_query: str, completed: bool): """ @@ -3636,80 +3121,9 @@ class AIAgent: reason: str, error: Optional[Exception] = None, ) -> Optional[Path]: - """ - Dump a debug-friendly HTTP request record for the active inference API. - - Captures the request body from api_kwargs (excluding transport-only keys - like timeout). Intended for debugging provider-side 4xx failures where - retries are not useful. - """ - try: - body = copy.deepcopy(api_kwargs) - body.pop("timeout", None) - body = {k: v for k, v in body.items() if v is not None} - - api_key = None - try: - api_key = getattr(self.client, "api_key", None) - except Exception as e: - logger.debug("Could not extract API key for debug dump: %s", e) - - dump_payload: Dict[str, Any] = { - "timestamp": datetime.now().isoformat(), - "session_id": self.session_id, - "reason": reason, - "request": { - "method": "POST", - "url": f"{self.base_url.rstrip('/')}{'/responses' if self.api_mode == 'codex_responses' else '/chat/completions'}", - "headers": { - "Authorization": f"Bearer {self._mask_api_key_for_logs(api_key)}", - "Content-Type": "application/json", - }, - "body": body, - }, - } - - if error is not None: - error_info: Dict[str, Any] = { - "type": type(error).__name__, - "message": str(error), - } - for attr_name in ("status_code", "request_id", "code", "param", "type"): - attr_value = getattr(error, attr_name, None) - if attr_value is not None: - error_info[attr_name] = attr_value - - body_attr = getattr(error, "body", None) - if body_attr is not None: - error_info["body"] = body_attr - - response_obj = getattr(error, "response", None) - if response_obj is not None: - try: - error_info["response_status"] = getattr(response_obj, "status_code", None) - error_info["response_text"] = response_obj.text - except Exception as e: - logger.debug("Could not extract error response details: %s", e) - - dump_payload["error"] = error_info - - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") - dump_file = self.logs_dir / f"request_dump_{self.session_id}_{timestamp}.json" - dump_file.write_text( - json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str), - encoding="utf-8", - ) - - self._vprint(f"{self.log_prefix}🧾 Request debug dump written to: {dump_file}") - - if env_var_enabled("HERMES_DUMP_REQUEST_STDOUT"): - print(json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str)) - - return dump_file - except Exception as dump_error: - if self.verbose_logging: - logging.warning(f"Failed to dump API request debug payload: {dump_error}") - return None + """Forwarder — see ``agent.agent_runtime_helpers.dump_api_request_debug``.""" + from agent.agent_runtime_helpers import dump_api_request_debug + return dump_api_request_debug(self, api_kwargs, reason=reason, error=error) @staticmethod def _clean_session_content(content: str) -> str: @@ -4584,86 +3998,9 @@ class AIAgent: def _drop_thinking_only_and_merge_users( messages: List[Dict[str, Any]], ) -> List[Dict[str, Any]]: - """Drop thinking-only assistant turns; merge any adjacent user messages left behind. - - Runs on the per-call ``api_messages`` copy only. The stored - conversation history (``self.messages``) is never mutated, so the - user still sees the thinking block in the CLI/gateway transcript and - session persistence keeps the full trace. Only the wire copy sent to - the provider is cleaned. - - Why drop-and-merge rather than inject stub text: - - Fabricating ``"."`` / ``"(continued)"`` text lies in the history - and makes future turns see model output the model didn't emit. - - Dropping the turn preserves honesty; merging adjacent user messages - preserves the provider's role-alternation invariant. - - This is the pattern used by Claude Code's ``normalizeMessagesForAPI`` - (filterOrphanedThinkingOnlyMessages + mergeAdjacentUserMessages). - """ - if not messages: - return messages - - # Pass 1: drop thinking-only assistant turns. - kept = [m for m in messages if not AIAgent._is_thinking_only_assistant(m)] - dropped = len(messages) - len(kept) - if dropped == 0: - return messages - - # Pass 2: merge any newly-adjacent user messages. - merged: List[Dict[str, Any]] = [] - merges = 0 - for m in kept: - prev = merged[-1] if merged else None - if ( - prev is not None - and prev.get("role") == "user" - and m.get("role") == "user" - ): - prev_content = prev.get("content", "") - cur_content = m.get("content", "") - # Work on a copy of ``prev`` so the caller's input dicts are - # never mutated. ``_sanitize_api_messages`` upstream already - # hands us per-call copies, but staying pure here means we - # can be called safely from anywhere (tests, other loops). - prev_copy = dict(prev) - # Only string-content merge is meaningful for role-alternation - # purposes. If either side is a list (multimodal), append as a - # separate block rather than collapsing. - if isinstance(prev_content, str) and isinstance(cur_content, str): - sep = "\n\n" if prev_content and cur_content else "" - prev_copy["content"] = prev_content + sep + cur_content - elif isinstance(prev_content, list) and isinstance(cur_content, list): - prev_copy["content"] = list(prev_content) + list(cur_content) - elif isinstance(prev_content, list) and isinstance(cur_content, str): - if cur_content: - prev_copy["content"] = list(prev_content) + [ - {"type": "text", "text": cur_content} - ] - else: - prev_copy["content"] = list(prev_content) - elif isinstance(prev_content, str) and isinstance(cur_content, list): - new_blocks: List[Dict[str, Any]] = [] - if prev_content: - new_blocks.append({"type": "text", "text": prev_content}) - new_blocks.extend(cur_content) - prev_copy["content"] = new_blocks - else: - # Unknown content shape — fall back to appending separately - # (violates alternation, but safer than raising in a hot path). - merged.append(m) - continue - merged[-1] = prev_copy - merges += 1 - else: - merged.append(m) - - logger.debug( - "Pre-call sanitizer: dropped %d thinking-only assistant turn(s), " - "merged %d adjacent user message(s)", - dropped, - merges, - ) - return merged + """Forwarder — see ``agent.agent_runtime_helpers.drop_thinking_only_and_merge_users``.""" + from agent.agent_runtime_helpers import drop_thinking_only_and_merge_users + return drop_thinking_only_and_merge_users(messages) @staticmethod def _cap_delegate_task_calls(tool_calls: list) -> list: @@ -4890,97 +4227,9 @@ class AIAgent: return None def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: bool) -> Any: - from agent.auxiliary_client import _validate_base_url, _validate_proxy_env_urls - # Treat client_kwargs as read-only. Callers pass self._client_kwargs (or shallow - # copies of it) in; any in-place mutation leaks back into the stored dict and is - # reused on subsequent requests. #10933 hit this by injecting an httpx.Client - # transport that was torn down after the first request, so the next request - # wrapped a closed transport and raised "Cannot send a request, as the client - # has been closed" on every retry. The revert resolved that specific path; this - # copy locks the contract so future transport/keepalive work can't reintroduce - # the same class of bug. - client_kwargs = dict(client_kwargs) - _validate_proxy_env_urls() - _validate_base_url(client_kwargs.get("base_url")) - if self.provider == "copilot-acp" or str(client_kwargs.get("base_url", "")).startswith("acp://copilot"): - from agent.copilot_acp_client import CopilotACPClient - - client = CopilotACPClient(**client_kwargs) - logger.info( - "Copilot ACP client created (%s, shared=%s) %s", - reason, - shared, - self._client_log_context(), - ) - return client - if self.provider == "google-gemini-cli" or str(client_kwargs.get("base_url", "")).startswith("cloudcode-pa://"): - from agent.gemini_cloudcode_adapter import GeminiCloudCodeClient - - # Strip OpenAI-specific kwargs the Gemini client doesn't accept - safe_kwargs = { - k: v for k, v in client_kwargs.items() - if k in {"api_key", "base_url", "default_headers", "project_id", "timeout"} - } - client = GeminiCloudCodeClient(**safe_kwargs) - logger.info( - "Gemini Cloud Code Assist client created (%s, shared=%s) %s", - reason, - shared, - self._client_log_context(), - ) - return client - if self.provider == "gemini": - from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url - - base_url = str(client_kwargs.get("base_url", "") or "") - if is_native_gemini_base_url(base_url): - safe_kwargs = { - k: v for k, v in client_kwargs.items() - if k in {"api_key", "base_url", "default_headers", "timeout", "http_client"} - } - if "http_client" not in safe_kwargs: - keepalive_http = self._build_keepalive_http_client(base_url) - if keepalive_http is not None: - safe_kwargs["http_client"] = keepalive_http - client = GeminiNativeClient(**safe_kwargs) - logger.info( - "Gemini native client created (%s, shared=%s) %s", - reason, - shared, - self._client_log_context(), - ) - return client - # Inject TCP keepalives so the kernel detects dead provider connections - # instead of letting them sit silently in CLOSE-WAIT (#10324). Without - # this, a peer that drops mid-stream leaves the socket in a state where - # epoll_wait never fires, ``httpx`` read timeout may not trigger, and - # the agent hangs until manually killed. Probes after 30s idle, retry - # every 10s, give up after 3 → dead peer detected within ~60s. - # - # Safety against #10933: the ``client_kwargs = dict(client_kwargs)`` - # above means this injection only lands in the local per-call copy, - # never back into ``self._client_kwargs``. Each ``_create_openai_client`` - # invocation therefore gets its OWN fresh ``httpx.Client`` whose - # lifetime is tied to the OpenAI client it is passed to. When the - # OpenAI client is closed (rebuild, teardown, credential rotation), - # the paired ``httpx.Client`` closes with it, and the next call - # constructs a fresh one — no stale closed transport can be reused. - # Tests in ``tests/run_agent/test_create_openai_client_reuse.py`` and - # ``tests/run_agent/test_sequential_chats_live.py`` pin this invariant. - if "http_client" not in client_kwargs: - keepalive_http = self._build_keepalive_http_client(client_kwargs.get("base_url", "")) - if keepalive_http is not None: - client_kwargs["http_client"] = keepalive_http - # Uses the module-level `OpenAI` name, resolved lazily on first - # access via __getattr__ below. Tests patch via `run_agent.OpenAI`. - client = OpenAI(**client_kwargs) - logger.info( - "OpenAI client created (%s, shared=%s) %s", - reason, - shared, - self._client_log_context(), - ) - return client + """Forwarder — see ``agent.agent_runtime_helpers.create_openai_client``.""" + from agent.agent_runtime_helpers import create_openai_client + return create_openai_client(self, client_kwargs, reason=reason, shared=shared) @staticmethod def _force_close_tcp_sockets(client: Any) -> int: @@ -5230,207 +4479,14 @@ class AIAgent: self._close_openai_client(client, reason=reason, shared=False) def _run_codex_stream(self, api_kwargs: dict, client: Any = None, on_first_delta: callable = None): - """Execute one streaming Responses API request and return the final response.""" - import httpx as _httpx - - active_client = client or self._ensure_primary_openai_client(reason="codex_stream_direct") - max_stream_retries = 1 - has_tool_calls = False - first_delta_fired = False - # Accumulate streamed text so we can recover if get_final_response() - # returns empty output (e.g. chatgpt.com backend-api sends - # response.incomplete instead of response.completed). - self._codex_streamed_text_parts: list = [] - for attempt in range(max_stream_retries + 1): - if self._interrupt_requested: - raise InterruptedError("Agent interrupted before Codex stream retry") - collected_output_items: list = [] - try: - with active_client.responses.stream(**api_kwargs) as stream: - for event in stream: - self._touch_activity("receiving stream response") - if self._interrupt_requested: - break - event_type = getattr(event, "type", "") - # Fire callbacks on text content deltas (suppress during tool calls) - if "output_text.delta" in event_type or event_type == "response.output_text.delta": - delta_text = getattr(event, "delta", "") - if delta_text: - self._codex_streamed_text_parts.append(delta_text) - if delta_text and not has_tool_calls: - if not first_delta_fired: - first_delta_fired = True - if on_first_delta: - try: - on_first_delta() - except Exception: - pass - self._fire_stream_delta(delta_text) - # Track tool calls to suppress text streaming - elif "function_call" in event_type: - has_tool_calls = True - # Fire reasoning callbacks - elif "reasoning" in event_type and "delta" in event_type: - reasoning_text = getattr(event, "delta", "") - if reasoning_text: - self._fire_reasoning_delta(reasoning_text) - # Collect completed output items — some backends - # (chatgpt.com/backend-api/codex) stream valid items - # via response.output_item.done but the SDK's - # get_final_response() returns an empty output list. - elif event_type == "response.output_item.done": - done_item = getattr(event, "item", None) - if done_item is not None: - collected_output_items.append(done_item) - # Log non-completed terminal events for diagnostics - elif event_type in {"response.incomplete", "response.failed"}: - resp_obj = getattr(event, "response", None) - status = getattr(resp_obj, "status", None) if resp_obj else None - incomplete_details = getattr(resp_obj, "incomplete_details", None) if resp_obj else None - logger.warning( - "Codex Responses stream received terminal event %s " - "(status=%s, incomplete_details=%s, streamed_chars=%d). %s", - event_type, status, incomplete_details, - sum(len(p) for p in self._codex_streamed_text_parts), - self._client_log_context(), - ) - final_response = stream.get_final_response() - # PATCH: ChatGPT Codex backend streams valid output items - # but get_final_response() can return an empty output list. - # Backfill from collected items or synthesize from deltas. - _out = getattr(final_response, "output", None) - if isinstance(_out, list) and not _out: - if collected_output_items: - final_response.output = list(collected_output_items) - logger.debug( - "Codex stream: backfilled %d output items from stream events", - len(collected_output_items), - ) - elif self._codex_streamed_text_parts and not has_tool_calls: - assembled = "".join(self._codex_streamed_text_parts) - final_response.output = [SimpleNamespace( - type="message", - role="assistant", - status="completed", - content=[SimpleNamespace(type="output_text", text=assembled)], - )] - logger.debug( - "Codex stream: synthesized output from %d text deltas (%d chars)", - len(self._codex_streamed_text_parts), len(assembled), - ) - return final_response - except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: - if attempt < max_stream_retries: - logger.debug( - "Codex Responses stream transport failed (attempt %s/%s); retrying. %s error=%s", - attempt + 1, - max_stream_retries + 1, - self._client_log_context(), - exc, - ) - continue - logger.debug( - "Codex Responses stream transport failed; falling back to create(stream=True). %s error=%s", - self._client_log_context(), - exc, - ) - return self._run_codex_create_stream_fallback(api_kwargs, client=active_client) - except RuntimeError as exc: - err_text = str(exc) - missing_completed = "response.completed" in err_text - if missing_completed and attempt < max_stream_retries: - logger.debug( - "Responses stream closed before completion (attempt %s/%s); retrying. %s", - attempt + 1, - max_stream_retries + 1, - self._client_log_context(), - ) - continue - if missing_completed: - logger.debug( - "Responses stream did not emit response.completed; falling back to create(stream=True). %s", - self._client_log_context(), - ) - return self._run_codex_create_stream_fallback(api_kwargs, client=active_client) - raise + """Forwarder — see ``agent.codex_runtime.run_codex_stream``.""" + from agent.codex_runtime import run_codex_stream + return run_codex_stream(self, api_kwargs, client, on_first_delta) def _run_codex_create_stream_fallback(self, api_kwargs: dict, client: Any = None): - """Fallback path for stream completion edge cases on Codex-style Responses backends.""" - active_client = client or self._ensure_primary_openai_client(reason="codex_create_stream_fallback") - fallback_kwargs = dict(api_kwargs) - fallback_kwargs["stream"] = True - fallback_kwargs = self._get_transport().preflight_kwargs(fallback_kwargs, allow_stream=True) - stream_or_response = active_client.responses.create(**fallback_kwargs) - - # Compatibility shim for mocks or providers that still return a concrete response. - if hasattr(stream_or_response, "output"): - return stream_or_response - if not hasattr(stream_or_response, "__iter__"): - return stream_or_response - - terminal_response = None - collected_output_items: list = [] - collected_text_deltas: list = [] - try: - for event in stream_or_response: - self._touch_activity("receiving stream response") - event_type = getattr(event, "type", None) - if not event_type and isinstance(event, dict): - event_type = event.get("type") - - # Collect output items and text deltas for backfill - if event_type == "response.output_item.done": - done_item = getattr(event, "item", None) - if done_item is None and isinstance(event, dict): - done_item = event.get("item") - if done_item is not None: - collected_output_items.append(done_item) - elif event_type in {"response.output_text.delta",}: - delta = getattr(event, "delta", "") - if not delta and isinstance(event, dict): - delta = event.get("delta", "") - if delta: - collected_text_deltas.append(delta) - - if event_type not in {"response.completed", "response.incomplete", "response.failed"}: - continue - - terminal_response = getattr(event, "response", None) - if terminal_response is None and isinstance(event, dict): - terminal_response = event.get("response") - if terminal_response is not None: - # Backfill empty output from collected stream events - _out = getattr(terminal_response, "output", None) - if isinstance(_out, list) and not _out: - if collected_output_items: - terminal_response.output = list(collected_output_items) - logger.debug( - "Codex fallback stream: backfilled %d output items", - len(collected_output_items), - ) - elif collected_text_deltas: - assembled = "".join(collected_text_deltas) - terminal_response.output = [SimpleNamespace( - type="message", role="assistant", - status="completed", - content=[SimpleNamespace(type="output_text", text=assembled)], - )] - logger.debug( - "Codex fallback stream: synthesized from %d deltas (%d chars)", - len(collected_text_deltas), len(assembled), - ) - return terminal_response - finally: - close_fn = getattr(stream_or_response, "close", None) - if callable(close_fn): - try: - close_fn() - except Exception: - pass - - if terminal_response is not None: - return terminal_response - raise RuntimeError("Responses create(stream=True) fallback did not emit a terminal response.") + """Forwarder — see ``agent.codex_runtime.run_codex_create_stream_fallback``.""" + from agent.codex_runtime import run_codex_create_stream_fallback + return run_codex_create_stream_fallback(self, api_kwargs, client) def _try_refresh_codex_client_credentials(self, *, force: bool = True) -> bool: if self.api_mode != "codex_responses" or self.provider != "openai-codex": @@ -5657,81 +4713,9 @@ class AIAgent: classified_reason: Optional[FailoverReason] = None, error_context: Optional[Dict[str, Any]] = None, ) -> tuple[bool, bool]: - """Attempt credential recovery via pool rotation. - - Returns (recovered, has_retried_429). - On rate limits: first occurrence retries same credential (sets flag True). - second consecutive failure rotates to next credential. - On billing exhaustion: immediately rotates. - On auth failures: attempts token refresh before rotating. - - `classified_reason` lets the recovery path honor the structured error - classifier instead of relying only on raw HTTP codes. This matters for - providers that surface billing/rate-limit/auth conditions under a - different status code, such as Anthropic returning HTTP 400 for - "out of extra usage". - """ - pool = self._credential_pool - if pool is None: - return False, has_retried_429 - - effective_reason = classified_reason - if effective_reason is None: - if status_code == 402: - effective_reason = FailoverReason.billing - elif status_code == 429: - effective_reason = FailoverReason.rate_limit - elif status_code in {401, 403}: - effective_reason = FailoverReason.auth - - if effective_reason == FailoverReason.billing: - rotate_status = status_code if status_code is not None else 402 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) - if next_entry is not None: - logger.info( - "Credential %s (billing) — rotated to pool entry %s", - rotate_status, - getattr(next_entry, "id", "?"), - ) - self._swap_credential(next_entry) - return True, False - return False, has_retried_429 - - if effective_reason == FailoverReason.rate_limit: - if not has_retried_429: - return False, True - rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) - if next_entry is not None: - logger.info( - "Credential %s (rate limit) — rotated to pool entry %s", - rotate_status, - getattr(next_entry, "id", "?"), - ) - self._swap_credential(next_entry) - return True, False - return False, True - - if effective_reason == FailoverReason.auth: - refreshed = pool.try_refresh_current() - if refreshed is not None: - logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") - self._swap_credential(refreshed) - return True, has_retried_429 - # Refresh failed — rotate to next credential instead of giving up. - # The failed entry is already marked exhausted by try_refresh_current(). - rotate_status = status_code if status_code is not None else 401 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) - if next_entry is not None: - logger.info( - "Credential %s (auth refresh failed) — rotated to pool entry %s", - rotate_status, - getattr(next_entry, "id", "?"), - ) - self._swap_credential(next_entry) - return True, False - - return False, has_retried_429 + """Forwarder — see ``agent.agent_runtime_helpers.recover_with_credential_pool``.""" + from agent.agent_runtime_helpers import recover_with_credential_pool + return recover_with_credential_pool(self, status_code=status_code, has_retried_429=has_retried_429, classified_reason=classified_reason, error_context=error_context) def _credential_pool_may_recover_rate_limit(self) -> bool: """Whether a rate-limit retry should wait for same-provider credentials.""" @@ -5965,171 +4949,16 @@ class AIAgent: # ── Per-turn primary restoration ───────────────────────────────────── def _restore_primary_runtime(self) -> bool: - """Restore the primary runtime at the start of a new turn. - - In long-lived CLI sessions a single AIAgent instance spans multiple - turns. Without restoration, one transient failure pins the session - to the fallback provider for every subsequent turn. Calling this at - the top of ``run_conversation()`` makes fallback turn-scoped. - - The gateway caches agents across messages (``_agent_cache`` in - ``gateway/run.py``), so this restoration IS needed there too. - """ - if not self._fallback_activated: - return False - - if getattr(self, "_rate_limited_until", 0) > time.monotonic(): - return False # primary still in rate-limit cooldown, stay on fallback - - rt = self._primary_runtime - try: - # ── Core runtime state ── - self.model = rt["model"] - self.provider = rt["provider"] - self.base_url = rt["base_url"] # setter updates _base_url_lower - self.api_mode = rt["api_mode"] - if hasattr(self, "_transport_cache"): - self._transport_cache.clear() - self.api_key = rt["api_key"] - self._client_kwargs = dict(rt["client_kwargs"]) - self._use_prompt_caching = rt["use_prompt_caching"] - # Default to native layout when the restored snapshot predates the - # native-vs-proxy split (older sessions saved before this PR). - self._use_native_cache_layout = rt.get( - "use_native_cache_layout", - self.api_mode == "anthropic_messages" and self.provider == "anthropic", - ) - - # ── Rebuild client for the primary provider ── - if self.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client - self._anthropic_api_key = rt["anthropic_api_key"] - self._anthropic_base_url = rt["anthropic_base_url"] - self._anthropic_client = build_anthropic_client( - rt["anthropic_api_key"], rt["anthropic_base_url"], - timeout=get_provider_request_timeout(self.provider, self.model), - ) - self._is_anthropic_oauth = rt["is_anthropic_oauth"] - self.client = None - else: - self.client = self._create_openai_client( - dict(rt["client_kwargs"]), - reason="restore_primary", - shared=True, - ) - - # ── Restore context engine state ── - cc = self.context_compressor - cc.update_model( - model=rt["compressor_model"], - context_length=rt["compressor_context_length"], - base_url=rt["compressor_base_url"], - api_key=rt["compressor_api_key"], - provider=rt["compressor_provider"], - ) - - # ── Reset fallback chain for the new turn ── - self._fallback_activated = False - self._fallback_index = 0 - - logging.info( - "Primary runtime restored for new turn: %s (%s)", - self.model, self.provider, - ) - return True - except Exception as e: - logging.warning("Failed to restore primary runtime: %s", e) - return False - - # Which error types indicate a transient transport failure worth - # one more attempt with a rebuilt client / connection pool. - _TRANSIENT_TRANSPORT_ERRORS = frozenset({ - "ReadTimeout", "ConnectTimeout", "PoolTimeout", - "ConnectError", "RemoteProtocolError", - "APIConnectionError", "APITimeoutError", - }) + """Forwarder — see ``agent.agent_runtime_helpers.restore_primary_runtime``.""" + from agent.agent_runtime_helpers import restore_primary_runtime + return restore_primary_runtime(self) def _try_recover_primary_transport( self, api_error: Exception, *, retry_count: int, max_retries: int, ) -> bool: - """Attempt one extra primary-provider recovery cycle for transient transport failures. - - After ``max_retries`` exhaust, rebuild the primary client (clearing - stale connection pools) and give it one more attempt before falling - back. This is most useful for direct endpoints (custom, Z.AI, - Anthropic, OpenAI, local models) where a TCP-level hiccup does not - mean the provider is down. - - Skipped for proxy/aggregator providers (OpenRouter, Nous) which - already manage connection pools and retries server-side — if our - retries through them are exhausted, one more rebuilt client won't help. - """ - if self._fallback_activated: - return False - - # Only for transient transport errors - error_type = type(api_error).__name__ - if error_type not in self._TRANSIENT_TRANSPORT_ERRORS: - return False - - # Skip for aggregator providers — they manage their own retry infra - if self._is_openrouter_url(): - return False - provider_lower = (self.provider or "").strip().lower() - if provider_lower in {"nous", "nous-research"}: - return False - - try: - # Close existing client to release stale connections - if getattr(self, "client", None) is not None: - try: - self._close_openai_client( - self.client, reason="primary_recovery", shared=True, - ) - except Exception: - pass - - # Rebuild from primary snapshot - rt = self._primary_runtime - self._client_kwargs = dict(rt["client_kwargs"]) - self.model = rt["model"] - self.provider = rt["provider"] - self.base_url = rt["base_url"] - self.api_mode = rt["api_mode"] - if hasattr(self, "_transport_cache"): - self._transport_cache.clear() - self.api_key = rt["api_key"] - - if self.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client - self._anthropic_api_key = rt["anthropic_api_key"] - self._anthropic_base_url = rt["anthropic_base_url"] - self._anthropic_client = build_anthropic_client( - rt["anthropic_api_key"], rt["anthropic_base_url"], - timeout=get_provider_request_timeout(self.provider, self.model), - ) - self._is_anthropic_oauth = rt["is_anthropic_oauth"] - self.client = None - else: - self.client = self._create_openai_client( - dict(rt["client_kwargs"]), - reason="primary_recovery", - shared=True, - ) - - wait_time = min(3 + retry_count, 8) - self._vprint( - f"{self.log_prefix}🔁 Transient {error_type} on {self.provider} — " - f"rebuilt client, waiting {wait_time}s before one last primary attempt.", - force=True, - ) - time.sleep(wait_time) - return True - except Exception as e: - logging.warning("Primary transport recovery failed: %s", e) - return False - - # ── End provider fallback ────────────────────────────────────────────── + """Forwarder — see ``agent.agent_runtime_helpers.try_recover_primary_transport``.""" + from agent.agent_runtime_helpers import try_recover_primary_transport + return try_recover_primary_transport(self, api_error, retry_count=retry_count, max_retries=max_retries) @staticmethod def _content_has_image_parts(content: Any) -> bool: @@ -6802,108 +5631,9 @@ class AIAgent: logger=None, session_id: str = None, ) -> int: - """Repair corrupted assistant tool-call argument JSON in-place.""" - log = logger or logging.getLogger(__name__) - if not isinstance(messages, list): - return 0 - - repaired = 0 - marker = AIAgent._TOOL_CALL_ARGUMENTS_CORRUPTION_MARKER - - def _prepend_marker(tool_msg: dict) -> None: - existing = tool_msg.get("content") - if isinstance(existing, str): - if not existing: - tool_msg["content"] = marker - elif not existing.startswith(marker): - tool_msg["content"] = f"{marker}\n{existing}" - return - if existing is None: - tool_msg["content"] = marker - return - try: - existing_text = json.dumps(existing) - except TypeError: - existing_text = str(existing) - tool_msg["content"] = f"{marker}\n{existing_text}" - - message_index = 0 - while message_index < len(messages): - msg = messages[message_index] - if not isinstance(msg, dict) or msg.get("role") != "assistant": - message_index += 1 - continue - - tool_calls = msg.get("tool_calls") - if not isinstance(tool_calls, list) or not tool_calls: - message_index += 1 - continue - - insert_at = message_index + 1 - for tool_call in tool_calls: - if not isinstance(tool_call, dict): - continue - function = tool_call.get("function") - if not isinstance(function, dict): - continue - - arguments = function.get("arguments") - if arguments is None or arguments == "": - function["arguments"] = "{}" - continue - if isinstance(arguments, str) and not arguments.strip(): - function["arguments"] = "{}" - continue - if not isinstance(arguments, str): - continue - - try: - json.loads(arguments) - except json.JSONDecodeError: - tool_call_id = tool_call.get("id") - function_name = function.get("name", "?") - preview = arguments[:80] - log.warning( - "Corrupted tool_call arguments repaired before request " - "(session=%s, message_index=%s, tool_call_id=%s, function=%s, preview=%r)", - session_id or "-", - message_index, - tool_call_id or "-", - function_name, - preview, - ) - function["arguments"] = "{}" - - existing_tool_msg = None - scan_index = message_index + 1 - while scan_index < len(messages): - candidate = messages[scan_index] - if not isinstance(candidate, dict) or candidate.get("role") != "tool": - break - if candidate.get("tool_call_id") == tool_call_id: - existing_tool_msg = candidate - break - scan_index += 1 - - if existing_tool_msg is None: - messages.insert( - insert_at, - { - "role": "tool", - "name": function_name if function_name != "?" else "", - "tool_call_id": tool_call_id, - "content": marker, - }, - ) - insert_at += 1 - else: - _prepend_marker(existing_tool_msg) - - repaired += 1 - - message_index += 1 - - return repaired + """Forwarder — see ``agent.agent_runtime_helpers.sanitize_tool_call_arguments``.""" + from agent.agent_runtime_helpers import sanitize_tool_call_arguments + return sanitize_tool_call_arguments(messages, logger=logger, session_id=session_id) def _should_sanitize_tool_calls(self) -> bool: """Determine if tool_calls need sanitization for strict APIs. @@ -11033,144 +9763,9 @@ class AIAgent: effective_task_id: str, should_review_memory: bool = False, ) -> Dict[str, Any]: - """Codex app-server runtime path. Hands the entire turn to a `codex - app-server` subprocess and projects its events back into Hermes' - messages list so memory/skill review keep working. - - Called from run_conversation() when self.api_mode == "codex_app_server". - Returns the same dict shape as the chat_completions path. - """ - from agent.transports.codex_app_server_session import CodexAppServerSession - - # Lazy session: one CodexAppServerSession per AIAgent instance. - # Spawned on first turn, reused across turns, closed at AIAgent - # shutdown (see _cleanup hook). - if not hasattr(self, "_codex_session") or self._codex_session is None: - cwd = getattr(self, "session_cwd", None) or os.getcwd() - # Approval callback: defer to Hermes' standard prompt flow if a - # CLI thread has installed one. Gateway / cron contexts get the - # codex-side fail-closed default. - try: - from tools.terminal_tool import _get_approval_callback - approval_callback = _get_approval_callback() - except Exception: - approval_callback = None - self._codex_session = CodexAppServerSession( - cwd=cwd, - approval_callback=approval_callback, - ) - - # NOTE: the user message is ALREADY appended to messages by the - # standard run_conversation() flow (line ~11823) before the early - # return reaches us. Do NOT append again — that would duplicate. - - try: - turn = self._codex_session.run_turn(user_input=user_message) - except Exception as exc: - logger.exception("codex app-server turn failed") - # Crash → unconditionally drop the session so the next turn - # respawns from scratch instead of reusing a dead client. - try: - self._codex_session.close() - except Exception: - pass - self._codex_session = None - return { - "final_response": ( - f"Codex app-server turn failed: {exc}. " - f"Fall back to default runtime with `/codex-runtime auto`." - ), - "messages": messages, - "api_calls": 0, - "completed": False, - "partial": True, - "error": str(exc), - } - - # If the turn signalled the underlying client is wedged (deadline - # blown, post-tool watchdog tripped, OAuth refresh died, subprocess - # exited), retire the session so the next turn respawns codex - # rather than riding the broken process. Mirrors openclaw beta.8's - # "retire timed-out app-server clients" fix. - if getattr(turn, "should_retire", False): - logger.warning( - "codex app-server session retired (turn error: %s)", - turn.error, - ) - try: - self._codex_session.close() - except Exception: - pass - self._codex_session = None - - # Splice projected messages into the conversation. The projector emits - # standard {role, content, tool_calls, tool_call_id} entries, which - # is exactly what curator.py / sessions DB expect. - if turn.projected_messages: - messages.extend(turn.projected_messages) - - # Counter ticks for the self-improvement loop. - # _turns_since_memory and _user_turn_count are ALREADY incremented - # in the run_conversation() pre-loop block (lines ~11793-11817) so we - # do NOT touch them here — that would double-count. - # Only _iters_since_skill needs explicit increment, since the - # chat_completions loop bumps it per tool iteration (line ~12110) - # and that loop is bypassed on this path. - self._iters_since_skill = ( - getattr(self, "_iters_since_skill", 0) + turn.tool_iterations - ) - - # Now check the skill nudge AFTER iters were incremented — same - # pattern the chat_completions path uses (line ~15432). - should_review_skills = False - if ( - self._skill_nudge_interval > 0 - and self._iters_since_skill >= self._skill_nudge_interval - and "skill_manage" in self.valid_tool_names - ): - should_review_skills = True - self._iters_since_skill = 0 - - # External memory provider sync (mirrors line ~15439). Skipped on - # interrupt/error to avoid feeding partial transcripts to memory. - if not turn.interrupted and turn.error is None: - try: - self._sync_external_memory_for_turn( - original_user_message=original_user_message, - final_response=turn.final_text, - interrupted=False, - ) - except Exception: - logger.debug("external memory sync raised", exc_info=True) - - # Background review fork — same cadence + signature as the default - # path (line ~15449). Only fires when a trigger actually tripped AND - # we have a real final response. - if ( - turn.final_text - and not turn.interrupted - and (should_review_memory or should_review_skills) - ): - try: - self._spawn_background_review( - messages_snapshot=list(messages), - review_memory=should_review_memory, - review_skills=should_review_skills, - ) - except Exception: - logger.debug("background review spawn raised", exc_info=True) - - return { - "final_response": turn.final_text, - "messages": messages, - "api_calls": 1, # one app-server "turn" maps to one logical API call - "completed": not turn.interrupted and turn.error is None, - "partial": turn.interrupted or turn.error is not None, - "error": turn.error, - "codex_thread_id": turn.thread_id, - "codex_turn_id": turn.turn_id, - } - + """Forwarder — see ``agent.codex_runtime.run_codex_app_server_turn``.""" + from agent.codex_runtime import run_codex_app_server_turn + return run_codex_app_server_turn(self, user_message=user_message, original_user_message=original_user_message, messages=messages, effective_task_id=effective_task_id, should_review_memory=should_review_memory) def main( query: str = None, From d35ee7bcdd652715864d0d0c262790293a2555c6 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 19:11:58 -0700 Subject: [PATCH 012/142] refactor(run_agent): move review prompts to agent/background_review.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three big review-prompt strings (_MEMORY_REVIEW_PROMPT, _SKILL_REVIEW_PROMPT, _COMBINED_REVIEW_PROMPT — 183 lines combined) move out of the AIAgent class body and into agent/background_review.py where they're consumed. AIAgent re-exposes them as class attributes via 'from ... import' inside the class body — Python binds those names into the class namespace so existing AIAgent._MEMORY_REVIEW_PROMPT references keep working. spawn_background_review_thread also falls back to the module-level constants if an agent doesn't have the attribute (preserves the test pattern of mocking these on the agent). tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 9986 -> 9800 lines (-186). --- agent/background_review.py | 202 ++++++++++++++++++++++++++++++++++++- run_agent.py | 188 +--------------------------------- 2 files changed, 203 insertions(+), 187 deletions(-) diff --git a/agent/background_review.py b/agent/background_review.py index 351ab1d43dc..0319bbfa046 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -27,6 +27,195 @@ from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) +# Review-prompt strings — used by ``spawn_background_review_thread`` to build +# the user-message that the forked review agent receives. AIAgent exposes +# them as class attributes (``_MEMORY_REVIEW_PROMPT`` etc.) for back-compat; +# the actual text lives here so future edits are one-place. +_MEMORY_REVIEW_PROMPT = ( + "Review the conversation above and consider saving to memory if appropriate.\n\n" + "Focus on:\n" + "1. Has the user revealed things about themselves — their persona, desires, " + "preferences, or personal details worth remembering?\n" + "2. Has the user expressed expectations about how you should behave, their work " + "style, or ways they want you to operate?\n\n" + "If something stands out, save it using the memory tool. " + "If nothing is worth saving, just say 'Nothing to save.' and stop." +) + +_SKILL_REVIEW_PROMPT = ( + "Review the conversation above and update the skill library. Be " + "ACTIVE — most sessions produce at least one skill update, even if " + "small. A pass that does nothing is a missed learning opportunity, " + "not a neutral outcome.\n\n" + "Target shape of the library: CLASS-LEVEL skills, each with a rich " + "SKILL.md and a `references/` directory for session-specific detail. " + "Not a long flat list of narrow one-session-one-skill entries. This " + "shapes HOW you update, not WHETHER you update.\n\n" + "Signals to look for (any one of these warrants action):\n" + " • User corrected your style, tone, format, legibility, or " + "verbosity. Frustration signals like 'stop doing X', 'this is too " + "verbose', 'don't format like this', 'why are you explaining', " + "'just give me the answer', 'you always do Y and I hate it', or an " + "explicit 'remember this' are FIRST-CLASS skill signals, not just " + "memory signals. Update the relevant skill(s) to embed the " + "preference so the next session starts already knowing.\n" + " • User corrected your workflow, approach, or sequence of steps. " + "Encode the correction as a pitfall or explicit step in the skill " + "that governs that class of task.\n" + " • Non-trivial technique, fix, workaround, debugging path, or " + "tool-usage pattern emerged that a future session would benefit " + "from. Capture it.\n" + " • A skill that got loaded or consulted this session turned out " + "to be wrong, missing a step, or outdated. Patch it NOW.\n\n" + "Preference order — prefer the earliest action that fits, but do " + "pick one when a signal above fired:\n" + " 1. UPDATE A CURRENTLY-LOADED SKILL. Look back through the " + "conversation for skills the user loaded via /skill-name or you " + "read via skill_view. If any of them covers the territory of the " + "new learning, PATCH that one first. It is the skill that was in " + "play, so it's the right one to extend.\n" + " 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). " + "If no loaded skill fits but an existing class-level skill does, " + "patch it. Add a subsection, a pitfall, or broaden a trigger.\n" + " 3. ADD A SUPPORT FILE under an existing umbrella. Skills can be " + "packaged with three kinds of support files — use the right " + "directory per kind:\n" + " • `references/.md` — session-specific detail (error " + "transcripts, reproduction recipes, provider quirks) AND " + "condensed knowledge banks: quoted research, API docs, external " + "authoritative excerpts, or domain notes you found while working " + "on the problem. Write it concise and for the value of the task, " + "not as a full mirror of upstream docs.\n" + " • `templates/.` — starter files meant to be " + "copied and modified (boilerplate configs, scaffolding, a " + "known-good example the agent can `reproduce with modifications`).\n" + " • `scripts/.` — statically re-runnable actions " + "the skill can invoke directly (verification scripts, fixture " + "generators, deterministic probes, anything the agent should run " + "rather than hand-type each time).\n" + " Add support files via skill_manage action=write_file with " + "file_path starting 'references/', 'templates/', or 'scripts/'. " + "The umbrella's SKILL.md should gain a one-line pointer to any " + "new support file so future agents know it exists.\n" + " 4. CREATE A NEW CLASS-LEVEL UMBRELLA SKILL when no existing " + "skill covers the class. The name MUST be at the class level. " + "The name MUST NOT be a specific PR number, error string, feature " + "codename, library-alone name, or 'fix-X / debug-Y / audit-Z-today' " + "session artifact. If the proposed name only makes sense for " + "today's task, it's wrong — fall back to (1), (2), or (3).\n\n" + "User-preference embedding (important): when the user expressed a " + "style/format/workflow preference, the update belongs in the " + "SKILL.md body, not just in memory. Memory captures 'who the user " + "is and what the current situation and state of your operations " + "are'; skills capture 'how to do this class of task for this " + "user'. When they complain about how you handled a task, the " + "skill that governs that task needs to carry the lesson.\n\n" + "If you notice two existing skills that overlap, note it in your " + "reply — the background curator handles consolidation at scale.\n\n" + "Do NOT capture (these become persistent self-imposed constraints " + "that bite you later when the environment changes):\n" + " • Environment-dependent failures: missing binaries, fresh-install " + "errors, post-migration path mismatches, 'command not found', " + "unconfigured credentials, uninstalled packages. The user can fix " + "these — they are not durable rules.\n" + " • Negative claims about tools or features ('browser tools do not " + "work', 'X tool is broken', 'cannot use Y from execute_code'). These " + "harden into refusals the agent cites against itself for months " + "after the actual problem was fixed.\n" + " • Session-specific transient errors that resolved before the " + "conversation ended. If retrying worked, the lesson is the retry " + "pattern, not the original failure.\n" + " • One-off task narratives. A user asking 'summarize today's " + "market' or 'analyze this PR' is not a class of work that warrants " + "a skill.\n\n" + "If a tool failed because of setup state, capture the FIX (install " + "command, config step, env var to set) under an existing setup or " + "troubleshooting skill — never 'this tool does not work' as a " + "standalone constraint.\n\n" + "'Nothing to save.' is a real option but should NOT be the " + "default. If the session ran smoothly with no corrections and " + "produced no new technique, just say 'Nothing to save.' and stop. " + "Otherwise, act." +) + +_COMBINED_REVIEW_PROMPT = ( + "Review the conversation above and update two things:\n\n" + "**Memory**: who the user is. Did the user reveal persona, " + "desires, preferences, personal details, or expectations about " + "how you should behave? Save facts about the user and durable " + "preferences with the memory tool.\n\n" + "**Skills**: how to do this class of task. Be ACTIVE — most " + "sessions produce at least one skill update. A pass that does " + "nothing is a missed learning opportunity, not a neutral outcome.\n\n" + "Target shape of the skill library: CLASS-LEVEL skills with a rich " + "SKILL.md and a `references/` directory for session-specific detail. " + "Not a long flat list of narrow one-session-one-skill entries.\n\n" + "Signals that warrant a skill update (any one is enough):\n" + " • User corrected your style, tone, format, legibility, " + "verbosity, or approach. Frustration is a FIRST-CLASS skill " + "signal, not just a memory signal. 'stop doing X', 'don't format " + "like this', 'I hate when you Y' — embed the lesson in the skill " + "that governs that task so the next session starts fixed.\n" + " • Non-trivial technique, fix, workaround, or debugging path " + "emerged.\n" + " • A skill that was loaded or consulted turned out wrong, " + "missing, or outdated — patch it now.\n\n" + "Preference order for skills — pick the earliest that fits:\n" + " 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were " + "loaded via /skill-name or skill_view in the conversation. If one " + "of them covers the learning, PATCH it first. It was in play; " + "it's the right place.\n" + " 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to " + "find the right one). Patch it.\n" + " 3. ADD A SUPPORT FILE under an existing umbrella via " + "skill_manage action=write_file. Three kinds: " + "`references/.md` for session-specific detail OR condensed " + "knowledge banks (quoted research, API docs excerpts, domain " + "notes) written concise and task-focused; `templates/.` " + "for starter files meant to be copied and modified; " + "`scripts/.` for statically re-runnable actions " + "(verification, fixture generators, probes). Add a one-line " + "pointer in SKILL.md so future agents find them.\n" + " 4. CREATE A NEW CLASS-LEVEL UMBRELLA when nothing exists. " + "Name at the class level — NOT a PR number, error string, " + "codename, library-alone name, or 'fix-X / debug-Y' session " + "artifact. If the name only fits today's task, fall back to (1), " + "(2), or (3).\n\n" + "User-preference embedding: when the user complains about how " + "you handled a task, update the skill that governs that task — " + "memory alone isn't enough. Memory says 'who the user is and " + "what the current situation and state of your operations are'; " + "skills say 'how to do this class of task for this user'. Both " + "should carry user-preference lessons when relevant.\n\n" + "If you notice overlapping existing skills, mention it — the " + "background curator handles consolidation.\n\n" + "Do NOT capture as skills (these become persistent self-imposed " + "constraints that bite you later when the environment changes):\n" + " • Environment-dependent failures: missing binaries, fresh-install " + "errors, post-migration path mismatches, 'command not found', " + "unconfigured credentials, uninstalled packages. The user can fix " + "these — they are not durable rules.\n" + " • Negative claims about tools or features ('browser tools do not " + "work', 'X tool is broken', 'cannot use Y from execute_code'). These " + "harden into refusals the agent cites against itself for months " + "after the actual problem was fixed.\n" + " • Session-specific transient errors that resolved before the " + "conversation ended. If retrying worked, the lesson is the retry " + "pattern, not the original failure.\n" + " • One-off task narratives. A user asking 'summarize today's " + "market' or 'analyze this PR' is not a class of work that warrants " + "a skill.\n\n" + "If a tool failed because of setup state, capture the FIX (install " + "command, config step, env var to set) under an existing setup or " + "troubleshooting skill — never 'this tool does not work' as a " + "standalone constraint.\n\n" + "Act on whichever of the two dimensions has real signal. If " + "genuinely nothing stands out on either, say 'Nothing to save.' " + "and stop — but don't reach for that conclusion as a default." +) + + + def summarize_background_review_actions( review_messages: List[Dict], prior_snapshot: List[Dict], @@ -339,13 +528,15 @@ def spawn_background_review_thread( owns the actual ``threading.Thread`` construction so test-level patches of ``run_agent.threading.Thread`` keep working. """ - # Pick the right prompt based on which triggers fired + # Pick the right prompt based on which triggers fired. Allow per-agent + # override (the prompts moved to module-level constants but old code paths + # that set agent._MEMORY_REVIEW_PROMPT etc. directly keep working). if review_memory and review_skills: - prompt = agent._COMBINED_REVIEW_PROMPT + prompt = getattr(agent, "_COMBINED_REVIEW_PROMPT", _COMBINED_REVIEW_PROMPT) elif review_memory: - prompt = agent._MEMORY_REVIEW_PROMPT + prompt = getattr(agent, "_MEMORY_REVIEW_PROMPT", _MEMORY_REVIEW_PROMPT) else: - prompt = agent._SKILL_REVIEW_PROMPT + prompt = getattr(agent, "_SKILL_REVIEW_PROMPT", _SKILL_REVIEW_PROMPT) def _target() -> None: _run_review_in_thread(agent, messages_snapshot, prompt) @@ -354,6 +545,9 @@ def spawn_background_review_thread( __all__ = [ + "_MEMORY_REVIEW_PROMPT", + "_SKILL_REVIEW_PROMPT", + "_COMBINED_REVIEW_PROMPT", "spawn_background_review_thread", "summarize_background_review_actions", "build_memory_write_metadata", diff --git a/run_agent.py b/run_agent.py index 8d6f7c3f35c..8ea73167ac9 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2521,190 +2521,12 @@ class AIAgent: return cleanup_task_resources(self, task_id) # ------------------------------------------------------------------ - # Background memory/skill review + # Background memory/skill review — prompts live in agent.background_review # ------------------------------------------------------------------ - - _MEMORY_REVIEW_PROMPT = ( - "Review the conversation above and consider saving to memory if appropriate.\n\n" - "Focus on:\n" - "1. Has the user revealed things about themselves — their persona, desires, " - "preferences, or personal details worth remembering?\n" - "2. Has the user expressed expectations about how you should behave, their work " - "style, or ways they want you to operate?\n\n" - "If something stands out, save it using the memory tool. " - "If nothing is worth saving, just say 'Nothing to save.' and stop." - ) - - _SKILL_REVIEW_PROMPT = ( - "Review the conversation above and update the skill library. Be " - "ACTIVE — most sessions produce at least one skill update, even if " - "small. A pass that does nothing is a missed learning opportunity, " - "not a neutral outcome.\n\n" - "Target shape of the library: CLASS-LEVEL skills, each with a rich " - "SKILL.md and a `references/` directory for session-specific detail. " - "Not a long flat list of narrow one-session-one-skill entries. This " - "shapes HOW you update, not WHETHER you update.\n\n" - "Signals to look for (any one of these warrants action):\n" - " • User corrected your style, tone, format, legibility, or " - "verbosity. Frustration signals like 'stop doing X', 'this is too " - "verbose', 'don't format like this', 'why are you explaining', " - "'just give me the answer', 'you always do Y and I hate it', or an " - "explicit 'remember this' are FIRST-CLASS skill signals, not just " - "memory signals. Update the relevant skill(s) to embed the " - "preference so the next session starts already knowing.\n" - " • User corrected your workflow, approach, or sequence of steps. " - "Encode the correction as a pitfall or explicit step in the skill " - "that governs that class of task.\n" - " • Non-trivial technique, fix, workaround, debugging path, or " - "tool-usage pattern emerged that a future session would benefit " - "from. Capture it.\n" - " • A skill that got loaded or consulted this session turned out " - "to be wrong, missing a step, or outdated. Patch it NOW.\n\n" - "Preference order — prefer the earliest action that fits, but do " - "pick one when a signal above fired:\n" - " 1. UPDATE A CURRENTLY-LOADED SKILL. Look back through the " - "conversation for skills the user loaded via /skill-name or you " - "read via skill_view. If any of them covers the territory of the " - "new learning, PATCH that one first. It is the skill that was in " - "play, so it's the right one to extend.\n" - " 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). " - "If no loaded skill fits but an existing class-level skill does, " - "patch it. Add a subsection, a pitfall, or broaden a trigger.\n" - " 3. ADD A SUPPORT FILE under an existing umbrella. Skills can be " - "packaged with three kinds of support files — use the right " - "directory per kind:\n" - " • `references/.md` — session-specific detail (error " - "transcripts, reproduction recipes, provider quirks) AND " - "condensed knowledge banks: quoted research, API docs, external " - "authoritative excerpts, or domain notes you found while working " - "on the problem. Write it concise and for the value of the task, " - "not as a full mirror of upstream docs.\n" - " • `templates/.` — starter files meant to be " - "copied and modified (boilerplate configs, scaffolding, a " - "known-good example the agent can `reproduce with modifications`).\n" - " • `scripts/.` — statically re-runnable actions " - "the skill can invoke directly (verification scripts, fixture " - "generators, deterministic probes, anything the agent should run " - "rather than hand-type each time).\n" - " Add support files via skill_manage action=write_file with " - "file_path starting 'references/', 'templates/', or 'scripts/'. " - "The umbrella's SKILL.md should gain a one-line pointer to any " - "new support file so future agents know it exists.\n" - " 4. CREATE A NEW CLASS-LEVEL UMBRELLA SKILL when no existing " - "skill covers the class. The name MUST be at the class level. " - "The name MUST NOT be a specific PR number, error string, feature " - "codename, library-alone name, or 'fix-X / debug-Y / audit-Z-today' " - "session artifact. If the proposed name only makes sense for " - "today's task, it's wrong — fall back to (1), (2), or (3).\n\n" - "User-preference embedding (important): when the user expressed a " - "style/format/workflow preference, the update belongs in the " - "SKILL.md body, not just in memory. Memory captures 'who the user " - "is and what the current situation and state of your operations " - "are'; skills capture 'how to do this class of task for this " - "user'. When they complain about how you handled a task, the " - "skill that governs that task needs to carry the lesson.\n\n" - "If you notice two existing skills that overlap, note it in your " - "reply — the background curator handles consolidation at scale.\n\n" - "Do NOT capture (these become persistent self-imposed constraints " - "that bite you later when the environment changes):\n" - " • Environment-dependent failures: missing binaries, fresh-install " - "errors, post-migration path mismatches, 'command not found', " - "unconfigured credentials, uninstalled packages. The user can fix " - "these — they are not durable rules.\n" - " • Negative claims about tools or features ('browser tools do not " - "work', 'X tool is broken', 'cannot use Y from execute_code'). These " - "harden into refusals the agent cites against itself for months " - "after the actual problem was fixed.\n" - " • Session-specific transient errors that resolved before the " - "conversation ended. If retrying worked, the lesson is the retry " - "pattern, not the original failure.\n" - " • One-off task narratives. A user asking 'summarize today's " - "market' or 'analyze this PR' is not a class of work that warrants " - "a skill.\n\n" - "If a tool failed because of setup state, capture the FIX (install " - "command, config step, env var to set) under an existing setup or " - "troubleshooting skill — never 'this tool does not work' as a " - "standalone constraint.\n\n" - "'Nothing to save.' is a real option but should NOT be the " - "default. If the session ran smoothly with no corrections and " - "produced no new technique, just say 'Nothing to save.' and stop. " - "Otherwise, act." - ) - - _COMBINED_REVIEW_PROMPT = ( - "Review the conversation above and update two things:\n\n" - "**Memory**: who the user is. Did the user reveal persona, " - "desires, preferences, personal details, or expectations about " - "how you should behave? Save facts about the user and durable " - "preferences with the memory tool.\n\n" - "**Skills**: how to do this class of task. Be ACTIVE — most " - "sessions produce at least one skill update. A pass that does " - "nothing is a missed learning opportunity, not a neutral outcome.\n\n" - "Target shape of the skill library: CLASS-LEVEL skills with a rich " - "SKILL.md and a `references/` directory for session-specific detail. " - "Not a long flat list of narrow one-session-one-skill entries.\n\n" - "Signals that warrant a skill update (any one is enough):\n" - " • User corrected your style, tone, format, legibility, " - "verbosity, or approach. Frustration is a FIRST-CLASS skill " - "signal, not just a memory signal. 'stop doing X', 'don't format " - "like this', 'I hate when you Y' — embed the lesson in the skill " - "that governs that task so the next session starts fixed.\n" - " • Non-trivial technique, fix, workaround, or debugging path " - "emerged.\n" - " • A skill that was loaded or consulted turned out wrong, " - "missing, or outdated — patch it now.\n\n" - "Preference order for skills — pick the earliest that fits:\n" - " 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were " - "loaded via /skill-name or skill_view in the conversation. If one " - "of them covers the learning, PATCH it first. It was in play; " - "it's the right place.\n" - " 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to " - "find the right one). Patch it.\n" - " 3. ADD A SUPPORT FILE under an existing umbrella via " - "skill_manage action=write_file. Three kinds: " - "`references/.md` for session-specific detail OR condensed " - "knowledge banks (quoted research, API docs excerpts, domain " - "notes) written concise and task-focused; `templates/.` " - "for starter files meant to be copied and modified; " - "`scripts/.` for statically re-runnable actions " - "(verification, fixture generators, probes). Add a one-line " - "pointer in SKILL.md so future agents find them.\n" - " 4. CREATE A NEW CLASS-LEVEL UMBRELLA when nothing exists. " - "Name at the class level — NOT a PR number, error string, " - "codename, library-alone name, or 'fix-X / debug-Y' session " - "artifact. If the name only fits today's task, fall back to (1), " - "(2), or (3).\n\n" - "User-preference embedding: when the user complains about how " - "you handled a task, update the skill that governs that task — " - "memory alone isn't enough. Memory says 'who the user is and " - "what the current situation and state of your operations are'; " - "skills say 'how to do this class of task for this user'. Both " - "should carry user-preference lessons when relevant.\n\n" - "If you notice overlapping existing skills, mention it — the " - "background curator handles consolidation.\n\n" - "Do NOT capture as skills (these become persistent self-imposed " - "constraints that bite you later when the environment changes):\n" - " • Environment-dependent failures: missing binaries, fresh-install " - "errors, post-migration path mismatches, 'command not found', " - "unconfigured credentials, uninstalled packages. The user can fix " - "these — they are not durable rules.\n" - " • Negative claims about tools or features ('browser tools do not " - "work', 'X tool is broken', 'cannot use Y from execute_code'). These " - "harden into refusals the agent cites against itself for months " - "after the actual problem was fixed.\n" - " • Session-specific transient errors that resolved before the " - "conversation ended. If retrying worked, the lesson is the retry " - "pattern, not the original failure.\n" - " • One-off task narratives. A user asking 'summarize today's " - "market' or 'analyze this PR' is not a class of work that warrants " - "a skill.\n\n" - "If a tool failed because of setup state, capture the FIX (install " - "command, config step, env var to set) under an existing setup or " - "troubleshooting skill — never 'this tool does not work' as a " - "standalone constraint.\n\n" - "Act on whichever of the two dimensions has real signal. If " - "genuinely nothing stands out on either, say 'Nothing to save.' " - "and stop — but don't reach for that conclusion as a default." + from agent.background_review import ( + _MEMORY_REVIEW_PROMPT, + _SKILL_REVIEW_PROMPT, + _COMBINED_REVIEW_PROMPT, ) @staticmethod From 053025238434cfbf121873977b39888d7f27d1c1 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 19:26:52 -0700 Subject: [PATCH 013/142] refactor(run_agent): extract run_conversation to agent/conversation_loop.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3,877-line run_conversation body — the agent loop itself — moves out of run_agent.py into a dedicated module. AIAgent.run_conversation is now a thin forwarder that delegates to agent.conversation_loop.run_conversation with the AIAgent instance as the first argument. This is the largest single extraction in the run_agent.py refactor. The body keeps all 163 self.X references intact (rewritten as agent.X), all nested closures, all retry/backoff/compression machinery. Symbols that tests or callers patch on run_agent (_set_interrupt, handle_function_call, AIAgent class attrs) are resolved through _ra() inside the extracted module so the patch surface is preserved. Five tests doing inspect.getsource(AIAgent.run_conversation) updated to scan agent.conversation_loop.run_conversation. Two source-introspection tests (TestMemoryNudgeCounterPersistence, TestMemoryProviderTurnStart) updated to accept either self.X (legacy) or agent.X (extracted form) in the matched assertions. Live E2E verified on three model paths: * openai/gpt-5.4 (OpenAI chat completions via OpenRouter) * anthropic/claude-sonnet-4.6 (Anthropic Messages via OpenRouter) * moonshotai/kimi-k2-thinking (reasoning model, reasoning_content path) Plus read_file tool execution, terminal tool, web_search. tests/run_agent/ + tests/agent/: 4313 passed, 1 pre-existing failure (test_auxiliary_client::test_custom_endpoint... — same as on main). run_agent.py: 9800 -> 5944 lines (-3856). Total reduction since baseline: 16083 -> 5944 (-10139, 63%). --- agent/conversation_loop.py | 3964 +++++++++++++++++ run_agent.py | 3870 +--------------- .../test_jsondecodeerror_retryable.py | 4 +- .../test_memory_nudge_counter_hydration.py | 16 +- tests/run_agent/test_run_agent.py | 30 +- 5 files changed, 4005 insertions(+), 3879 deletions(-) create mode 100644 agent/conversation_loop.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py new file mode 100644 index 00000000000..c95f1b63385 --- /dev/null +++ b/agent/conversation_loop.py @@ -0,0 +1,3964 @@ +"""The agent conversation loop — extracted from ``run_agent.AIAgent``. + +This is the biggest single chunk pulled out of ``run_agent.py``: the +roughly 3,900-line :func:`run_conversation` body that drives one user +turn through the agent (model call, tool dispatch, retries, fallbacks, +compression, post-turn hooks, background memory/skill review nudges). + +The function takes the parent ``AIAgent`` instance as its first +argument (``agent``) and accesses its state via attribute lookup. +``_ra().AIAgent.run_conversation`` is now a thin forwarder. + +Symbols that production code or tests patch on ``run_agent`` directly +(``handle_function_call``, ``_set_interrupt``, ``OpenAI``, ...) are +resolved through :func:`_ra` so those patches keep working. +""" + +from __future__ import annotations + +import json +import logging +import os +import random +import re +import ssl +import threading +import time +import uuid +from typing import Any, Dict, List, Optional + +from agent.anthropic_adapter import _is_oauth_token +from agent.auxiliary_client import set_runtime_main +from agent.codex_responses_adapter import _summarize_user_message_for_log +from agent.display import KawaiiSpinner +from agent.error_classifier import FailoverReason, classify_api_error +from agent.iteration_budget import IterationBudget +from agent.memory_manager import build_memory_context_block +from agent.message_sanitization import ( + _repair_tool_call_arguments, + _sanitize_messages_non_ascii, + _sanitize_messages_surrogates, + _sanitize_structure_non_ascii, + _sanitize_structure_surrogates, + _sanitize_surrogates, + _sanitize_tools_non_ascii, + _strip_images_from_messages, + _strip_non_ascii, +) +from agent.model_metadata import ( + estimate_messages_tokens_rough, + estimate_request_tokens_rough, + get_next_probe_tier, + parse_available_output_tokens_from_error, + parse_context_limit_from_error, + save_context_length, +) +from agent.nous_rate_guard import ( + clear_nous_rate_limit, + is_genuine_nous_rate_limit, + nous_rate_limit_remaining, + record_nous_rate_limit, +) +from agent.process_bootstrap import _install_safe_stdio +from agent.prompt_caching import apply_anthropic_cache_control +from agent.retry_utils import jittered_backoff +from agent.trajectory import has_incomplete_scratchpad +from agent.usage_pricing import estimate_usage_cost, normalize_usage +from hermes_constants import display_hermes_home as _dhh_fn +from hermes_logging import set_session_context +from tools.schema_sanitizer import strip_pattern_and_format +from tools.skill_provenance import set_current_write_origin +from utils import base_url_host_matches, env_var_enabled + +logger = logging.getLogger(__name__) + + +def _ra(): + """Lazy reference to ``run_agent`` so callers can patch + ``run_agent.handle_function_call`` / ``run_agent._set_interrupt`` / + ``run_agent.OpenAI`` and have those patches reach this code path. + """ + import run_agent + return run_agent + + +def run_conversation( + agent, + user_message: str, + system_message: str = None, + conversation_history: List[Dict[str, Any]] = None, + task_id: str = None, + stream_callback: Optional[callable] = None, + persist_user_message: Optional[str] = None, +) -> Dict[str, Any]: + """ + Run a complete conversation with tool calling until completion. + + Args: + user_message (str): The user's message/question + system_message (str): Custom system message (optional, overrides ephemeral_system_prompt if provided) + conversation_history (List[Dict]): Previous conversation messages (optional) + task_id (str): Unique identifier for this task to isolate VMs between concurrent tasks (optional, auto-generated if not provided) + stream_callback: Optional callback invoked with each text delta during streaming. + Used by the TTS pipeline to start audio generation before the full response. + When None (default), API calls use the standard non-streaming path. + persist_user_message: Optional clean user message to store in + transcripts/history when user_message contains API-only + synthetic prefixes. + or queuing follow-up prefetch work. + + Returns: + Dict: Complete conversation result with final response and message history + """ + # Guard stdio against OSError from broken pipes (systemd/headless/daemon). + # Installed once, transparent when streams are healthy, prevents crash on write. + _install_safe_stdio() + + agent._ensure_db_session() + + # Tell auxiliary_client what the live main provider/model are for + # this turn. Used by tools whose behaviour depends on the active + # main model (e.g. vision_analyze's native fast path) so they see + # the CLI/gateway override instead of the stale config.yaml + # default. Idempotent — fine to call every turn. + try: + from agent.auxiliary_client import set_runtime_main + set_runtime_main( + getattr(agent, "provider", "") or "", + getattr(agent, "model", "") or "", + ) + except Exception: + pass + + # Tag all log records on this thread with the session ID so + # ``hermes logs --session `` can filter a single conversation. + from hermes_logging import set_session_context + set_session_context(agent.session_id) + + # Bind the skill write-origin ContextVar for this thread so tool + # handlers (e.g. skill_manage create) can tell whether they are + # running inside the background agent-improvement review fork vs. + # a foreground user-directed turn. Set at the top of each call; + # the review fork runs on its own thread with a fresh context, + # so the foreground value here does not leak into it. + from tools.skill_provenance import set_current_write_origin + set_current_write_origin(getattr(agent, "_memory_write_origin", "assistant_tool")) + + # If the previous turn activated fallback, restore the primary + # runtime so this turn gets a fresh attempt with the preferred model. + # No-op when _fallback_activated is False (gateway, first turn, etc.). + agent._restore_primary_runtime() + + # Sanitize surrogate characters from user input. Clipboard paste from + # rich-text editors (Google Docs, Word, etc.) can inject lone surrogates + # that are invalid UTF-8 and crash JSON serialization in the OpenAI SDK. + if isinstance(user_message, str): + user_message = _sanitize_surrogates(user_message) + if isinstance(persist_user_message, str): + persist_user_message = _sanitize_surrogates(persist_user_message) + + # Store stream callback for _interruptible_api_call to pick up + agent._stream_callback = stream_callback + agent._persist_user_message_idx = None + agent._persist_user_message_override = persist_user_message + # Generate unique task_id if not provided to isolate VMs between concurrent tasks + effective_task_id = task_id or str(uuid.uuid4()) + # Expose the active task_id so tools running mid-turn (e.g. delegate_task + # in delegate_tool.py) can identify this agent for the cross-agent file + # state registry. Set BEFORE any tool dispatch so snapshots taken at + # child-launch time see the parent's real id, not None. + agent._current_task_id = effective_task_id + + # Reset retry counters and iteration budget at the start of each turn + # so subagent usage from a previous turn doesn't eat into the next one. + agent._invalid_tool_retries = 0 + agent._invalid_json_retries = 0 + agent._empty_content_retries = 0 + agent._incomplete_scratchpad_retries = 0 + agent._codex_incomplete_retries = 0 + agent._thinking_prefill_retries = 0 + agent._post_tool_empty_retried = False + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + agent._mute_post_response = False + agent._unicode_sanitization_passes = 0 + agent._tool_guardrails.reset_for_turn() + agent._tool_guardrail_halt_decision = None + # True until the server rejects an image_url content part with an error + # like "Only 'text' content type is supported." Set to False on first + # rejection and kept False for the rest of the session so we never re-send + # images to a text-only endpoint. Scoped per `_run()` call, not per instance. + agent._vision_supported = True + + # Pre-turn connection health check: detect and clean up dead TCP + # connections left over from provider outages or dropped streams. + # This prevents the next API call from hanging on a zombie socket. + if agent.api_mode != "anthropic_messages": + try: + if agent._cleanup_dead_connections(): + agent._emit_status( + "🔌 Detected stale connections from a previous provider " + "issue — cleaned up automatically. Proceeding with fresh " + "connection." + ) + except Exception: + pass + # Replay compression warning through status_callback for gateway + # platforms (the callback was not wired during __init__). + if agent._compression_warning: + agent._replay_compression_warning() + agent._compression_warning = None # send once + + # NOTE: _turns_since_memory and _iters_since_skill are NOT reset here. + # They are initialized in __init__ and must persist across run_conversation + # calls so that nudge logic accumulates correctly in CLI mode. + agent.iteration_budget = IterationBudget(agent.max_iterations) + + # Log conversation turn start for debugging/observability + _preview_text = _summarize_user_message_for_log(user_message) + _msg_preview = (_preview_text[:80] + "...") if len(_preview_text) > 80 else _preview_text + _msg_preview = _msg_preview.replace("\n", " ") + logger.info( + "conversation turn: session=%s model=%s provider=%s platform=%s history=%d msg=%r", + agent.session_id or "none", agent.model, agent.provider or "unknown", + agent.platform or "unknown", len(conversation_history or []), + _msg_preview, + ) + + # Initialize conversation (copy to avoid mutating the caller's list) + messages = list(conversation_history) if conversation_history else [] + + # Hydrate todo store from conversation history (gateway creates a fresh + # AIAgent per message, so the in-memory store is empty -- we need to + # recover the todo state from the most recent todo tool response in history) + if conversation_history and not agent._todo_store.has_items(): + agent._hydrate_todo_store(conversation_history) + + # Hydrate per-session nudge counters from persisted history. + # Gateway creates a fresh AIAgent per inbound message (cache miss / + # 1h idle eviction / config-signature mismatch / process restart), so + # _turns_since_memory and _user_turn_count start at 0 every turn and + # the memory.nudge_interval trigger may never be reached. Reconstruct + # an effective count from prior user turns in conversation_history. + # Idempotent: a cached agent that already accumulated counters keeps + # them; only a freshly-built agent with empty in-memory state hydrates. + # See issue #22357. + if conversation_history and agent._user_turn_count == 0: + prior_user_turns = sum( + 1 for m in conversation_history if m.get("role") == "user" + ) + if prior_user_turns > 0: + agent._user_turn_count = prior_user_turns + if agent._memory_nudge_interval > 0 and agent._turns_since_memory == 0: + # % preserves original 1-in-N cadence rather than firing a + # review immediately on resume (which would surprise users + # whose session happened to land just past a multiple of N). + agent._turns_since_memory = prior_user_turns % agent._memory_nudge_interval + + + # Prefill messages (few-shot priming) are injected at API-call time only, + # never stored in the messages list. This keeps them ephemeral: they won't + # be saved to session DB, session logs, or batch trajectories, but they're + # automatically re-applied on every API call (including session continuations). + + # Track user turns for memory flush and periodic nudge logic + agent._user_turn_count += 1 + + # Reset the streaming context scrubber at the top of each turn so a + # hung span from a prior interrupted stream can't taint this turn's + # output. + scrubber = getattr(agent, "_stream_context_scrubber", None) + if scrubber is not None: + scrubber.reset() + # Reset the think scrubber for the same reason — an interrupted + # prior stream may have left us inside an unterminated block. + think_scrubber = getattr(agent, "_stream_think_scrubber", None) + if think_scrubber is not None: + think_scrubber.reset() + + # Preserve the original user message (no nudge injection). + original_user_message = persist_user_message if persist_user_message is not None else user_message + + # Track memory nudge trigger (turn-based, checked here). + # Skill trigger is checked AFTER the agent loop completes, based on + # how many tool iterations THIS turn used. + _should_review_memory = False + if (agent._memory_nudge_interval > 0 + and "memory" in agent.valid_tool_names + and agent._memory_store): + agent._turns_since_memory += 1 + if agent._turns_since_memory >= agent._memory_nudge_interval: + _should_review_memory = True + agent._turns_since_memory = 0 + + # Add user message + user_msg = {"role": "user", "content": user_message} + messages.append(user_msg) + current_turn_user_idx = len(messages) - 1 + agent._persist_user_message_idx = current_turn_user_idx + + if not agent.quiet_mode: + _print_preview = _summarize_user_message_for_log(user_message) + agent._safe_print(f"💬 Starting conversation: '{_print_preview[:60]}{'...' if len(_print_preview) > 60 else ''}'") + + # ── System prompt (cached per session for prefix caching) ── + # Built once on first call, reused for all subsequent calls. + # Only rebuilt after context compression events (which invalidate + # the cache and reload memory from disk). + # + # For continuing sessions (gateway creates a fresh AIAgent per + # message), we load the stored system prompt from the session DB + # instead of rebuilding. Rebuilding would pick up memory changes + # from disk that the model already knows about (it wrote them!), + # producing a different system prompt and breaking the Anthropic + # prefix cache. + if agent._cached_system_prompt is None: + stored_prompt = None + if conversation_history and agent._session_db: + try: + session_row = agent._session_db.get_session(agent.session_id) + if session_row: + stored_prompt = session_row.get("system_prompt") or None + except Exception: + pass # Fall through to build fresh + + if stored_prompt: + # Continuing session — reuse the exact system prompt from + # the previous turn so the Anthropic cache prefix matches. + agent._cached_system_prompt = stored_prompt + else: + # First turn of a new session — build from scratch. + agent._cached_system_prompt = agent._build_system_prompt(system_message) + # Plugin hook: on_session_start + # Fired once when a brand-new session is created (not on + # continuation). Plugins can use this to initialise + # session-scoped state (e.g. warm a memory cache). + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _invoke_hook( + "on_session_start", + session_id=agent.session_id, + model=agent.model, + platform=getattr(agent, "platform", None) or "", + ) + except Exception as exc: + logger.warning("on_session_start hook failed: %s", exc) + + # Store the system prompt snapshot in SQLite + if agent._session_db: + try: + agent._session_db.update_system_prompt(agent.session_id, agent._cached_system_prompt) + except Exception as e: + logger.debug("Session DB update_system_prompt failed: %s", e) + + active_system_prompt = agent._cached_system_prompt + + # ── Preflight context compression ── + # Before entering the main loop, check if the loaded conversation + # history already exceeds the model's context threshold. This handles + # cases where a user switches to a model with a smaller context window + # while having a large existing session — compress proactively rather + # than waiting for an API error (which might be caught as a non-retryable + # 4xx and abort the request entirely). + if ( + agent.compression_enabled + and len(messages) > agent.context_compressor.protect_first_n + + agent.context_compressor.protect_last_n + 1 + ): + # Include tool schema tokens — with many tools these can add + # 20-30K+ tokens that the old sys+msg estimate missed entirely. + _preflight_tokens = estimate_request_tokens_rough( + messages, + system_prompt=active_system_prompt or "", + tools=agent.tools or None, + ) + + if _preflight_tokens >= agent.context_compressor.threshold_tokens: + logger.info( + "Preflight compression: ~%s tokens >= %s threshold (model %s, ctx %s)", + f"{_preflight_tokens:,}", + f"{agent.context_compressor.threshold_tokens:,}", + agent.model, + f"{agent.context_compressor.context_length:,}", + ) + agent._emit_status( + f"📦 Preflight compression: ~{_preflight_tokens:,} tokens " + f">= {agent.context_compressor.threshold_tokens:,} threshold. " + "This may take a moment." + ) + # May need multiple passes for very large sessions with small + # context windows (each pass summarises the middle N turns). + for _pass in range(3): + _orig_len = len(messages) + messages, active_system_prompt = agent._compress_context( + messages, system_message, approx_tokens=_preflight_tokens, + task_id=effective_task_id, + ) + if len(messages) >= _orig_len: + break # Cannot compress further + # Compression created a new session — clear the history + # reference so _flush_messages_to_session_db writes ALL + # compressed messages to the new session's SQLite, not + # skipping them because conversation_history is still the + # pre-compression length. + conversation_history = None + # Fix: reset retry counters after compression so the model + # gets a fresh budget on the compressed context. Without + # this, pre-compression retries carry over and the model + # hits "(empty)" immediately after compression-induced + # context loss. + agent._empty_content_retries = 0 + agent._thinking_prefill_retries = 0 + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + agent._mute_post_response = False + # Re-estimate after compression + _preflight_tokens = estimate_request_tokens_rough( + messages, + system_prompt=active_system_prompt or "", + tools=agent.tools or None, + ) + if _preflight_tokens < agent.context_compressor.threshold_tokens: + break # Under threshold + + # Plugin hook: pre_llm_call + # Fired once per turn before the tool-calling loop. Plugins can + # return a dict with a ``context`` key (or a plain string) whose + # value is appended to the current turn's user message. + # + # Context is ALWAYS injected into the user message, never the + # system prompt. This preserves the prompt cache prefix — the + # system prompt stays identical across turns so cached tokens + # are reused. The system prompt is Hermes's territory; plugins + # contribute context alongside the user's input. + # + # All injected context is ephemeral (not persisted to session DB). + _plugin_user_context = "" + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _pre_results = _invoke_hook( + "pre_llm_call", + session_id=agent.session_id, + user_message=original_user_message, + conversation_history=list(messages), + is_first_turn=(not bool(conversation_history)), + model=agent.model, + platform=getattr(agent, "platform", None) or "", + sender_id=getattr(agent, "_user_id", None) or "", + ) + _ctx_parts: list[str] = [] + for r in _pre_results: + if isinstance(r, dict) and r.get("context"): + _ctx_parts.append(str(r["context"])) + elif isinstance(r, str) and r.strip(): + _ctx_parts.append(r) + if _ctx_parts: + _plugin_user_context = "\n\n".join(_ctx_parts) + except Exception as exc: + logger.warning("pre_llm_call hook failed: %s", exc) + + # Main conversation loop + api_call_count = 0 + final_response = None + interrupted = False + codex_ack_continuations = 0 + length_continue_retries = 0 + truncated_tool_call_retries = 0 + truncated_response_prefix = "" + compression_attempts = 0 + _turn_exit_reason = "unknown" # Diagnostic: why the loop ended + + # Per-turn file-mutation verifier state. Keyed by resolved path; + # each failed ``write_file`` / ``patch`` call records the error + # preview. Later successful writes to the same path remove the + # entry (the model recovered). At end-of-turn, any entries still + # present are surfaced in an advisory footer so the model cannot + # over-claim success while the file is actually unchanged on disk. + agent._turn_failed_file_mutations: Dict[str, Dict[str, Any]] = {} + + # Record the execution thread so interrupt()/clear_interrupt() can + # scope the tool-level interrupt signal to THIS agent's thread only. + # Must be set before any thread-scoped interrupt syncing. + agent._execution_thread_id = threading.current_thread().ident + + # Always clear stale per-thread state from a previous turn. If an + # interrupt arrived before startup finished, preserve it and bind it + # to this execution thread now instead of dropping it on the floor. + _ra()._set_interrupt(False, agent._execution_thread_id) + if agent._interrupt_requested: + _ra()._set_interrupt(True, agent._execution_thread_id) + agent._interrupt_thread_signal_pending = False + else: + agent._interrupt_message = None + agent._interrupt_thread_signal_pending = False + + # Notify memory providers of the new turn so cadence tracking works. + # Must happen BEFORE prefetch_all() so providers know which turn it is + # and can gate context/dialectic refresh via contextCadence/dialecticCadence. + if agent._memory_manager: + try: + _turn_msg = original_user_message if isinstance(original_user_message, str) else "" + agent._memory_manager.on_turn_start(agent._user_turn_count, _turn_msg) + except Exception: + pass + + # External memory provider: prefetch once before the tool loop. + # Reuse the cached result on every iteration to avoid re-calling + # prefetch_all() on each tool call (10 tool calls = 10x latency + cost). + # Use original_user_message (clean input) — user_message may contain + # injected skill content that bloats / breaks provider queries. + _ext_prefetch_cache = "" + if agent._memory_manager: + try: + _query = original_user_message if isinstance(original_user_message, str) else "" + _ext_prefetch_cache = agent._memory_manager.prefetch_all(_query) or "" + except Exception: + pass + + # Optional opt-in runtime: if api_mode == codex_app_server, hand the + # turn to the codex app-server subprocess (terminal/file ops/patching + # all run inside Codex). Default Hermes path is bypassed entirely. + # See agent/transports/codex_app_server_session.py for the adapter + # and references/codex-app-server-runtime.md for the rationale. + if agent.api_mode == "codex_app_server": + return agent._run_codex_app_server_turn( + user_message=user_message, + original_user_message=original_user_message, + messages=messages, + effective_task_id=effective_task_id, + should_review_memory=_should_review_memory, + ) + + while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: + # Reset per-turn checkpoint dedup so each iteration can take one snapshot + agent._checkpoint_mgr.new_turn() + + # Check for interrupt request (e.g., user sent new message) + if agent._interrupt_requested: + interrupted = True + _turn_exit_reason = "interrupted_by_user" + if not agent.quiet_mode: + agent._safe_print("\n⚡ Breaking out of tool loop due to interrupt...") + break + + api_call_count += 1 + agent._api_call_count = api_call_count + agent._touch_activity(f"starting API call #{api_call_count}") + + # Grace call: the budget is exhausted but we gave the model one + # more chance. Consume the grace flag so the loop exits after + # this iteration regardless of outcome. + if agent._budget_grace_call: + agent._budget_grace_call = False + elif not agent.iteration_budget.consume(): + _turn_exit_reason = "budget_exhausted" + if not agent.quiet_mode: + agent._safe_print(f"\n⚠️ Iteration budget exhausted ({agent.iteration_budget.used}/{agent.iteration_budget.max_total} iterations used)") + break + + # Fire step_callback for gateway hooks (agent:step event) + if agent.step_callback is not None: + try: + prev_tools = [] + for _idx, _m in enumerate(reversed(messages)): + if _m.get("role") == "assistant" and _m.get("tool_calls"): + _fwd_start = len(messages) - _idx + _results_by_id = {} + for _tm in messages[_fwd_start:]: + if _tm.get("role") != "tool": + break + _tcid = _tm.get("tool_call_id") + if _tcid: + _results_by_id[_tcid] = _tm.get("content", "") + prev_tools = [ + { + "name": tc["function"]["name"], + "result": _results_by_id.get(tc.get("id")), + "arguments": tc["function"].get("arguments"), + } + for tc in _m["tool_calls"] + if isinstance(tc, dict) + ] + break + agent.step_callback(api_call_count, prev_tools) + except Exception as _step_err: + logger.debug("step_callback error (iteration %s): %s", api_call_count, _step_err) + + # Track tool-calling iterations for skill nudge. + # Counter resets whenever skill_manage is actually used. + if (agent._skill_nudge_interval > 0 + and "skill_manage" in agent.valid_tool_names): + agent._iters_since_skill += 1 + + # ── Pre-API-call /steer drain ────────────────────────────────── + # If a /steer arrived during the previous API call (while the model + # was thinking), drain it now — before we build api_messages — so + # the model sees the steer text on THIS iteration. Without this, + # steers sent during an API call only land after the NEXT tool batch, + # which may never come if the model returns a final response. + # + # We scan backwards for the last tool-role message in the messages + # list. If found, the steer is appended there. If not (first + # iteration, no tools yet), the steer stays pending for the next + # tool batch — injecting into a user message would break role + # alternation, and there's no tool output to piggyback on. + _pre_api_steer = agent._drain_pending_steer() + if _pre_api_steer: + _injected = False + for _si in range(len(messages) - 1, -1, -1): + _sm = messages[_si] + if isinstance(_sm, dict) and _sm.get("role") == "tool": + marker = f"\n\nUser guidance: {_pre_api_steer}" + existing = _sm.get("content", "") + if isinstance(existing, str): + _sm["content"] = existing + marker + else: + # Multimodal content blocks — append text block + try: + blocks = list(existing) if existing else [] + blocks.append({"type": "text", "text": marker}) + _sm["content"] = blocks + except Exception: + pass + _injected = True + logger.debug( + "Pre-API-call steer drain: injected into tool msg at index %d", + _si, + ) + break + if not _injected: + # No tool message to inject into — put it back so + # the post-tool-execution drain picks it up later. + _lock = getattr(agent, "_pending_steer_lock", None) + if _lock is not None: + with _lock: + if agent._pending_steer: + agent._pending_steer = agent._pending_steer + "\n" + _pre_api_steer + else: + agent._pending_steer = _pre_api_steer + else: + existing = getattr(agent, "_pending_steer", None) + agent._pending_steer = (existing + "\n" + _pre_api_steer) if existing else _pre_api_steer + + # Prepare messages for API call + # If we have an ephemeral system prompt, prepend it to the messages + # Note: Reasoning is embedded in content via tags for trajectory storage. + # However, providers like Moonshot AI require a separate 'reasoning_content' field + # on assistant messages with tool_calls. We handle both cases here. + request_logger = getattr(agent, "logger", None) or logging.getLogger(__name__) + repaired_tool_calls = agent._sanitize_tool_call_arguments( + messages, + logger=request_logger, + session_id=agent.session_id, + ) + if repaired_tool_calls > 0: + request_logger.info( + "Sanitized %s corrupted tool_call arguments before request (session=%s)", + repaired_tool_calls, + agent.session_id or "-", + ) + + # Defensive: repair malformed role-alternation before API call. + # Catches cases where the history got wedged into a + # ``tool → user`` or ``user → user`` tail (e.g. after empty- + # response scaffolding was stripped and a new user message + # landed after an orphan tool result). Most providers return + # empty content on malformed sequences, which would otherwise + # retrigger the empty-retry loop indefinitely. + repaired_seq = agent._repair_message_sequence(messages) + if repaired_seq > 0: + request_logger.info( + "Repaired %s message-alternation violations before request (session=%s)", + repaired_seq, + agent.session_id or "-", + ) + + api_messages = [] + for idx, msg in enumerate(messages): + api_msg = msg.copy() + + # Inject ephemeral context into the current turn's user message. + # Sources: memory manager prefetch + plugin pre_llm_call hooks + # with target="user_message" (the default). Both are + # API-call-time only — the original message in `messages` is + # never mutated, so nothing leaks into session persistence. + if idx == current_turn_user_idx and msg.get("role") == "user": + _injections = [] + if _ext_prefetch_cache: + _fenced = build_memory_context_block(_ext_prefetch_cache) + if _fenced: + _injections.append(_fenced) + if _plugin_user_context: + _injections.append(_plugin_user_context) + if _injections: + _base = api_msg.get("content", "") + if isinstance(_base, str): + api_msg["content"] = _base + "\n\n" + "\n\n".join(_injections) + + # For ALL assistant messages, pass reasoning back to the API + # This ensures multi-turn reasoning context is preserved + agent._copy_reasoning_content_for_api(msg, api_msg) + + # Remove 'reasoning' field - it's for trajectory storage only + # We've copied it to 'reasoning_content' for the API above + if "reasoning" in api_msg: + api_msg.pop("reasoning") + # Remove finish_reason - not accepted by strict APIs (e.g. Mistral) + if "finish_reason" in api_msg: + api_msg.pop("finish_reason") + # Strip internal thinking-prefill marker + api_msg.pop("_thinking_prefill", None) + # Strip Codex Responses API fields (call_id, response_item_id) for + # strict providers like Mistral, Fireworks, etc. that reject unknown fields. + # Uses new dicts so the internal messages list retains the fields + # for Codex Responses compatibility. + if agent._should_sanitize_tool_calls(): + agent._sanitize_tool_calls_for_strict_api(api_msg) + # Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context + # The signature field helps maintain reasoning continuity + api_messages.append(api_msg) + + # Build the final system message: cached prompt + ephemeral system prompt. + # Ephemeral additions are API-call-time only (not persisted to session DB). + # External recall context is injected into the user message, not the system + # prompt, so the stable cache prefix remains unchanged. + # + # NOTE: Plugin context from pre_llm_call hooks is injected into the + # user message (see injection block above), NOT the system prompt. + # This is intentional — system prompt modifications break the prompt + # cache prefix. The system prompt is reserved for Hermes internals. + # + # Hermes invariant: the system prompt is built ONCE per session + # (cached on ``_cached_system_prompt``) and replayed verbatim on + # every turn. We send it as a single content string so the + # bytes are byte-stable across turns and upstream prompt caches + # stay warm. + effective_system = active_system_prompt or "" + if agent.ephemeral_system_prompt: + effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip() + if effective_system: + api_messages = [{"role": "system", "content": effective_system}] + api_messages + + # Inject ephemeral prefill messages right after the system prompt + # but before conversation history. Same API-call-time-only pattern. + if agent.prefill_messages: + sys_offset = 1 if (api_messages and api_messages[0].get("role") == "system") else 0 + for idx, pfm in enumerate(agent.prefill_messages): + api_messages.insert(sys_offset + idx, pfm.copy()) + + # Apply Anthropic prompt caching for Claude models on native + # Anthropic, OpenRouter, and third-party Anthropic-compatible + # gateways. Auto-detected: if ``_use_prompt_caching`` is set, + # inject cache_control breakpoints (system + last 3 messages) + # to reduce input token costs by ~75% on multi-turn + # conversations. + if agent._use_prompt_caching: + api_messages = apply_anthropic_cache_control( + api_messages, + cache_ttl=agent._cache_ttl, + native_anthropic=agent._use_native_cache_layout, + ) + + # Safety net: strip orphaned tool results / add stubs for missing + # results before sending to the API. Runs unconditionally — not + # gated on context_compressor — so orphans from session loading or + # manual message manipulation are always caught. + api_messages = agent._sanitize_api_messages(api_messages) + + # Drop thinking-only assistant turns (reasoning but no visible + # output and no tool_calls) and merge any adjacent user messages + # left behind. Prevents Anthropic 400s ("The final block in an + # assistant message cannot be `thinking`.") and equivalent errors + # from third-party Anthropic-compatible gateways that can't replay + # a thinking-only turn. Runs on the per-call copy only — the + # stored conversation history keeps the reasoning block for the + # UI transcript and session persistence. + api_messages = agent._drop_thinking_only_and_merge_users(api_messages) + + # Normalize message whitespace and tool-call JSON for consistent + # prefix matching. Ensures bit-perfect prefixes across turns, + # which enables KV cache reuse on local inference servers + # (llama.cpp, vLLM, Ollama) and improves cache hit rates for + # cloud providers. Operates on api_messages (the API copy) so + # the original conversation history in `messages` is untouched. + for am in api_messages: + if isinstance(am.get("content"), str): + am["content"] = am["content"].strip() + for am in api_messages: + tcs = am.get("tool_calls") + if not tcs: + continue + new_tcs = [] + for tc in tcs: + if isinstance(tc, dict) and "function" in tc: + try: + args_obj = json.loads(tc["function"]["arguments"]) + tc = {**tc, "function": { + **tc["function"], + "arguments": json.dumps( + args_obj, separators=(",", ":"), + sort_keys=True, + ), + }} + except Exception: + tc["function"]["arguments"] = _repair_tool_call_arguments( + tc["function"]["arguments"], + tc["function"].get("name", "?"), + ) + new_tcs.append(tc) + am["tool_calls"] = new_tcs + + # Proactively strip any surrogate characters before the API call. + # Models served via Ollama (Kimi K2.5, GLM-5, Qwen) can return + # lone surrogates (U+D800-U+DFFF) that crash json.dumps() inside + # the OpenAI SDK. Sanitizing here prevents the 3-retry cycle. + _sanitize_messages_surrogates(api_messages) + + # Calculate approximate request size for logging + total_chars = sum(len(str(msg)) for msg in api_messages) + approx_tokens = estimate_messages_tokens_rough(api_messages) + + # Thinking spinner for quiet mode (animated during API call) + thinking_spinner = None + + if not agent.quiet_mode: + agent._vprint(f"\n{agent.log_prefix}🔄 Making API call #{api_call_count}/{agent.max_iterations}...") + agent._vprint(f"{agent.log_prefix} 📊 Request size: {len(api_messages)} messages, ~{approx_tokens:,} tokens (~{total_chars:,} chars)") + agent._vprint(f"{agent.log_prefix} 🔧 Available tools: {len(agent.tools) if agent.tools else 0}") + else: + # Animated thinking spinner in quiet mode + face = random.choice(KawaiiSpinner.get_thinking_faces()) + verb = random.choice(KawaiiSpinner.get_thinking_verbs()) + if agent.thinking_callback: + # CLI TUI mode: use prompt_toolkit widget instead of raw spinner + # (works in both streaming and non-streaming modes) + agent.thinking_callback(f"{face} {verb}...") + elif not agent._has_stream_consumers() and agent._should_start_quiet_spinner(): + # Raw KawaiiSpinner only when no streaming consumers and the + # spinner output has a safe sink. + spinner_type = random.choice(['brain', 'sparkle', 'pulse', 'moon', 'star']) + thinking_spinner = KawaiiSpinner(f"{face} {verb}...", spinner_type=spinner_type, print_fn=agent._print_fn) + thinking_spinner.start() + + # Log request details if verbose + if agent.verbose_logging: + logging.debug(f"API Request - Model: {agent.model}, Messages: {len(messages)}, Tools: {len(agent.tools) if agent.tools else 0}") + logging.debug(f"Last message role: {messages[-1]['role'] if messages else 'none'}") + logging.debug(f"Total message size: ~{approx_tokens:,} tokens") + + api_start_time = time.time() + retry_count = 0 + max_retries = agent._api_max_retries + primary_recovery_attempted = False + max_compression_attempts = 3 + codex_auth_retry_attempted=False + anthropic_auth_retry_attempted=False + nous_auth_retry_attempted=False + copilot_auth_retry_attempted=False + thinking_sig_retry_attempted = False + image_shrink_retry_attempted = False + oauth_1m_beta_retry_attempted = False + llama_cpp_grammar_retry_attempted = False + has_retried_429 = False + restart_with_compressed_messages = False + restart_with_length_continuation = False + + finish_reason = "stop" + response = None # Guard against UnboundLocalError if all retries fail + api_kwargs = None # Guard against UnboundLocalError in except handler + + while retry_count < max_retries: + # ── Nous Portal rate limit guard ────────────────────── + # If another session already recorded that Nous is rate- + # limited, skip the API call entirely. Each attempt + # (including SDK-level retries) counts against RPH and + # deepens the rate limit hole. + if agent.provider == "nous": + try: + from agent.nous_rate_guard import ( + nous_rate_limit_remaining, + format_remaining as _fmt_nous_remaining, + ) + _nous_remaining = nous_rate_limit_remaining() + if _nous_remaining is not None and _nous_remaining > 0: + _nous_msg = ( + f"Nous Portal rate limit active — " + f"resets in {_fmt_nous_remaining(_nous_remaining)}." + ) + agent._vprint( + f"{agent.log_prefix}⏳ {_nous_msg} Trying fallback...", + force=True, + ) + agent._emit_status(f"⏳ {_nous_msg}") + if agent._try_activate_fallback(): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + # No fallback available — return with clear message + agent._persist_session(messages, conversation_history) + return { + "final_response": ( + f"⏳ {_nous_msg}\n\n" + "No fallback provider available. " + "Try again after the reset, or add a " + "fallback provider in config.yaml." + ), + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "failed": True, + "error": _nous_msg, + } + except ImportError: + pass + except Exception: + pass # Never let rate guard break the agent loop + + try: + agent._reset_stream_delivery_tracking() + api_kwargs = agent._build_api_kwargs(api_messages) + if agent._force_ascii_payload: + _sanitize_structure_non_ascii(api_kwargs) + if agent.api_mode == "codex_responses": + api_kwargs = agent._get_transport().preflight_kwargs(api_kwargs, allow_stream=False) + + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _invoke_hook( + "pre_api_request", + task_id=effective_task_id, + session_id=agent.session_id or "", + platform=agent.platform or "", + model=agent.model, + provider=agent.provider, + base_url=agent.base_url, + api_mode=agent.api_mode, + api_call_count=api_call_count, + message_count=len(api_messages), + tool_count=len(agent.tools or []), + approx_input_tokens=approx_tokens, + request_char_count=total_chars, + max_tokens=agent.max_tokens, + ) + except Exception: + pass + + if env_var_enabled("HERMES_DUMP_REQUESTS"): + agent._dump_api_request_debug(api_kwargs, reason="preflight") + + # Always prefer the streaming path — even without stream + # consumers. Streaming gives us fine-grained health + # checking (90s stale-stream detection, 60s read timeout) + # that the non-streaming path lacks. Without this, + # subagents and other quiet-mode callers can hang + # indefinitely when the provider keeps the connection + # alive with SSE pings but never delivers a response. + # The streaming path is a no-op for callbacks when no + # consumers are registered, and falls back to non- + # streaming automatically if the provider doesn't + # support it. + def _stop_spinner(): + nonlocal thinking_spinner + if thinking_spinner: + thinking_spinner.stop("") + thinking_spinner = None + if agent.thinking_callback: + agent.thinking_callback("") + + _use_streaming = True + # Provider signaled "stream not supported" on a previous + # attempt — switch to non-streaming for the rest of this + # session instead of re-failing every retry. + if getattr(agent, "_disable_streaming", False): + _use_streaming = False + # CopilotACPClient communicates via subprocess stdio and + # returns a plain SimpleNamespace — not an iterable + # stream. Mirror the ACP exclusion used for Responses + # API upgrade (lines ~1083-1085). + elif ( + agent.provider == "copilot-acp" + or str(agent.base_url or "").lower().startswith("acp://copilot") + or str(agent.base_url or "").lower().startswith("acp+tcp://") + ): + _use_streaming = False + elif not agent._has_stream_consumers(): + # No display/TTS consumer. Still prefer streaming for + # health checking, but skip for Mock clients in tests + # (mocks return SimpleNamespace, not stream iterators). + from unittest.mock import Mock + if isinstance(getattr(agent, "client", None), Mock): + _use_streaming = False + + if _use_streaming: + response = agent._interruptible_streaming_api_call( + api_kwargs, on_first_delta=_stop_spinner + ) + else: + response = agent._interruptible_api_call(api_kwargs) + + api_duration = time.time() - api_start_time + + # Stop thinking spinner silently -- the response box or tool + # execution messages that follow are more informative. + if thinking_spinner: + thinking_spinner.stop("") + thinking_spinner = None + if agent.thinking_callback: + agent.thinking_callback("") + + if not agent.quiet_mode: + agent._vprint(f"{agent.log_prefix}⏱️ API call completed in {api_duration:.2f}s") + + if agent.verbose_logging: + # Log response with provider info if available + resp_model = getattr(response, 'model', 'N/A') if response else 'N/A' + logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}") + + # Validate response shape before proceeding + response_invalid = False + error_details = [] + if agent.api_mode == "codex_responses": + _ct_v = agent._get_transport() + if not _ct_v.validate_response(response): + if response is None: + response_invalid = True + error_details.append("response is None") + else: + # Provider returned a terminal failure (e.g. quota exhaustion). + # Treat as invalid so the fallback chain is triggered instead of + # letting the error bubble up outside the retry/fallback loop. + _codex_resp_status = str(getattr(response, "status", "") or "").strip().lower() + if _codex_resp_status in {"failed", "cancelled"}: + _codex_error_obj = getattr(response, "error", None) + _codex_error_msg = ( + _codex_error_obj.get("message") if isinstance(_codex_error_obj, dict) + else str(_codex_error_obj) if _codex_error_obj + else f"Responses API returned status '{_codex_resp_status}'" + ) + logging.warning( + "Codex response status='%s' (error=%s). Routing to fallback. %s", + _codex_resp_status, _codex_error_msg, + agent._client_log_context(), + ) + response_invalid = True + error_details.append(f"response.status={_codex_resp_status}: {_codex_error_msg}") + else: + # output_text fallback: stream backfill may have failed + # but normalize can still recover from output_text + _out_text = getattr(response, "output_text", None) + _out_text_stripped = _out_text.strip() if isinstance(_out_text, str) else "" + if _out_text_stripped: + logger.debug( + "Codex response.output is empty but output_text is present " + "(%d chars); deferring to normalization.", + len(_out_text_stripped), + ) + else: + _resp_status = getattr(response, "status", None) + _resp_incomplete = getattr(response, "incomplete_details", None) + logger.warning( + "Codex response.output is empty after stream backfill " + "(status=%s, incomplete_details=%s, model=%s). %s", + _resp_status, _resp_incomplete, + getattr(response, "model", None), + f"api_mode={agent.api_mode} provider={agent.provider}", + ) + response_invalid = True + error_details.append("response.output is empty") + elif agent.api_mode == "anthropic_messages": + _tv = agent._get_transport() + if not _tv.validate_response(response): + response_invalid = True + if response is None: + error_details.append("response is None") + else: + error_details.append("response.content invalid (not a non-empty list)") + elif agent.api_mode == "bedrock_converse": + _btv = agent._get_transport() + if not _btv.validate_response(response): + response_invalid = True + if response is None: + error_details.append("response is None") + else: + error_details.append("Bedrock response invalid (no output or choices)") + else: + _ctv = agent._get_transport() + if not _ctv.validate_response(response): + response_invalid = True + if response is None: + error_details.append("response is None") + elif not hasattr(response, 'choices'): + error_details.append("response has no 'choices' attribute") + elif response.choices is None: + error_details.append("response.choices is None") + else: + error_details.append("response.choices is empty") + + if response_invalid: + # Stop spinner before printing error messages + if thinking_spinner: + thinking_spinner.stop("(´;ω;`) oops, retrying...") + thinking_spinner = None + if agent.thinking_callback: + agent.thinking_callback("") + + # Invalid response — could be rate limiting, provider timeout, + # upstream server error, or malformed response. + retry_count += 1 + + # Eager fallback: empty/malformed responses are a common + # rate-limit symptom. Switch to fallback immediately + # rather than retrying with extended backoff. + if agent._fallback_index < len(agent._fallback_chain): + agent._emit_status("⚠️ Empty/malformed response — switching to fallback...") + if agent._try_activate_fallback(): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + + # Check for error field in response (some providers include this) + error_msg = "Unknown" + provider_name = "Unknown" + if response and hasattr(response, 'error') and response.error: + error_msg = str(response.error) + # Try to extract provider from error metadata + if hasattr(response.error, 'metadata') and response.error.metadata: + provider_name = response.error.metadata.get('provider_name', 'Unknown') + elif response and hasattr(response, 'message') and response.message: + error_msg = str(response.message) + + # Try to get provider from model field (OpenRouter often returns actual model used) + if provider_name == "Unknown" and response and hasattr(response, 'model') and response.model: + provider_name = f"model={response.model}" + + # Check for x-openrouter-provider or similar metadata + if provider_name == "Unknown" and response: + # Log all response attributes for debugging + resp_attrs = {k: str(v)[:100] for k, v in vars(response).items() if not k.startswith('_')} + if agent.verbose_logging: + logging.debug(f"Response attributes for invalid response: {resp_attrs}") + + # Extract error code from response for contextual diagnostics + _resp_error_code = None + if response and hasattr(response, 'error') and response.error: + _code_raw = getattr(response.error, 'code', None) + if _code_raw is None and isinstance(response.error, dict): + _code_raw = response.error.get('code') + if _code_raw is not None: + try: + _resp_error_code = int(_code_raw) + except (TypeError, ValueError): + pass + + # Build a human-readable failure hint from the error code + # and response time, instead of always assuming rate limiting. + if _resp_error_code == 524: + _failure_hint = f"upstream provider timed out (Cloudflare 524, {api_duration:.0f}s)" + elif _resp_error_code == 504: + _failure_hint = f"upstream gateway timeout (504, {api_duration:.0f}s)" + elif _resp_error_code == 429: + _failure_hint = f"rate limited by upstream provider (429)" + elif _resp_error_code in {500, 502}: + _failure_hint = f"upstream server error ({_resp_error_code}, {api_duration:.0f}s)" + elif _resp_error_code in {503, 529}: + _failure_hint = f"upstream provider overloaded ({_resp_error_code})" + elif _resp_error_code is not None: + _failure_hint = f"upstream error (code {_resp_error_code}, {api_duration:.0f}s)" + elif api_duration < 10: + _failure_hint = f"fast response ({api_duration:.1f}s) — likely rate limited" + elif api_duration > 60: + _failure_hint = f"slow response ({api_duration:.0f}s) — likely upstream timeout" + else: + _failure_hint = f"response time {api_duration:.1f}s" + + agent._vprint(f"{agent.log_prefix}⚠️ Invalid API response (attempt {retry_count}/{max_retries}): {', '.join(error_details)}", force=True) + agent._vprint(f"{agent.log_prefix} 🏢 Provider: {provider_name}", force=True) + cleaned_provider_error = agent._clean_error_message(error_msg) + agent._vprint(f"{agent.log_prefix} 📝 Provider message: {cleaned_provider_error}", force=True) + agent._vprint(f"{agent.log_prefix} ⏱️ {_failure_hint}", force=True) + + if retry_count >= max_retries: + # Try fallback before giving up + agent._emit_status(f"⚠️ Max retries ({max_retries}) for invalid responses — trying fallback...") + if agent._try_activate_fallback(): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + agent._emit_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.") + logging.error(f"{agent.log_prefix}Invalid API response after {max_retries} retries.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": f"Invalid API response after {max_retries} retries: {_failure_hint}", + "failed": True # Mark as failure for filtering + } + + # Backoff before retry — jittered exponential: 5s base, 120s cap + wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0) + agent._vprint(f"{agent.log_prefix}⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...", force=True) + logging.warning(f"Invalid API response (retry {retry_count}/{max_retries}): {', '.join(error_details)} | Provider: {provider_name}") + + # Sleep in small increments to stay responsive to interrupts + sleep_end = time.time() + wait_time + _backoff_touch_counter = 0 + while time.time() < sleep_end: + if agent._interrupt_requested: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) + agent._persist_session(messages, conversation_history) + agent.clear_interrupt() + return { + "final_response": f"Operation interrupted during retry ({_failure_hint}, attempt {retry_count}/{max_retries}).", + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "interrupted": True, + } + time.sleep(0.2) + # Touch activity every ~30s so the gateway's inactivity + # monitor knows we're alive during backoff waits. + _backoff_touch_counter += 1 + if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s + agent._touch_activity( + f"retry backoff ({retry_count}/{max_retries}), " + f"{int(sleep_end - time.time())}s remaining" + ) + continue # Retry the API call + + # Check finish_reason before proceeding + if agent.api_mode == "codex_responses": + status = getattr(response, "status", None) + incomplete_details = getattr(response, "incomplete_details", None) + incomplete_reason = None + if isinstance(incomplete_details, dict): + incomplete_reason = incomplete_details.get("reason") + else: + incomplete_reason = getattr(incomplete_details, "reason", None) + if status == "incomplete" and incomplete_reason in {"max_output_tokens", "length"}: + finish_reason = "length" + else: + finish_reason = "stop" + elif agent.api_mode == "anthropic_messages": + _tfr = agent._get_transport() + finish_reason = _tfr.map_finish_reason(response.stop_reason) + elif agent.api_mode == "bedrock_converse": + # Bedrock response already normalized at dispatch — use transport + _bt_fr = agent._get_transport() + _bedrock_result = _bt_fr.normalize_response(response) + finish_reason = _bedrock_result.finish_reason + else: + _cc_fr = agent._get_transport() + _finish_result = _cc_fr.normalize_response(response) + finish_reason = _finish_result.finish_reason + assistant_message = _finish_result + if agent._should_treat_stop_as_truncated( + finish_reason, + assistant_message, + messages, + ): + agent._vprint( + f"{agent.log_prefix}⚠️ Treating suspicious Ollama/GLM stop response as truncated", + force=True, + ) + finish_reason = "length" + + if finish_reason == "length": + agent._vprint(f"{agent.log_prefix}⚠️ Response truncated (finish_reason='length') - model hit max output tokens", force=True) + + # Normalize the truncated response to a single OpenAI-style + # message shape so text-continuation and tool-call retry + # work uniformly across chat_completions, bedrock_converse, + # and anthropic_messages. For Anthropic we use the same + # adapter the agent loop already relies on so the rebuilt + # interim assistant message is byte-identical to what + # would have been appended in the non-truncated path. + _trunc_msg = None + _trunc_transport = agent._get_transport() + if agent.api_mode == "anthropic_messages": + _trunc_result = _trunc_transport.normalize_response( + response, strip_tool_prefix=agent._is_anthropic_oauth + ) + else: + _trunc_result = _trunc_transport.normalize_response(response) + _trunc_msg = _trunc_result + + _trunc_content = getattr(_trunc_msg, "content", None) if _trunc_msg else None + _trunc_has_tool_calls = bool(getattr(_trunc_msg, "tool_calls", None)) if _trunc_msg else False + + # ── Detect thinking-budget exhaustion ────────────── + # When the model spends ALL output tokens on reasoning + # and has none left for the response, continuation + # retries are pointless. Detect this early and give a + # targeted error instead of wasting 3 API calls. + # A response is "thinking exhausted" only when the model + # actually produced reasoning blocks but no visible text after + # them. Models that do not use tags (e.g. GLM-4.7 on + # NVIDIA Build, minimax) may return content=None or an empty + # string for unrelated reasons — treat those as normal + # truncations that deserve continuation retries, not as + # thinking-budget exhaustion. + _has_think_tags = bool( + _trunc_content and re.search( + r'<(?:think|thinking|reasoning|REASONING_SCRATCHPAD)[^>]*>', + _trunc_content, + re.IGNORECASE, + ) + ) + _thinking_exhausted = ( + not _trunc_has_tool_calls + and _has_think_tags + and ( + (_trunc_content is not None and not agent._has_content_after_think_block(_trunc_content)) + or _trunc_content is None + ) + ) + + if _thinking_exhausted: + _exhaust_error = ( + "Model used all output tokens on reasoning with none left " + "for the response. Try lowering reasoning effort or " + "increasing max_tokens." + ) + agent._vprint( + f"{agent.log_prefix}💭 Reasoning exhausted the output token budget — " + f"no visible response was produced.", + force=True, + ) + # Return a user-friendly message as the response so + # CLI (response box) and gateway (chat message) both + # display it naturally instead of a suppressed error. + _exhaust_response = ( + "⚠️ **Thinking Budget Exhausted**\n\n" + "The model used all its output tokens on reasoning " + "and had none left for the actual response.\n\n" + "To fix this:\n" + "→ Lower reasoning effort: `/thinkon low` or `/thinkon minimal`\n" + "→ Or switch to a larger/non-reasoning model with `/model`" + ) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": _exhaust_response, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": _exhaust_error, + } + + if agent.api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"}: + assistant_message = _trunc_msg + if assistant_message is not None and not _trunc_has_tool_calls: + length_continue_retries += 1 + interim_msg = agent._build_assistant_message(assistant_message, finish_reason) + messages.append(interim_msg) + if assistant_message.content: + truncated_response_prefix += assistant_message.content + + if length_continue_retries < 3: + agent._vprint( + f"{agent.log_prefix}↻ Requesting continuation " + f"({length_continue_retries}/3)..." + ) + continue_msg = { + "role": "user", + "content": ( + "[System: Your previous response was truncated by the output " + "length limit. Continue exactly where you left off. Do not " + "restart or repeat prior text. Finish the answer directly.]" + ), + } + messages.append(continue_msg) + agent._session_messages = messages + agent._save_session_log(messages) + restart_with_length_continuation = True + break + + partial_response = agent._strip_think_blocks(truncated_response_prefix).strip() + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": partial_response or None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Response remained truncated after 3 continuation attempts", + } + + if agent.api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"}: + assistant_message = _trunc_msg + if assistant_message is not None and _trunc_has_tool_calls: + if truncated_tool_call_retries < 1: + truncated_tool_call_retries += 1 + agent._vprint( + f"{agent.log_prefix}⚠️ Truncated tool call detected — retrying API call...", + force=True, + ) + # Don't append the broken response to messages; + # just re-run the same API call from the current + # message state, giving the model another chance. + continue + agent._vprint( + f"{agent.log_prefix}⚠️ Truncated tool call response detected again — refusing to execute incomplete tool arguments.", + force=True, + ) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Response truncated due to output length limit", + } + + # If we have prior messages, roll back to last complete state + if len(messages) > 1: + agent._vprint(f"{agent.log_prefix} ⏪ Rolling back to last complete assistant turn") + rolled_back_messages = agent._get_messages_up_to_last_assistant(messages) + + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + + return { + "final_response": None, + "messages": rolled_back_messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Response truncated due to output length limit" + } + else: + # First message was truncated - mark as failed + agent._vprint(f"{agent.log_prefix}❌ First response truncated - cannot recover", force=True) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "failed": True, + "error": "First response truncated due to output length limit" + } + + # Track actual token usage from response for context management + if hasattr(response, 'usage') and response.usage: + canonical_usage = normalize_usage( + response.usage, + provider=agent.provider, + api_mode=agent.api_mode, + ) + prompt_tokens = canonical_usage.prompt_tokens + completion_tokens = canonical_usage.output_tokens + total_tokens = canonical_usage.total_tokens + usage_dict = { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + agent.context_compressor.update_from_response(usage_dict) + + # Cache discovered context length after successful call. + # Only persist limits confirmed by the provider (parsed + # from the error message), not guessed probe tiers. + if getattr(agent.context_compressor, "_context_probed", False): + ctx = agent.context_compressor.context_length + if getattr(agent.context_compressor, "_context_probe_persistable", False): + save_context_length(agent.model, agent.base_url, ctx) + agent._safe_print(f"{agent.log_prefix}💾 Cached context length: {ctx:,} tokens for {agent.model}") + agent.context_compressor._context_probed = False + agent.context_compressor._context_probe_persistable = False + + agent.session_prompt_tokens += prompt_tokens + agent.session_completion_tokens += completion_tokens + agent.session_total_tokens += total_tokens + agent.session_api_calls += 1 + agent.session_input_tokens += canonical_usage.input_tokens + agent.session_output_tokens += canonical_usage.output_tokens + agent.session_cache_read_tokens += canonical_usage.cache_read_tokens + agent.session_cache_write_tokens += canonical_usage.cache_write_tokens + agent.session_reasoning_tokens += canonical_usage.reasoning_tokens + + # Log API call details for debugging/observability + _cache_pct = "" + if canonical_usage.cache_read_tokens and prompt_tokens: + _cache_pct = f" cache={canonical_usage.cache_read_tokens}/{prompt_tokens} ({100*canonical_usage.cache_read_tokens/prompt_tokens:.0f}%)" + logger.info( + "API call #%d: model=%s provider=%s in=%d out=%d total=%d latency=%.1fs%s", + agent.session_api_calls, agent.model, agent.provider or "unknown", + prompt_tokens, completion_tokens, total_tokens, + api_duration, _cache_pct, + ) + + cost_result = estimate_usage_cost( + agent.model, + canonical_usage, + provider=agent.provider, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + ) + if cost_result.amount_usd is not None: + agent.session_estimated_cost_usd += float(cost_result.amount_usd) + agent.session_cost_status = cost_result.status + agent.session_cost_source = cost_result.source + + # Persist token counts to session DB for /insights. + # Do this for every platform with a session_id so non-CLI + # sessions (gateway, cron, delegated runs) cannot lose + # token/accounting data if a higher-level persistence path + # is skipped or fails. Gateway/session-store writes use + # absolute totals, so they safely overwrite these per-call + # deltas instead of double-counting them. + if agent._session_db and agent.session_id: + try: + # Ensure the session row exists before attempting UPDATE. + # Under concurrent load (cron/kanban), the initial + # _ensure_db_session() may have failed due to SQLite + # locking. Retry here so per-call token deltas are + # not silently lost (UPDATE on a non-existent row + # affects 0 rows without error). + if not agent._session_db_created: + agent._ensure_db_session() + agent._session_db.update_token_counts( + agent.session_id, + input_tokens=canonical_usage.input_tokens, + output_tokens=canonical_usage.output_tokens, + cache_read_tokens=canonical_usage.cache_read_tokens, + cache_write_tokens=canonical_usage.cache_write_tokens, + reasoning_tokens=canonical_usage.reasoning_tokens, + estimated_cost_usd=float(cost_result.amount_usd) + if cost_result.amount_usd is not None else None, + cost_status=cost_result.status, + cost_source=cost_result.source, + billing_provider=agent.provider, + billing_base_url=agent.base_url, + billing_mode="subscription_included" + if cost_result.status == "included" else None, + model=agent.model, + api_call_count=1, + ) + except Exception as e: + # Log token persistence failures so they're + # visible in agent.log — silent loss here is + # the root cause of undercounted analytics. + logger.debug( + "Token persistence failed (session=%s, tokens=%d): %s", + agent.session_id, total_tokens, e, + ) + + if agent.verbose_logging: + logging.debug(f"Token usage: prompt={usage_dict['prompt_tokens']:,}, completion={usage_dict['completion_tokens']:,}, total={usage_dict['total_tokens']:,}") + + # Surface cache hit stats for any provider that reports + # them — not just those where we inject cache_control + # markers. OpenAI/Kimi/DeepSeek/Qwen all do automatic + # server-side prefix caching and return + # ``prompt_tokens_details.cached_tokens``; users + # previously could not see their cache % because this + # line was gated on ``_use_prompt_caching``, which is + # only True for Anthropic-style marker injection. + # ``canonical_usage`` is already normalised from all + # three API shapes (Anthropic / Codex / OpenAI-chat) + # so we can rely on its values directly. + cached = canonical_usage.cache_read_tokens + written = canonical_usage.cache_write_tokens + prompt = usage_dict["prompt_tokens"] + if (cached or written) and not agent.quiet_mode: + hit_pct = (cached / prompt * 100) if prompt > 0 else 0 + agent._vprint( + f"{agent.log_prefix} 💾 Cache: " + f"{cached:,}/{prompt:,} tokens " + f"({hit_pct:.0f}% hit, {written:,} written)" + ) + + has_retried_429 = False # Reset on success + # Clear Nous rate limit state on successful request — + # proves the limit has reset and other sessions can + # resume hitting Nous. + if agent.provider == "nous": + try: + from agent.nous_rate_guard import clear_nous_rate_limit + clear_nous_rate_limit() + except Exception: + pass + agent._touch_activity(f"API call #{api_call_count} completed") + break # Success, exit retry loop + + except InterruptedError: + if thinking_spinner: + thinking_spinner.stop("") + thinking_spinner = None + if agent.thinking_callback: + agent.thinking_callback("") + api_elapsed = time.time() - api_start_time + agent._vprint(f"{agent.log_prefix}⚡ Interrupted during API call.", force=True) + agent._persist_session(messages, conversation_history) + interrupted = True + final_response = f"Operation interrupted: waiting for model response ({api_elapsed:.1f}s elapsed)." + break + + except Exception as api_error: + # Stop spinner before printing error messages + if thinking_spinner: + thinking_spinner.stop("(╥_╥) error, retrying...") + thinking_spinner = None + if agent.thinking_callback: + agent.thinking_callback("") + + # ----------------------------------------------------------- + # UnicodeEncodeError recovery. Two common causes: + # 1. Lone surrogates (U+D800..U+DFFF) from clipboard paste + # (Google Docs, rich-text editors) — sanitize and retry. + # 2. ASCII codec on systems with LANG=C or non-UTF-8 locale + # (e.g. Chromebooks) — any non-ASCII character fails. + # Detect via the error message mentioning 'ascii' codec. + # We sanitize messages in-place and may retry twice: + # first to strip surrogates, then once more for pure + # ASCII-only locale sanitization if needed. + # ----------------------------------------------------------- + if isinstance(api_error, UnicodeEncodeError) and getattr(agent, '_unicode_sanitization_passes', 0) < 2: + _err_str = str(api_error).lower() + _is_ascii_codec = "'ascii'" in _err_str or "ascii" in _err_str + # Detect surrogate errors — utf-8 codec refusing to + # encode U+D800..U+DFFF. The error text is: + # "'utf-8' codec can't encode characters in position + # N-M: surrogates not allowed" + _is_surrogate_error = ( + "surrogate" in _err_str + or ("'utf-8'" in _err_str and not _is_ascii_codec) + ) + # Sanitize surrogates from both the canonical `messages` + # list AND `api_messages` (the API-copy, which may carry + # `reasoning_content`/`reasoning_details` transformed + # from `reasoning` — fields the canonical list doesn't + # have directly). Also clean `api_kwargs` if built and + # `prefill_messages` if present. Mirrors the ASCII + # codec recovery below. + _surrogates_found = _sanitize_messages_surrogates(messages) + if isinstance(api_messages, list): + if _sanitize_messages_surrogates(api_messages): + _surrogates_found = True + if isinstance(api_kwargs, dict): + if _sanitize_structure_surrogates(api_kwargs): + _surrogates_found = True + if isinstance(getattr(agent, "prefill_messages", None), list): + if _sanitize_messages_surrogates(agent.prefill_messages): + _surrogates_found = True + # Gate the retry on the error type, not on whether we + # found anything — _force_ascii_payload / the extended + # surrogate walker above cover all known paths, but a + # new transformed field could still slip through. If + # the error was a surrogate encode failure, always let + # the retry run; the proactive sanitizer at line ~8781 + # runs again on the next iteration. Bounded by + # _unicode_sanitization_passes < 2 (outer guard). + if _surrogates_found or _is_surrogate_error: + agent._unicode_sanitization_passes += 1 + if _surrogates_found: + agent._vprint( + f"{agent.log_prefix}⚠️ Stripped invalid surrogate characters from messages. Retrying...", + force=True, + ) + else: + agent._vprint( + f"{agent.log_prefix}⚠️ Surrogate encoding error — retrying after full-payload sanitization...", + force=True, + ) + continue + if _is_ascii_codec: + agent._force_ascii_payload = True + # ASCII codec: the system encoding can't handle + # non-ASCII characters at all. Sanitize all + # non-ASCII content from messages/tool schemas and retry. + # Sanitize both the canonical `messages` list and + # `api_messages` (the API-copy built before the retry + # loop, which may contain extra fields like + # reasoning_content that are not in `messages`). + _messages_sanitized = _sanitize_messages_non_ascii(messages) + if isinstance(api_messages, list): + _sanitize_messages_non_ascii(api_messages) + # Also sanitize the last api_kwargs if already built, + # so a leftover non-ASCII value in a transformed field + # (e.g. extra_body, reasoning_content) doesn't survive + # into the next attempt via _build_api_kwargs cache paths. + if isinstance(api_kwargs, dict): + _sanitize_structure_non_ascii(api_kwargs) + _prefill_sanitized = False + if isinstance(getattr(agent, "prefill_messages", None), list): + _prefill_sanitized = _sanitize_messages_non_ascii(agent.prefill_messages) + + _tools_sanitized = False + if isinstance(getattr(agent, "tools", None), list): + _tools_sanitized = _sanitize_tools_non_ascii(agent.tools) + + _system_sanitized = False + if isinstance(active_system_prompt, str): + _sanitized_system = _strip_non_ascii(active_system_prompt) + if _sanitized_system != active_system_prompt: + active_system_prompt = _sanitized_system + agent._cached_system_prompt = _sanitized_system + _system_sanitized = True + if isinstance(getattr(agent, "ephemeral_system_prompt", None), str): + _sanitized_ephemeral = _strip_non_ascii(agent.ephemeral_system_prompt) + if _sanitized_ephemeral != agent.ephemeral_system_prompt: + agent.ephemeral_system_prompt = _sanitized_ephemeral + _system_sanitized = True + + _headers_sanitized = False + _default_headers = ( + agent._client_kwargs.get("default_headers") + if isinstance(getattr(agent, "_client_kwargs", None), dict) + else None + ) + if isinstance(_default_headers, dict): + _headers_sanitized = _sanitize_structure_non_ascii(_default_headers) + + # Sanitize the API key — non-ASCII characters in + # credentials (e.g. ʋ instead of v from a bad + # copy-paste) cause httpx to fail when encoding + # the Authorization header as ASCII. This is the + # most common cause of persistent UnicodeEncodeError + # that survives message/tool sanitization (#6843). + _credential_sanitized = False + _raw_key = getattr(agent, "api_key", None) or "" + if _raw_key: + _clean_key = _strip_non_ascii(_raw_key) + if _clean_key != _raw_key: + agent.api_key = _clean_key + if isinstance(getattr(agent, "_client_kwargs", None), dict): + agent._client_kwargs["api_key"] = _clean_key + # Also update the live client — it holds its + # own copy of api_key which auth_headers reads + # dynamically on every request. + if getattr(agent, "client", None) is not None and hasattr(agent.client, "api_key"): + agent.client.api_key = _clean_key + _credential_sanitized = True + agent._vprint( + f"{agent.log_prefix}⚠️ API key contained non-ASCII characters " + f"(bad copy-paste?) — stripped them. If auth fails, " + f"re-copy the key from your provider's dashboard.", + force=True, + ) + + # Always retry on ASCII codec detection — + # _force_ascii_payload guarantees the full + # api_kwargs payload is sanitized on the + # next iteration (line ~8475). Even when + # per-component checks above find nothing + # (e.g. non-ASCII only in api_messages' + # reasoning_content), the flag catches it. + # Bounded by _unicode_sanitization_passes < 2. + agent._unicode_sanitization_passes += 1 + _any_sanitized = ( + _messages_sanitized + or _prefill_sanitized + or _tools_sanitized + or _system_sanitized + or _headers_sanitized + or _credential_sanitized + ) + if _any_sanitized: + agent._vprint( + f"{agent.log_prefix}⚠️ System encoding is ASCII — stripped non-ASCII characters from request payload. Retrying...", + force=True, + ) + else: + agent._vprint( + f"{agent.log_prefix}⚠️ System encoding is ASCII — enabling full-payload sanitization for retry...", + force=True, + ) + continue + + # ── Image-rejection recovery ────────────────────────────── + # Some providers (mlx-lm, text-only endpoints, text-only + # fallbacks on multimodal models) reject any message that + # contains image_url content with a 4xx error like + # "Only 'text' content type is supported." On first hit, + # strip all images from the message list, mark the session + # as vision-unsupported, and retry with text only. + # + # Detection is best-effort English phrase matching — a + # locale-translated or heavily-reworded upstream error + # will bypass this guard and fall through to the normal + # error handler. Expand the phrase list when new + # provider wordings are observed in the wild. + _err_body = "" + try: + _err_body = str(getattr(api_error, "body", None) or + getattr(api_error, "message", None) or + str(api_error)) + except Exception: + pass + _err_status = getattr(api_error, "status_code", None) + _IMAGE_REJECTION_PHRASES = ( + "only 'text' content type is supported", + "only text content type is supported", + "image_url is not supported", + "image content is not supported", + "multimodal is not supported", + "multimodal content is not supported", + "multimodal input is not supported", + "vision is not supported", + "vision input is not supported", + "does not support images", + "does not support image input", + "does not support multimodal", + "does not support vision", + "model does not support image", + # ChatGPT-account Codex backend + # (https://chatgpt.com/backend-api/codex) rejects + # data:image/...base64 URLs in input_image fields + # with HTTP 400 "Invalid 'input[N].content[K].image_url'. + # Expected a valid URL, but got a value with an + # invalid format." The OpenAI Responses API on the + # public endpoint accepts data URLs, but the + # ChatGPT-account variant does not. Without this + # phrase the agent cascaded into compression / + # context-too-large recovery instead of just + # stripping the images. Match is narrow on + # purpose — keyed on the field-path apostrophe so + # we don't false-trip on other URL validation + # errors. (issue #23570) + "image_url'. expected", + # DeepSeek's OpenAI-compatible API reports text-only + # request-body variants as: + # "unknown variant `image_url`, expected `text`". + "unknown variant `image_url`, expected `text`", + "unknown variant image_url, expected text", + ) + _err_lower = _err_body.lower() + _looks_like_image_rejection = any( + p in _err_lower for p in _IMAGE_REJECTION_PHRASES + ) + # 4xx-only gate: never interpret 5xx/timeout as "server + # said no to images" — those are transient and must + # route to the normal retry path. + _status_ok = _err_status is None or (400 <= int(_err_status) < 500) + if ( + getattr(agent, "_vision_supported", True) + and _looks_like_image_rejection + and _status_ok + ): + agent._vision_supported = False + _imgs_removed = _strip_images_from_messages(messages) + if isinstance(api_messages, list): + _strip_images_from_messages(api_messages) + agent._vprint( + f"{agent.log_prefix}⚠️ Server rejected image content — " + f"switching to text-only mode for this session" + + (". Stripped images from history and retrying." if _imgs_removed else "."), + force=True, + ) + continue + + status_code = getattr(api_error, "status_code", None) + error_context = agent._extract_api_error_context(api_error) + + # ── Classify the error for structured recovery decisions ── + _compressor = getattr(agent, "context_compressor", None) + _ctx_len = getattr(_compressor, "context_length", 200000) if _compressor else 200000 + classified = classify_api_error( + api_error, + provider=getattr(agent, "provider", "") or "", + model=getattr(agent, "model", "") or "", + approx_tokens=approx_tokens, + context_length=_ctx_len, + num_messages=len(api_messages) if api_messages else 0, + ) + logger.debug( + "Error classified: reason=%s status=%s retryable=%s compress=%s rotate=%s fallback=%s", + classified.reason.value, classified.status_code, + classified.retryable, classified.should_compress, + classified.should_rotate_credential, classified.should_fallback, + ) + + recovered_with_pool, has_retried_429 = agent._recover_with_credential_pool( + status_code=status_code, + has_retried_429=has_retried_429, + classified_reason=classified.reason, + error_context=error_context, + ) + if recovered_with_pool: + continue + + # Image-too-large recovery: shrink oversized native image + # parts in-place and retry once. Triggered by Anthropic's + # per-image 5 MB ceiling (400 with "image exceeds 5 MB + # maximum") or any other provider that complains about + # image size. If shrink fails or a second attempt still + # fails, fall through to normal error handling. + if ( + classified.reason == FailoverReason.image_too_large + and not image_shrink_retry_attempted + ): + image_shrink_retry_attempted = True + if agent._try_shrink_image_parts_in_messages(api_messages): + agent._vprint( + f"{agent.log_prefix}📐 Image(s) exceeded provider size limit — " + f"shrank and retrying...", + force=True, + ) + continue + else: + logger.info( + "image-shrink recovery: no data-URL image parts found " + "or shrink didn't reduce size; surfacing original error." + ) + + # Anthropic OAuth subscription rejected the 1M-context beta + # header ("long context beta is not yet available for this + # subscription"). Disable the beta for the rest of this + # session, rebuild the client, and retry once. 1M-capable + # subscriptions never hit this branch — they accept the + # beta and keep full 1M context. See PR #17680 for the + # original report (we chose reactive recovery over the + # proposed unconditional omit so capable subscriptions + # don't silently lose the capability). + if ( + classified.reason == FailoverReason.oauth_long_context_beta_forbidden + and agent.api_mode == "anthropic_messages" + and agent._is_anthropic_oauth + and not oauth_1m_beta_retry_attempted + ): + oauth_1m_beta_retry_attempted = True + if not getattr(agent, "_oauth_1m_beta_disabled", False): + agent._oauth_1m_beta_disabled = True + try: + agent._anthropic_client.close() + except Exception: + pass + agent._rebuild_anthropic_client() + agent._vprint( + f"{agent.log_prefix}🔕 OAuth subscription doesn't support " + f"the 1M-context beta — disabled for this session and retrying...", + force=True, + ) + continue + + if ( + agent.api_mode == "codex_responses" + and agent.provider == "openai-codex" + and status_code == 401 + and not codex_auth_retry_attempted + ): + codex_auth_retry_attempted = True + if agent._try_refresh_codex_client_credentials(force=True): + agent._vprint(f"{agent.log_prefix}🔐 Codex auth refreshed after 401. Retrying request...") + continue + if ( + agent.api_mode == "chat_completions" + and agent.provider == "nous" + and status_code == 401 + and not nous_auth_retry_attempted + ): + nous_auth_retry_attempted = True + if agent._try_refresh_nous_client_credentials(force=True): + print(f"{agent.log_prefix}🔐 Nous agent key refreshed after 401. Retrying request...") + continue + # Credential refresh didn't help — show diagnostic info. + # Most common causes: Portal OAuth expired/revoked, + # account out of credits, or agent key blocked. + from hermes_constants import display_hermes_home as _dhh_fn + _dhh = _dhh_fn() + _body_text = "" + try: + _body = getattr(api_error, "body", None) or getattr(api_error, "response", None) + if _body is not None: + _body_text = str(_body)[:200] + except Exception: + pass + print(f"{agent.log_prefix}🔐 Nous 401 — Portal authentication failed.") + if _body_text: + print(f"{agent.log_prefix} Response: {_body_text}") + print(f"{agent.log_prefix} Most likely: Portal OAuth expired, account out of credits, or agent key revoked.") + print(f"{agent.log_prefix} Troubleshooting:") + print(f"{agent.log_prefix} • Re-authenticate: hermes login --provider nous") + print(f"{agent.log_prefix} • Check credits / billing: https://portal.nousresearch.com") + print(f"{agent.log_prefix} • Verify stored credentials: {_dhh}/auth.json") + print(f"{agent.log_prefix} • Switch providers temporarily: /model --provider openrouter") + if ( + agent.provider == "copilot" + and status_code == 401 + and not copilot_auth_retry_attempted + ): + copilot_auth_retry_attempted = True + if agent._try_refresh_copilot_client_credentials(): + agent._vprint(f"{agent.log_prefix}🔐 Copilot credentials refreshed after 401. Retrying request...") + continue + if ( + agent.api_mode == "anthropic_messages" + and status_code == 401 + and hasattr(agent, '_anthropic_api_key') + and not anthropic_auth_retry_attempted + ): + anthropic_auth_retry_attempted = True + from agent.anthropic_adapter import _is_oauth_token + if agent._try_refresh_anthropic_client_credentials(): + print(f"{agent.log_prefix}🔐 Anthropic credentials refreshed after 401. Retrying request...") + continue + # Credential refresh didn't help — show diagnostic info + key = agent._anthropic_api_key + auth_method = "Bearer (OAuth/setup-token)" if _is_oauth_token(key) else "x-api-key (API key)" + print(f"{agent.log_prefix}🔐 Anthropic 401 — authentication failed.") + print(f"{agent.log_prefix} Auth method: {auth_method}") + print(f"{agent.log_prefix} Token prefix: {key[:12]}..." if key and len(key) > 12 else f"{agent.log_prefix} Token: (empty or short)") + print(f"{agent.log_prefix} Troubleshooting:") + from hermes_constants import display_hermes_home as _dhh_fn + _dhh = _dhh_fn() + print(f"{agent.log_prefix} • Check ANTHROPIC_TOKEN in {_dhh}/.env for Hermes-managed OAuth/setup tokens") + print(f"{agent.log_prefix} • Check ANTHROPIC_API_KEY in {_dhh}/.env for API keys or legacy token values") + print(f"{agent.log_prefix} • For API keys: verify at https://platform.claude.com/settings/keys") + print(f"{agent.log_prefix} • For Claude Code: run 'claude /login' to refresh, then retry") + print(f"{agent.log_prefix} • Legacy cleanup: hermes config set ANTHROPIC_TOKEN \"\"") + print(f"{agent.log_prefix} • Clear stale keys: hermes config set ANTHROPIC_API_KEY \"\"") + + # ── Thinking block signature recovery ───────────────── + # Anthropic signs thinking blocks against the full turn + # content. Any upstream mutation (context compression, + # session truncation, message merging) invalidates the + # signature → HTTP 400. Recovery: strip reasoning_details + # from all messages so the next retry sends no thinking + # blocks at all. One-shot — don't retry infinitely. + if ( + classified.reason == FailoverReason.thinking_signature + and not thinking_sig_retry_attempted + ): + thinking_sig_retry_attempted = True + for _m in messages: + if isinstance(_m, dict): + _m.pop("reasoning_details", None) + agent._vprint( + f"{agent.log_prefix}⚠️ Thinking block signature invalid — " + f"stripped all thinking blocks, retrying...", + force=True, + ) + logging.warning( + "%sThinking block signature recovery: stripped " + "reasoning_details from %d messages", + agent.log_prefix, len(messages), + ) + continue + + # ── llama.cpp grammar-parse recovery ────────────────── + # llama.cpp's ``json-schema-to-grammar`` converter rejects + # regex escape classes (``\d``, ``\w``, ``\s``) and most + # ``format`` values in tool schemas. MCP servers emit + # these routinely for date/phone/email params. Recovery: + # strip ``pattern``/``format`` from ``agent.tools`` and + # retry once. We keep the keywords by default so cloud + # providers get the full prompting hints; this branch + # fires only for users on llama.cpp's OAI server. + if ( + classified.reason == FailoverReason.llama_cpp_grammar_pattern + and not llama_cpp_grammar_retry_attempted + ): + llama_cpp_grammar_retry_attempted = True + try: + from tools.schema_sanitizer import strip_pattern_and_format + _, _stripped = strip_pattern_and_format(agent.tools) + except Exception as _strip_exc: # pragma: no cover — defensive + logging.warning( + "%sllama.cpp grammar recovery: strip helper failed: %s", + agent.log_prefix, _strip_exc, + ) + _stripped = 0 + if _stripped: + agent._vprint( + f"{agent.log_prefix}⚠️ llama.cpp rejected tool schema grammar — " + f"stripped {_stripped} pattern/format keyword(s), retrying...", + force=True, + ) + logging.warning( + "%sllama.cpp grammar recovery: stripped %d " + "pattern/format keyword(s) from tool schemas", + agent.log_prefix, _stripped, + ) + continue + # No keywords found to strip — fall through to normal + # retry path rather than loop forever on the same error. + logging.warning( + "%sllama.cpp grammar error but no pattern/format " + "keywords to strip — falling through to normal retry", + agent.log_prefix, + ) + + retry_count += 1 + elapsed_time = time.time() - api_start_time + agent._touch_activity( + f"API error recovery (attempt {retry_count}/{max_retries})" + ) + + error_type = type(api_error).__name__ + error_msg = str(api_error).lower() + _error_summary = agent._summarize_api_error(api_error) + logger.warning( + "API call failed (attempt %s/%s) error_type=%s %s summary=%s", + retry_count, + max_retries, + error_type, + agent._client_log_context(), + _error_summary, + ) + + _provider = getattr(agent, "provider", "unknown") + _base = getattr(agent, "base_url", "unknown") + _model = getattr(agent, "model", "unknown") + _status_code_str = f" [HTTP {status_code}]" if status_code else "" + agent._vprint(f"{agent.log_prefix}⚠️ API call failed (attempt {retry_count}/{max_retries}): {error_type}{_status_code_str}", force=True) + agent._vprint(f"{agent.log_prefix} 🔌 Provider: {_provider} Model: {_model}", force=True) + agent._vprint(f"{agent.log_prefix} 🌐 Endpoint: {_base}", force=True) + agent._vprint(f"{agent.log_prefix} 📝 Error: {_error_summary}", force=True) + if status_code and status_code < 500: + _err_body = getattr(api_error, "body", None) + _err_body_str = str(_err_body)[:300] if _err_body else None + if _err_body_str: + agent._vprint(f"{agent.log_prefix} 📋 Details: {_err_body_str}", force=True) + agent._vprint(f"{agent.log_prefix} ⏱️ Elapsed: {elapsed_time:.2f}s Context: {len(api_messages)} msgs, ~{approx_tokens:,} tokens") + + # Actionable hint for OpenRouter "no tool endpoints" error. + # This fires regardless of whether fallback succeeds — the + # user needs to know WHY their model failed so they can fix + # their provider routing, not just silently fall back. + if ( + agent._is_openrouter_url() + and "support tool use" in error_msg + ): + agent._vprint( + f"{agent.log_prefix} 💡 No OpenRouter providers for {_model} support tool calling with your current settings.", + force=True, + ) + if agent.providers_allowed: + agent._vprint( + f"{agent.log_prefix} Your provider_routing.only restriction is filtering out tool-capable providers.", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} Try removing the restriction or adding providers that support tools for this model.", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} Check which providers support tools: https://openrouter.ai/models/{_model}", + force=True, + ) + + # Check for interrupt before deciding to retry + if agent._interrupt_requested: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during error handling, aborting retries.", force=True) + agent._persist_session(messages, conversation_history) + agent.clear_interrupt() + return { + "final_response": f"Operation interrupted: handling API error ({error_type}: {agent._clean_error_message(str(api_error))}).", + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "interrupted": True, + } + + # Check for 413 payload-too-large BEFORE generic 4xx handler. + # A 413 is a payload-size error — the correct response is to + # compress history and retry, not abort immediately. + status_code = getattr(api_error, "status_code", None) + + # ── Anthropic Sonnet long-context tier gate ─────────── + # Anthropic returns HTTP 429 "Extra usage is required for + # long context requests" when a Claude Max (or similar) + # subscription doesn't include the 1M-context tier. This + # is NOT a transient rate limit — retrying or switching + # credentials won't help. Reduce context to 200k (the + # standard tier) and compress. + if classified.reason == FailoverReason.long_context_tier: + _reduced_ctx = 200000 + compressor = agent.context_compressor + old_ctx = compressor.context_length + if old_ctx > _reduced_ctx: + compressor.update_model( + model=agent.model, + context_length=_reduced_ctx, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + ) + # Context probing flags — only set on built-in + # compressor (plugin engines manage their own). + if hasattr(compressor, "_context_probed"): + compressor._context_probed = True + # Don't persist — this is a subscription-tier + # limitation, not a model capability. If the + # user later enables extra usage the 1M limit + # should come back automatically. + compressor._context_probe_persistable = False + agent._vprint( + f"{agent.log_prefix}⚠️ Anthropic long-context tier " + f"requires extra usage — reducing context: " + f"{old_ctx:,} → {_reduced_ctx:,} tokens", + force=True, + ) + + compression_attempts += 1 + if compression_attempts <= max_compression_attempts: + original_len = len(messages) + messages, active_system_prompt = agent._compress_context( + messages, system_message, + approx_tokens=approx_tokens, + task_id=effective_task_id, + ) + # Compression created a new session — clear history + # so _flush_messages_to_session_db writes compressed + # messages to the new session, not skipping them. + conversation_history = None + if len(messages) < original_len or old_ctx > _reduced_ctx: + agent._emit_status( + f"🗜️ Context reduced to {_reduced_ctx:,} tokens " + f"(was {old_ctx:,}), retrying..." + ) + time.sleep(2) + restart_with_compressed_messages = True + break + # Fall through to normal error handling if compression + # is exhausted or didn't help. + + # Eager fallback for rate-limit errors (429 or quota exhaustion). + # When a fallback model is configured, switch immediately instead + # of burning through retries with exponential backoff -- the + # primary provider won't recover within the retry window. + is_rate_limited = classified.reason in { + FailoverReason.rate_limit, + FailoverReason.billing, + } + if is_rate_limited and agent._fallback_index < len(agent._fallback_chain): + # Don't eagerly fallback if credential pool rotation may + # still recover. See _pool_may_recover_from_rate_limit + # for the single-credential-pool and CloudCode-quota + # exceptions. Fixes #11314 and #13636. + pool_may_recover = _pool_may_recover_from_rate_limit( + agent._credential_pool, + provider=agent.provider, + base_url=getattr(agent, "base_url", None), + ) + if not pool_may_recover: + agent._emit_status("⚠️ Rate limited — switching to fallback provider...") + if agent._try_activate_fallback(reason=classified.reason): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + + # ── Nous Portal: record rate limit & skip retries ───── + # When Nous returns a 429 that is a genuine account- + # level rate limit, record the reset time to a shared + # file so ALL sessions (cron, gateway, auxiliary) know + # not to pile on, then skip further retries -- each + # one burns another RPH request and deepens the hole. + # The retry loop's top-of-iteration guard will catch + # this on the next pass and try fallback or bail. + # + # IMPORTANT: Nous Portal multiplexes multiple upstream + # providers (DeepSeek, Kimi, MiMo, Hermes). A 429 can + # also mean an UPSTREAM provider is out of capacity + # for one specific model -- transient, clears in + # seconds, nothing to do with the caller's quota. + # Tripping the cross-session breaker on that would + # block every Nous model for minutes. We use + # ``is_genuine_nous_rate_limit`` to tell the two + # apart via the 429's own x-ratelimit-* headers and + # the last-known-good state captured on the previous + # successful response. + if ( + is_rate_limited + and agent.provider == "nous" + and classified.reason == FailoverReason.rate_limit + and not recovered_with_pool + ): + _genuine_nous_rate_limit = False + try: + from agent.nous_rate_guard import ( + is_genuine_nous_rate_limit, + record_nous_rate_limit, + ) + _err_resp = getattr(api_error, "response", None) + _err_hdrs = ( + getattr(_err_resp, "headers", None) + if _err_resp else None + ) + _genuine_nous_rate_limit = is_genuine_nous_rate_limit( + headers=_err_hdrs, + last_known_state=agent._rate_limit_state, + ) + if _genuine_nous_rate_limit: + record_nous_rate_limit( + headers=_err_hdrs, + error_context=error_context, + ) + else: + logging.info( + "Nous 429 looks like upstream capacity " + "(no exhausted bucket in headers or " + "last-known state) -- not tripping " + "cross-session breaker." + ) + except Exception: + pass + if _genuine_nous_rate_limit: + # Skip straight to max_retries -- the + # top-of-loop guard will handle fallback or + # bail cleanly. + retry_count = max_retries + continue + # Upstream capacity 429: fall through to normal + # retry logic. A different model (or the same + # model a moment later) will typically succeed. + + is_payload_too_large = ( + classified.reason == FailoverReason.payload_too_large + ) + + if is_payload_too_large: + compression_attempts += 1 + if compression_attempts > max_compression_attempts: + agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached for payload-too-large error.", force=True) + agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) + logging.error(f"{agent.log_prefix}413 compression failed after {max_compression_attempts} attempts.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": f"Request payload too large: max compression attempts ({max_compression_attempts}) reached.", + "partial": True, + "failed": True, + "compression_exhausted": True, + } + agent._emit_status(f"⚠️ Request payload too large (413) — compression attempt {compression_attempts}/{max_compression_attempts}...") + + original_len = len(messages) + messages, active_system_prompt = agent._compress_context( + messages, system_message, approx_tokens=approx_tokens, + task_id=effective_task_id, + ) + # Compression created a new session — clear history + # so _flush_messages_to_session_db writes compressed + # messages to the new session, not skipping them. + conversation_history = None + + if len(messages) < original_len: + agent._emit_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") + time.sleep(2) # Brief pause between compression retries + restart_with_compressed_messages = True + break + else: + agent._vprint(f"{agent.log_prefix}❌ Payload too large and cannot compress further.", force=True) + agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) + logging.error(f"{agent.log_prefix}413 payload too large. Cannot compress further.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": "Request payload too large (413). Cannot compress further.", + "partial": True, + "failed": True, + "compression_exhausted": True, + } + + # Check for context-length errors BEFORE generic 4xx handler. + # The classifier detects context overflow from: explicit error + # messages, generic 400 + large session heuristic (#1630), and + # server disconnect + large session pattern (#2153). + is_context_length_error = ( + classified.reason == FailoverReason.context_overflow + ) + + if is_context_length_error: + compressor = agent.context_compressor + old_ctx = compressor.context_length + + # ── Distinguish two very different errors ─────────── + # 1. "Prompt too long": the INPUT exceeds the context window. + # Fix: reduce context_length + compress history. + # 2. "max_tokens too large": input is fine, but + # input_tokens + requested max_tokens > context_window. + # Fix: reduce max_tokens (the OUTPUT cap) for this call. + # Do NOT shrink context_length — the window is unchanged. + # + # Note: max_tokens = output token cap (one response). + # context_length = total window (input + output combined). + available_out = parse_available_output_tokens_from_error(error_msg) + if available_out is not None: + # Error is purely about the output cap being too large. + # Cap output to the available space and retry without + # touching context_length or triggering compression. + safe_out = max(1, available_out - 64) # small safety margin + agent._ephemeral_max_output_tokens = safe_out + agent._vprint( + f"{agent.log_prefix}⚠️ Output cap too large for current prompt — " + f"retrying with max_tokens={safe_out:,} " + f"(available_tokens={available_out:,}; context_length unchanged at {old_ctx:,})", + force=True, + ) + # Still count against compression_attempts so we don't + # loop forever if the error keeps recurring. + compression_attempts += 1 + if compression_attempts > max_compression_attempts: + agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) + agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) + logging.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached.", + "partial": True, + "failed": True, + "compression_exhausted": True, + } + restart_with_compressed_messages = True + break + + # Error is about the INPUT being too large — reduce context_length. + # Try to parse the actual limit from the error message + parsed_limit = parse_context_limit_from_error(error_msg) + _provider_lower = (getattr(agent, "provider", "") or "").lower() + _base_lower = (getattr(agent, "base_url", "") or "").rstrip("/").lower() + is_minimax_provider = ( + _provider_lower in {"minimax", "minimax-cn"} + or _base_lower.startswith(( + "https://api.minimax.io/anthropic", + "https://api.minimaxi.com/anthropic", + )) + ) + minimax_delta_only_overflow = ( + is_minimax_provider + and parsed_limit is None + and "context window exceeds limit (" in error_msg + ) + if parsed_limit and parsed_limit < old_ctx: + new_ctx = parsed_limit + agent._vprint(f"{agent.log_prefix}Context limit detected from API: {new_ctx:,} tokens (was {old_ctx:,})", force=True) + elif minimax_delta_only_overflow: + new_ctx = old_ctx + agent._vprint( + f"{agent.log_prefix}Provider reported overflow amount only; " + f"keeping context_length at {old_ctx:,} tokens and compressing.", + force=True, + ) + else: + # Step down to the next probe tier + new_ctx = get_next_probe_tier(old_ctx) + + if new_ctx and new_ctx < old_ctx: + compressor.update_model( + model=agent.model, + context_length=new_ctx, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + ) + # Context probing flags — only set on built-in + # compressor (plugin engines manage their own). + if hasattr(compressor, "_context_probed"): + compressor._context_probed = True + # Only persist limits parsed from the provider's + # error message (a real number). Guessed fallback + # tiers from get_next_probe_tier() should stay + # in-memory only — persisting them pollutes the + # cache with wrong values. + compressor._context_probe_persistable = bool( + parsed_limit and parsed_limit == new_ctx + ) + agent._vprint(f"{agent.log_prefix}⚠️ Context length exceeded — stepping down: {old_ctx:,} → {new_ctx:,} tokens", force=True) + else: + agent._vprint(f"{agent.log_prefix}⚠️ Context length exceeded at minimum tier — attempting compression...", force=True) + + compression_attempts += 1 + if compression_attempts > max_compression_attempts: + agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) + agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) + logging.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached.", + "partial": True, + "failed": True, + "compression_exhausted": True, + } + agent._emit_status(f"🗜️ Context too large (~{approx_tokens:,} tokens) — compressing ({compression_attempts}/{max_compression_attempts})...") + + original_len = len(messages) + messages, active_system_prompt = agent._compress_context( + messages, system_message, approx_tokens=approx_tokens, + task_id=effective_task_id, + ) + # Compression created a new session — clear history + # so _flush_messages_to_session_db writes compressed + # messages to the new session, not skipping them. + conversation_history = None + + if len(messages) < original_len or new_ctx and new_ctx < old_ctx: + if len(messages) < original_len: + agent._emit_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") + time.sleep(2) # Brief pause between compression retries + restart_with_compressed_messages = True + break + else: + # Can't compress further and already at minimum tier + agent._vprint(f"{agent.log_prefix}❌ Context length exceeded and cannot compress further.", force=True) + agent._vprint(f"{agent.log_prefix} 💡 The conversation has accumulated too much content. Try /new to start fresh, or /compress to manually trigger compression.", force=True) + logging.error(f"{agent.log_prefix}Context length exceeded: {approx_tokens:,} tokens. Cannot compress further.") + agent._persist_session(messages, conversation_history) + return { + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": f"Context length exceeded ({approx_tokens:,} tokens). Cannot compress further.", + "partial": True, + "failed": True, + "compression_exhausted": True, + } + + # Check for non-retryable client errors. The classifier + # already accounts for 413, 429, 529 (transient), context + # overflow, and generic-400 heuristics. Local validation + # errors (ValueError, TypeError) are programming bugs. + # Exclude UnicodeEncodeError — it's a ValueError subclass + # but is handled separately by the surrogate sanitization + # path above. Exclude json.JSONDecodeError — also a + # ValueError subclass, but it indicates a transient + # provider/network failure (malformed response body, + # truncated stream, routing layer corruption), not a + # local programming bug, and should be retried (#14782). + is_local_validation_error = ( + isinstance(api_error, (ValueError, TypeError)) + and not isinstance( + api_error, (UnicodeEncodeError, json.JSONDecodeError) + ) + # ssl.SSLError (and its subclass SSLCertVerificationError) + # inherits from OSError *and* ValueError via Python MRO, + # so the isinstance(ValueError) check above would + # misclassify a TLS transport failure as a local + # programming bug and abort without retrying. Exclude + # ssl.SSLError explicitly so the error classifier's + # retryable=True mapping takes effect instead. + and not isinstance(api_error, ssl.SSLError) + ) + is_client_error = ( + is_local_validation_error + or ( + not classified.retryable + and not classified.should_compress + and classified.reason not in { + FailoverReason.rate_limit, + FailoverReason.billing, + FailoverReason.overloaded, + FailoverReason.context_overflow, + FailoverReason.payload_too_large, + FailoverReason.long_context_tier, + FailoverReason.thinking_signature, + } + ) + ) and not is_context_length_error + + if is_client_error: + # Try fallback before aborting — a different provider + # may not have the same issue (rate limit, auth, etc.) + agent._emit_status(f"⚠️ Non-retryable error (HTTP {status_code}) — trying fallback...") + if agent._try_activate_fallback(): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + if api_kwargs is not None: + agent._dump_api_request_debug( + api_kwargs, reason="non_retryable_client_error", error=api_error, + ) + agent._emit_status( + f"❌ Non-retryable error (HTTP {status_code}): " + f"{agent._summarize_api_error(api_error)}" + ) + agent._vprint(f"{agent.log_prefix}❌ Non-retryable client error (HTTP {status_code}). Aborting.", force=True) + agent._vprint(f"{agent.log_prefix} 🔌 Provider: {_provider} Model: {_model}", force=True) + agent._vprint(f"{agent.log_prefix} 🌐 Endpoint: {_base}", force=True) + # Actionable guidance for common auth errors + if classified.is_auth or classified.reason == FailoverReason.billing: + if _provider == "openai-codex" and status_code == 401: + agent._vprint(f"{agent.log_prefix} 💡 Codex OAuth token was rejected (HTTP 401). Your token may have been", force=True) + agent._vprint(f"{agent.log_prefix} refreshed by another client (Codex CLI, VS Code). To fix:", force=True) + agent._vprint(f"{agent.log_prefix} 1. Run `codex` in your terminal to generate fresh tokens.", force=True) + agent._vprint(f"{agent.log_prefix} 2. Then run `hermes auth` to re-authenticate.", force=True) + else: + agent._vprint(f"{agent.log_prefix} 💡 Your API key was rejected by the provider. Check:", force=True) + agent._vprint(f"{agent.log_prefix} • Is the key valid? Run: hermes setup", force=True) + agent._vprint(f"{agent.log_prefix} • Does your account have access to {_model}?", force=True) + if base_url_host_matches(str(_base), "openrouter.ai"): + agent._vprint(f"{agent.log_prefix} • Check credits: https://openrouter.ai/settings/credits", force=True) + else: + agent._vprint(f"{agent.log_prefix} 💡 This type of error won't be fixed by retrying.", force=True) + logging.error(f"{agent.log_prefix}Non-retryable client error: {api_error}") + # Skip session persistence when the error is likely + # context-overflow related (status 400 + large session). + # Persisting the failed user message would make the + # session even larger, causing the same failure on the + # next attempt. (#1630) + if status_code == 400 and (approx_tokens > 50000 or len(api_messages) > 80): + agent._vprint( + f"{agent.log_prefix}⚠️ Skipping session persistence " + f"for large failed session to prevent growth loop.", + force=True, + ) + else: + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "failed": True, + "error": str(api_error), + } + + if retry_count >= max_retries: + # Before falling back, try rebuilding the primary + # client once for transient transport errors (stale + # connection pool, TCP reset). Only attempted once + # per API call block. + if not primary_recovery_attempted and agent._try_recover_primary_transport( + api_error, retry_count=retry_count, max_retries=max_retries, + ): + primary_recovery_attempted = True + retry_count = 0 + continue + # Try fallback before giving up entirely + agent._emit_status(f"⚠️ Max retries ({max_retries}) exhausted — trying fallback...") + if agent._try_activate_fallback(): + retry_count = 0 + compression_attempts = 0 + primary_recovery_attempted = False + continue + _final_summary = agent._summarize_api_error(api_error) + if is_rate_limited: + agent._emit_status(f"❌ Rate limited after {max_retries} retries — {_final_summary}") + else: + agent._emit_status(f"❌ API failed after {max_retries} retries — {_final_summary}") + agent._vprint(f"{agent.log_prefix} 💀 Final error: {_final_summary}", force=True) + + # Detect SSE stream-drop pattern (e.g. "Network + # connection lost") and surface actionable guidance. + # This typically happens when the model generates a + # very large tool call (write_file with huge content) + # and the proxy/CDN drops the stream mid-response. + _is_stream_drop = ( + not getattr(api_error, "status_code", None) + and any(p in error_msg for p in ( + "connection lost", "connection reset", + "connection closed", "network connection", + "network error", "terminated", + )) + ) + if _is_stream_drop: + agent._vprint( + f"{agent.log_prefix} 💡 The provider's stream " + f"connection keeps dropping. This often happens " + f"when the model tries to write a very large " + f"file in a single tool call.", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} Try asking the model " + f"to use execute_code with Python's open() for " + f"large files, or to write the file in smaller " + f"sections.", + force=True, + ) + + logging.error( + "%sAPI call failed after %s retries. %s | provider=%s model=%s msgs=%s tokens=~%s", + agent.log_prefix, max_retries, _final_summary, + _provider, _model, len(api_messages), f"{approx_tokens:,}", + ) + if api_kwargs is not None: + agent._dump_api_request_debug( + api_kwargs, reason="max_retries_exhausted", error=api_error, + ) + agent._persist_session(messages, conversation_history) + _final_response = f"API call failed after {max_retries} retries: {_final_summary}" + if _is_stream_drop: + _final_response += ( + "\n\nThe provider's stream connection keeps " + "dropping — this often happens when generating " + "very large tool call responses (e.g. write_file " + "with long content). Try asking me to use " + "execute_code with Python's open() for large " + "files, or to write in smaller sections." + ) + return { + "final_response": _final_response, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "failed": True, + "error": _final_summary, + } + + # For rate limits, respect the Retry-After header if present + _retry_after = None + if is_rate_limited: + _resp_headers = getattr(getattr(api_error, "response", None), "headers", None) + if _resp_headers and hasattr(_resp_headers, "get"): + _ra_raw = _resp_headers.get("retry-after") or _resp_headers.get("Retry-After") + if _ra_raw: + try: + _retry_after = min(float(_ra_raw), 120) # Cap at 2 minutes + except (TypeError, ValueError): + pass + wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0) + if is_rate_limited: + agent._emit_status(f"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries})...") + else: + agent._emit_status(f"⏳ Retrying in {wait_time:.1f}s (attempt {retry_count}/{max_retries})...") + logger.warning( + "Retrying API call in %ss (attempt %s/%s) %s error=%s", + wait_time, + retry_count, + max_retries, + agent._client_log_context(), + api_error, + ) + # Sleep in small increments so we can respond to interrupts quickly + # instead of blocking the entire wait_time in one sleep() call + sleep_end = time.time() + wait_time + _backoff_touch_counter = 0 + while time.time() < sleep_end: + if agent._interrupt_requested: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) + agent._persist_session(messages, conversation_history) + agent.clear_interrupt() + return { + "final_response": f"Operation interrupted: retrying API call after error (retry {retry_count}/{max_retries}).", + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "interrupted": True, + } + time.sleep(0.2) # Check interrupt every 200ms + # Touch activity every ~30s so the gateway's inactivity + # monitor knows we're alive during backoff waits. + _backoff_touch_counter += 1 + if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s + agent._touch_activity( + f"error retry backoff ({retry_count}/{max_retries}), " + f"{int(sleep_end - time.time())}s remaining" + ) + + # If the API call was interrupted, skip response processing + if interrupted: + _turn_exit_reason = "interrupted_during_api_call" + break + + if restart_with_compressed_messages: + api_call_count -= 1 + agent.iteration_budget.refund() + # Count compression restarts toward the retry limit to prevent + # infinite loops when compression reduces messages but not enough + # to fit the context window. + retry_count += 1 + restart_with_compressed_messages = False + continue + + if restart_with_length_continuation: + # Progressively boost the output token budget on each retry. + # Retry 1 → 2× base, retry 2 → 3× base, capped at 32 768. + # Applies to all providers via _ephemeral_max_output_tokens. + _boost_base = agent.max_tokens if agent.max_tokens else 4096 + _boost = _boost_base * (length_continue_retries + 1) + agent._ephemeral_max_output_tokens = min(_boost, 32768) + continue + + # Guard: if all retries exhausted without a successful response + # (e.g. repeated context-length errors that exhausted retry_count), + # the `response` variable is still None. Break out cleanly. + if response is None: + _turn_exit_reason = "all_retries_exhausted_no_response" + print(f"{agent.log_prefix}❌ All API retries exhausted with no successful response.") + agent._persist_session(messages, conversation_history) + break + + try: + _transport = agent._get_transport() + _normalize_kwargs = {} + if agent.api_mode == "anthropic_messages": + _normalize_kwargs["strip_tool_prefix"] = agent._is_anthropic_oauth + normalized = _transport.normalize_response(response, **_normalize_kwargs) + assistant_message = normalized + finish_reason = normalized.finish_reason + + # Normalize content to string — some OpenAI-compatible servers + # (llama-server, etc.) return content as a dict or list instead + # of a plain string, which crashes downstream .strip() calls. + if assistant_message.content is not None and not isinstance(assistant_message.content, str): + raw = assistant_message.content + if isinstance(raw, dict): + assistant_message.content = raw.get("text", "") or raw.get("content", "") or json.dumps(raw) + elif isinstance(raw, list): + # Multimodal content list — extract text parts + parts = [] + for part in raw: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, dict) and part.get("type") == "text": + parts.append(part.get("text", "")) + elif isinstance(part, dict) and "text" in part: + parts.append(str(part["text"])) + assistant_message.content = "\n".join(parts) + else: + assistant_message.content = str(raw) + + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _assistant_tool_calls = getattr(assistant_message, "tool_calls", None) or [] + _assistant_text = assistant_message.content or "" + _invoke_hook( + "post_api_request", + task_id=effective_task_id, + session_id=agent.session_id or "", + platform=agent.platform or "", + model=agent.model, + provider=agent.provider, + base_url=agent.base_url, + api_mode=agent.api_mode, + api_call_count=api_call_count, + api_duration=api_duration, + finish_reason=finish_reason, + message_count=len(api_messages), + response_model=getattr(response, "model", None), + usage=agent._usage_summary_for_api_request_hook(response), + assistant_content_chars=len(_assistant_text), + assistant_tool_call_count=len(_assistant_tool_calls), + ) + except Exception: + pass + + # Handle assistant response + if assistant_message.content and not agent.quiet_mode: + if agent.verbose_logging: + agent._vprint(f"{agent.log_prefix}🤖 Assistant: {assistant_message.content}") + else: + agent._vprint(f"{agent.log_prefix}🤖 Assistant: {assistant_message.content[:100]}{'...' if len(assistant_message.content) > 100 else ''}") + + # Notify progress callback of model's thinking (used by subagent + # delegation to relay the child's reasoning to the parent display). + if (assistant_message.content and agent.tool_progress_callback): + _think_text = assistant_message.content.strip() + # Strip reasoning XML tags that shouldn't leak to parent display + _think_text = re.sub( + r'', '', _think_text + ).strip() + # For subagents: relay first line to parent display (existing behaviour). + # For all agents with a structured callback: emit reasoning.available event. + first_line = _think_text.split('\n')[0][:80] if _think_text else "" + if first_line and getattr(agent, '_delegate_depth', 0) > 0: + try: + agent.tool_progress_callback("_thinking", first_line) + except Exception: + pass + elif _think_text: + try: + agent.tool_progress_callback("reasoning.available", "_thinking", _think_text[:500], None) + except Exception: + pass + + # Check for incomplete (opened but never closed) + # This means the model ran out of output tokens mid-reasoning — retry up to 2 times + if has_incomplete_scratchpad(assistant_message.content or ""): + agent._incomplete_scratchpad_retries += 1 + + agent._vprint(f"{agent.log_prefix}⚠️ Incomplete detected (opened but never closed)") + + if agent._incomplete_scratchpad_retries <= 2: + agent._vprint(f"{agent.log_prefix}🔄 Retrying API call ({agent._incomplete_scratchpad_retries}/2)...") + # Don't add the broken message, just retry + continue + else: + # Max retries - discard this turn and save as partial + agent._vprint(f"{agent.log_prefix}❌ Max retries (2) for incomplete scratchpad. Saving as partial.", force=True) + agent._incomplete_scratchpad_retries = 0 + + rolled_back_messages = agent._get_messages_up_to_last_assistant(messages) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + + return { + "final_response": None, + "messages": rolled_back_messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Incomplete REASONING_SCRATCHPAD after 2 retries" + } + + # Reset incomplete scratchpad counter on clean response + agent._incomplete_scratchpad_retries = 0 + + if agent.api_mode == "codex_responses" and finish_reason == "incomplete": + agent._codex_incomplete_retries += 1 + + interim_msg = agent._build_assistant_message(assistant_message, finish_reason) + interim_has_content = bool((interim_msg.get("content") or "").strip()) + interim_has_reasoning = bool(interim_msg.get("reasoning", "").strip()) if isinstance(interim_msg.get("reasoning"), str) else False + interim_has_codex_reasoning = bool(interim_msg.get("codex_reasoning_items")) + interim_has_codex_message_items = bool(interim_msg.get("codex_message_items")) + + if ( + interim_has_content + or interim_has_reasoning + or interim_has_codex_reasoning + or interim_has_codex_message_items + ): + last_msg = messages[-1] if messages else None + # Duplicate detection: two consecutive incomplete assistant + # messages with identical content AND reasoning are collapsed. + # For provider-state-only changes (encrypted reasoning + # items or replayable message ids/phases/statuses differ + # while visible content/reasoning are unchanged), compare + # those opaque payloads too so we don't silently drop the + # newer continuation state. + last_codex_items = last_msg.get("codex_reasoning_items") if isinstance(last_msg, dict) else None + interim_codex_items = interim_msg.get("codex_reasoning_items") + last_codex_message_items = last_msg.get("codex_message_items") if isinstance(last_msg, dict) else None + interim_codex_message_items = interim_msg.get("codex_message_items") + duplicate_interim = ( + isinstance(last_msg, dict) + and last_msg.get("role") == "assistant" + and last_msg.get("finish_reason") == "incomplete" + and (last_msg.get("content") or "") == (interim_msg.get("content") or "") + and (last_msg.get("reasoning") or "") == (interim_msg.get("reasoning") or "") + and last_codex_items == interim_codex_items + and last_codex_message_items == interim_codex_message_items + ) + if not duplicate_interim: + messages.append(interim_msg) + agent._emit_interim_assistant_message(interim_msg) + + if agent._codex_incomplete_retries < 3: + if not agent.quiet_mode: + agent._vprint(f"{agent.log_prefix}↻ Codex response incomplete; continuing turn ({agent._codex_incomplete_retries}/3)") + agent._session_messages = messages + agent._save_session_log(messages) + continue + + agent._codex_incomplete_retries = 0 + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Codex response remained incomplete after 3 continuation attempts", + } + elif hasattr(agent, "_codex_incomplete_retries"): + agent._codex_incomplete_retries = 0 + + # Check for tool calls + if assistant_message.tool_calls: + if not agent.quiet_mode: + agent._vprint(f"{agent.log_prefix}🔧 Processing {len(assistant_message.tool_calls)} tool call(s)...") + + if agent.verbose_logging: + for tc in assistant_message.tool_calls: + logging.debug(f"Tool call: {tc.function.name} with args: {tc.function.arguments[:200]}...") + + # Validate tool call names - detect model hallucinations + # Repair mismatched tool names before validating + for tc in assistant_message.tool_calls: + if tc.function.name not in agent.valid_tool_names: + repaired = agent._repair_tool_call(tc.function.name) + if repaired: + print(f"{agent.log_prefix}🔧 Auto-repaired tool name: '{tc.function.name}' -> '{repaired}'") + tc.function.name = repaired + invalid_tool_calls = [ + tc.function.name for tc in assistant_message.tool_calls + if tc.function.name not in agent.valid_tool_names + ] + if invalid_tool_calls: + # Track retries for invalid tool calls + agent._invalid_tool_retries += 1 + + # Return helpful error to model — model can agent-correct next turn + available = ", ".join(sorted(agent.valid_tool_names)) + invalid_name = invalid_tool_calls[0] + invalid_preview = invalid_name[:80] + "..." if len(invalid_name) > 80 else invalid_name + agent._vprint(f"{agent.log_prefix}⚠️ Unknown tool '{invalid_preview}' — sending error to model for agent-correction ({agent._invalid_tool_retries}/3)") + + if agent._invalid_tool_retries >= 3: + agent._vprint(f"{agent.log_prefix}❌ Max retries (3) for invalid tool calls exceeded. Stopping as partial.", force=True) + agent._invalid_tool_retries = 0 + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": f"Model generated invalid tool call: {invalid_preview}" + } + + assistant_msg = agent._build_assistant_message(assistant_message, finish_reason) + messages.append(assistant_msg) + for tc in assistant_message.tool_calls: + if tc.function.name not in agent.valid_tool_names: + content = f"Tool '{tc.function.name}' does not exist. Available tools: {available}" + else: + content = "Skipped: another tool call in this turn used an invalid name. Please retry this tool call." + messages.append({ + "role": "tool", + "name": tc.function.name, + "tool_call_id": tc.id, + "content": content, + }) + continue + # Reset retry counter on successful tool call validation + agent._invalid_tool_retries = 0 + + # Validate tool call arguments are valid JSON + # Handle empty strings as empty objects (common model quirk) + invalid_json_args = [] + for tc in assistant_message.tool_calls: + args = tc.function.arguments + if isinstance(args, (dict, list)): + tc.function.arguments = json.dumps(args) + continue + if args is not None and not isinstance(args, str): + tc.function.arguments = str(args) + args = tc.function.arguments + # Treat empty/whitespace strings as empty object + if not args or not args.strip(): + tc.function.arguments = "{}" + continue + try: + json.loads(args) + except json.JSONDecodeError as e: + invalid_json_args.append((tc.function.name, str(e))) + + if invalid_json_args: + # Check if the invalid JSON is due to truncation rather + # than a model formatting mistake. Routers sometimes + # rewrite finish_reason from "length" to "tool_calls", + # hiding the truncation from the length handler above. + # Detect truncation: args that don't end with } or ] + # (after stripping whitespace) are cut off mid-stream. + _truncated = any( + not (tc.function.arguments or "").rstrip().endswith(("}", "]")) + for tc in assistant_message.tool_calls + if tc.function.name in {n for n, _ in invalid_json_args} + ) + if _truncated: + agent._vprint( + f"{agent.log_prefix}⚠️ Truncated tool call arguments detected " + f"(finish_reason={finish_reason!r}) — refusing to execute.", + force=True, + ) + agent._invalid_json_retries = 0 + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "error": "Response truncated due to output length limit", + } + + # Track retries for invalid JSON arguments + agent._invalid_json_retries += 1 + + tool_name, error_msg = invalid_json_args[0] + agent._vprint(f"{agent.log_prefix}⚠️ Invalid JSON in tool call arguments for '{tool_name}': {error_msg}") + + if agent._invalid_json_retries < 3: + agent._vprint(f"{agent.log_prefix}🔄 Retrying API call ({agent._invalid_json_retries}/3)...") + # Don't add anything to messages, just retry the API call + continue + else: + # Instead of returning partial, inject tool error results so the model can recover. + # Using tool results (not user messages) preserves role alternation. + agent._vprint(f"{agent.log_prefix}⚠️ Injecting recovery tool results for invalid JSON...") + agent._invalid_json_retries = 0 # Reset for next attempt + + # Append the assistant message with its (broken) tool_calls + recovery_assistant = agent._build_assistant_message(assistant_message, finish_reason) + messages.append(recovery_assistant) + + # Respond with tool error results for each tool call + invalid_names = {name for name, _ in invalid_json_args} + for tc in assistant_message.tool_calls: + if tc.function.name in invalid_names: + err = next(e for n, e in invalid_json_args if n == tc.function.name) + tool_result = ( + f"Error: Invalid JSON arguments. {err}. " + f"For tools with no required parameters, use an empty object: {{}}. " + f"Please retry with valid JSON." + ) + else: + tool_result = "Skipped: other tool call in this response had invalid JSON." + messages.append({ + "role": "tool", + "name": tc.function.name, + "tool_call_id": tc.id, + "content": tool_result, + }) + continue + + # Reset retry counter on successful JSON validation + agent._invalid_json_retries = 0 + + # ── Post-call guardrails ────────────────────────── + assistant_message.tool_calls = agent._cap_delegate_task_calls( + assistant_message.tool_calls + ) + assistant_message.tool_calls = agent._deduplicate_tool_calls( + assistant_message.tool_calls + ) + + assistant_msg = agent._build_assistant_message(assistant_message, finish_reason) + + # If this turn has both content AND tool_calls, capture the content + # as a fallback final response. Common pattern: model delivers its + # answer and calls memory/skill tools as a side-effect in the same + # turn. If the follow-up turn after tools is empty, we use this. + turn_content = assistant_message.content or "" + if turn_content and agent._has_content_after_think_block(turn_content): + agent._last_content_with_tools = turn_content + # Only mute subsequent output when EVERY tool call in + # this turn is post-response housekeeping (memory, todo, + # skill_manage, etc.). If any substantive tool is present + # (search_files, read_file, write_file, terminal, ...), + # keep output visible so the user sees progress. + _HOUSEKEEPING_TOOLS = frozenset({ + "memory", "todo", "skill_manage", "session_search", + }) + _all_housekeeping = all( + tc.function.name in _HOUSEKEEPING_TOOLS + for tc in assistant_message.tool_calls + ) + agent._last_content_tools_all_housekeeping = _all_housekeeping + if _all_housekeeping and agent._has_stream_consumers(): + agent._mute_post_response = True + elif agent._should_emit_quiet_tool_messages(): + clean = agent._strip_think_blocks(turn_content).strip() + if clean: + agent._vprint(f" ┊ 💬 {clean}") + + # Pop thinking-only prefill message(s) before appending + # (tool-call path — same rationale as the final-response path). + _had_prefill = False + while ( + messages + and isinstance(messages[-1], dict) + and messages[-1].get("_thinking_prefill") + ): + messages.pop() + _had_prefill = True + + # Reset prefill counter when tool calls follow a prefill + # recovery. Without this, the counter accumulates across + # the whole conversation — a model that intermittently + # empties (empty → prefill → tools → empty → prefill → + # tools) burns both prefill attempts and the third empty + # gets zero recovery. Resetting here treats each tool- + # call success as a fresh start. + if _had_prefill: + agent._thinking_prefill_retries = 0 + agent._empty_content_retries = 0 + # Successful tool execution — reset the post-tool nudge + # flag so it can fire again if the model goes empty on + # a LATER tool round. + agent._post_tool_empty_retried = False + + messages.append(assistant_msg) + agent._emit_interim_assistant_message(assistant_msg) + + # Close any open streaming display (response box, reasoning + # box) before tool execution begins. Intermediate turns may + # have streamed early content that opened the response box; + # flushing here prevents it from wrapping tool feed lines. + # Only signal the display callback — TTS (_stream_callback) + # should NOT receive None (it uses None as end-of-stream). + if agent.stream_delta_callback: + try: + agent.stream_delta_callback(None) + except Exception: + pass + + agent._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count) + + if agent._tool_guardrail_halt_decision is not None: + decision = agent._tool_guardrail_halt_decision + _turn_exit_reason = "guardrail_halt" + final_response = agent._toolguard_controlled_halt_response(decision) + agent._emit_status( + f"⚠️ Tool guardrail halted {decision.tool_name}: {decision.code}" + ) + messages.append({"role": "assistant", "content": final_response}) + break + + # Reset per-turn retry counters after successful tool + # execution so a single truncation doesn't poison the + # entire conversation. + truncated_tool_call_retries = 0 + + # Signal that a paragraph break is needed before the next + # streamed text. We don't emit it immediately because + # multiple consecutive tool iterations would stack up + # redundant blank lines. Instead, _fire_stream_delta() + # will prepend a single "\n\n" the next time real text + # arrives. + agent._stream_needs_break = True + + # Refund the iteration if the ONLY tool(s) called were + # execute_code (programmatic tool calling). These are + # cheap RPC-style calls that shouldn't eat the budget. + _tc_names = {tc.function.name for tc in assistant_message.tool_calls} + if _tc_names == {"execute_code"}: + agent.iteration_budget.refund() + + # Use real token counts from the API response to decide + # compression. prompt_tokens + completion_tokens is the + # actual context size the provider reported plus the + # assistant turn — a tight lower bound for the next prompt. + # Tool results appended above aren't counted yet, but the + # threshold (default 50%) leaves ample headroom; if tool + # results push past it, the next API call will report the + # real total and trigger compression then. + # + # If last_prompt_tokens is 0 (stale after API disconnect + # or provider returned no usage data), fall back to rough + # estimate to avoid missing compression. Without this, + # a session can grow unbounded after disconnects because + # should_compress(0) never fires. (#2153) + _compressor = agent.context_compressor + if _compressor.last_prompt_tokens > 0: + # Only use prompt_tokens — completion/reasoning + # tokens don't consume context window space. + # Thinking models (GLM-5.1, QwQ, DeepSeek R1) + # inflate completion_tokens with reasoning, + # causing premature compression. (#12026) + _real_tokens = _compressor.last_prompt_tokens + else: + # Include tool schemas — with 50+ tools enabled + # these add 20-30K tokens the messages-only + # estimate misses, which can skip compression + # past the configured threshold (#14695). + _real_tokens = estimate_request_tokens_rough( + messages, tools=agent.tools or None + ) + + if agent.compression_enabled and _compressor.should_compress(_real_tokens): + agent._safe_print(" ⟳ compacting context…") + messages, active_system_prompt = agent._compress_context( + messages, system_message, + approx_tokens=agent.context_compressor.last_prompt_tokens, + task_id=effective_task_id, + ) + # Compression created a new session — clear history so + # _flush_messages_to_session_db writes compressed messages + # to the new session (see preflight compression comment). + conversation_history = None + + # Save session log incrementally (so progress is visible even if interrupted) + agent._session_messages = messages + agent._save_session_log(messages) + + # Continue loop for next response + continue + + else: + # No tool calls - this is the final response + final_response = assistant_message.content or "" + + # Fix: unmute output when entering the no-tool-call branch + # so the user can see empty-response warnings and recovery + # status messages. _mute_post_response was set during a + # prior housekeeping tool turn and should not silence the + # final response path. + agent._mute_post_response = False + + # Check if response only has think block with no actual content after it + if not agent._has_content_after_think_block(final_response): + # ── Partial stream recovery ───────────────────── + # If content was already streamed to the user before + # the connection died, use it as the final response + # instead of falling through to prior-turn fallback + # or wasting API calls on retries. + _partial_streamed = ( + getattr(agent, "_current_streamed_assistant_text", "") or "" + ) + if agent._has_content_after_think_block(_partial_streamed): + _turn_exit_reason = "partial_stream_recovery" + _recovered = agent._strip_think_blocks(_partial_streamed).strip() + logger.info( + "Partial stream content delivered (%d chars) " + "— using as final response", + len(_recovered), + ) + agent._emit_status( + "↻ Stream interrupted — using delivered content " + "as final response" + ) + final_response = _recovered + agent._response_was_previewed = True + break + + # If the previous turn already delivered real content alongside + # HOUSEKEEPING tool calls (e.g. "You're welcome!" + memory save), + # the model has nothing more to say. Use the earlier content + # immediately instead of wasting API calls on retries. + # NOTE: Only use this shortcut when ALL tools in that turn were + # housekeeping (memory, todo, etc.). When substantive tools + # were called (terminal, search_files, etc.), the content was + # likely mid-task narration ("I'll scan the directory...") and + # the empty follow-up means the model choked — let the + # post-tool nudge below handle that instead of exiting early. + fallback = getattr(agent, '_last_content_with_tools', None) + if fallback and getattr(agent, '_last_content_tools_all_housekeeping', False): + _turn_exit_reason = "fallback_prior_turn_content" + logger.info("Empty follow-up after tool calls — using prior turn content as final response") + agent._emit_status("↻ Empty response after tool calls — using earlier content as final answer") + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + agent._empty_content_retries = 0 + # Do NOT modify the assistant message content — the + # old code injected "Calling the X tools..." which + # poisoned the conversation history. Just use the + # fallback text as the final response and break. + final_response = agent._strip_think_blocks(fallback).strip() + agent._response_was_previewed = True + break + + # ── Post-tool-call empty response nudge ─────────── + # The model returned empty after executing tool calls. + # This covers two cases: + # (a) No prior-turn content at all — model went silent + # (b) Prior turn had content + SUBSTANTIVE tools (the + # fallback above was skipped because the content + # was mid-task narration, not a final answer) + # Instead of giving up, nudge the model to continue by + # appending a user-level hint. This is the #9400 case: + # weaker models (mimo-v2-pro, GLM-5, etc.) sometimes + # return empty after tool results instead of continuing + # to the next step. One retry with a nudge usually + # fixes it. + _prior_was_tool = any( + m.get("role") == "tool" + for m in messages[-5:] # check recent messages + ) + # Detect Qwen3/Ollama-style in-content thinking blocks. + # Ollama puts in the content field (not in + # reasoning_content), so _has_structured below would + # miss it. We check here so thinking-only responses + # after tool calls route to prefill instead of nudge. + _has_inline_thinking = bool( + re.search( + r'||', + final_response or "", + re.IGNORECASE, + ) + ) + if ( + _prior_was_tool + and not getattr(agent, "_post_tool_empty_retried", False) + and not _has_inline_thinking # thinking model still working — let prefill handle + ): + agent._post_tool_empty_retried = True + # Clear stale narration so it doesn't resurface + # on a later empty response after the nudge. + agent._last_content_with_tools = None + agent._last_content_tools_all_housekeeping = False + logger.info( + "Empty response after tool calls — nudging model " + "to continue processing" + ) + agent._emit_status( + "⚠️ Model returned empty after tool calls — " + "nudging to continue" + ) + # Append the empty assistant message first so the + # message sequence stays valid: + # tool(result) → assistant("(empty)") → user(nudge) + # Without this, we'd have tool → user which most + # APIs reject as an invalid sequence. + _nudge_msg = agent._build_assistant_message(assistant_message, finish_reason) + _nudge_msg["content"] = "(empty)" + _nudge_msg["_empty_recovery_synthetic"] = True + messages.append(_nudge_msg) + messages.append({ + "role": "user", + "content": ( + "You just executed tool calls but returned an " + "empty response. Please process the tool " + "results above and continue with the task." + ), + "_empty_recovery_synthetic": True, + }) + continue + + # ── Thinking-only prefill continuation ────────── + # The model produced structured reasoning (via API + # fields) but no visible text content. Rather than + # giving up, append the assistant message as-is and + # continue — the model will see its own reasoning + # on the next turn and produce the text portion. + # Inspired by clawdbot's "incomplete-text" recovery. + # Also covers Qwen3/Ollama in-content blocks + # (detected above as _has_inline_thinking). + _has_structured = bool( + getattr(assistant_message, "reasoning", None) + or getattr(assistant_message, "reasoning_content", None) + or getattr(assistant_message, "reasoning_details", None) + or _has_inline_thinking + ) + if _has_structured and agent._thinking_prefill_retries < 2: + agent._thinking_prefill_retries += 1 + logger.info( + "Thinking-only response (no visible content) — " + "prefilling to continue (%d/2)", + agent._thinking_prefill_retries, + ) + agent._emit_status( + f"↻ Thinking-only response — prefilling to continue " + f"({agent._thinking_prefill_retries}/2)" + ) + interim_msg = agent._build_assistant_message( + assistant_message, "incomplete" + ) + interim_msg["_thinking_prefill"] = True + messages.append(interim_msg) + agent._session_messages = messages + agent._save_session_log(messages) + continue + + # ── Empty response retry ────────────────────── + # Model returned nothing usable. Retry up to 3 + # times before attempting fallback. This covers + # both truly empty responses (no content, no + # reasoning) AND reasoning-only responses after + # prefill exhaustion — models like mimo-v2-pro + # always populate reasoning fields via OpenRouter, + # so the old `not _has_structured` guard blocked + # retries for every reasoning model after prefill. + _truly_empty = not agent._strip_think_blocks( + final_response + ).strip() + _prefill_exhausted = ( + _has_structured + and agent._thinking_prefill_retries >= 2 + ) + if _truly_empty and (not _has_structured or _prefill_exhausted) and agent._empty_content_retries < 3: + agent._empty_content_retries += 1 + logger.warning( + "Empty response (no content or reasoning) — " + "retry %d/3 (model=%s)", + agent._empty_content_retries, agent.model, + ) + agent._emit_status( + f"⚠️ Empty response from model — retrying " + f"({agent._empty_content_retries}/3)" + ) + continue + + # ── Exhausted retries — try fallback provider ── + # Before giving up with "(empty)", attempt to + # switch to the next provider in the fallback + # chain. This covers the case where a model + # (e.g. GLM-4.5-Air) consistently returns empty + # due to context degradation or provider issues. + if _truly_empty and agent._fallback_chain: + logger.warning( + "Empty response after %d retries — " + "attempting fallback (model=%s, provider=%s)", + agent._empty_content_retries, agent.model, + agent.provider, + ) + agent._emit_status( + "⚠️ Model returning empty responses — " + "switching to fallback provider..." + ) + if agent._try_activate_fallback(): + agent._empty_content_retries = 0 + agent._emit_status( + f"↻ Switched to fallback: {agent.model} " + f"({agent.provider})" + ) + logger.info( + "Fallback activated after empty responses: " + "now using %s on %s", + agent.model, agent.provider, + ) + continue + + # Exhausted retries and fallback chain (or no + # fallback configured). Fall through to the + # "(empty)" terminal. + _turn_exit_reason = "empty_response_exhausted" + reasoning_text = agent._extract_reasoning(assistant_message) + agent._drop_trailing_empty_response_scaffolding(messages) + assistant_msg = agent._build_assistant_message(assistant_message, finish_reason) + assistant_msg["content"] = "(empty)" + # This is a user-facing failure sentinel for the gateway, + # not real assistant content. Persisting it makes later + # "continue" turns replay assistant("(empty)") as if it + # were a meaningful model response, which can keep long + # tool-heavy sessions stuck in empty-response loops. + assistant_msg["_empty_terminal_sentinel"] = True + messages.append(assistant_msg) + + if reasoning_text: + reasoning_preview = reasoning_text[:500] + "..." if len(reasoning_text) > 500 else reasoning_text + logger.warning( + "Reasoning-only response (no visible content) " + "after exhausting retries and fallback. " + "Reasoning: %s", reasoning_preview, + ) + agent._emit_status( + "⚠️ Model produced reasoning but no visible " + "response after all retries. Returning empty." + ) + else: + logger.warning( + "Empty response (no content or reasoning) " + "after %d retries. No fallback available. " + "model=%s provider=%s", + agent._empty_content_retries, agent.model, + agent.provider, + ) + agent._emit_status( + "❌ Model returned no content after all retries" + + (" and fallback attempts." if agent._fallback_chain else + ". No fallback providers configured.") + ) + + final_response = "(empty)" + break + + # Reset retry counter/signature on successful content + agent._empty_content_retries = 0 + agent._thinking_prefill_retries = 0 + + if ( + agent.api_mode == "codex_responses" + and agent.valid_tool_names + and codex_ack_continuations < 2 + and agent._looks_like_codex_intermediate_ack( + user_message=user_message, + assistant_content=final_response, + messages=messages, + ) + ): + codex_ack_continuations += 1 + interim_msg = agent._build_assistant_message(assistant_message, "incomplete") + messages.append(interim_msg) + agent._emit_interim_assistant_message(interim_msg) + + continue_msg = { + "role": "user", + "content": ( + "[System: Continue now. Execute the required tool calls and only " + "send your final answer after completing the task.]" + ), + } + messages.append(continue_msg) + agent._session_messages = messages + agent._save_session_log(messages) + continue + + codex_ack_continuations = 0 + + if truncated_response_prefix: + final_response = truncated_response_prefix + final_response + truncated_response_prefix = "" + length_continue_retries = 0 + + final_response = agent._strip_think_blocks(final_response).strip() + + final_msg = agent._build_assistant_message(assistant_message, finish_reason) + + # Pop thinking-only prefill and empty-response retry + # scaffolding before appending the final response. These + # internal turns are only for the next API retry and should + # not become durable transcript context. + while ( + messages + and isinstance(messages[-1], dict) + and ( + messages[-1].get("_thinking_prefill") + or messages[-1].get("_empty_recovery_synthetic") + or messages[-1].get("_empty_terminal_sentinel") + ) + ): + messages.pop() + + messages.append(final_msg) + + _turn_exit_reason = f"text_response(finish_reason={finish_reason})" + if not agent.quiet_mode: + agent._safe_print(f"🎉 Conversation completed after {api_call_count} OpenAI-compatible API call(s)") + break + + except Exception as e: + error_msg = f"Error during OpenAI-compatible API call #{api_call_count}: {str(e)}" + try: + print(f"❌ {error_msg}") + except (OSError, ValueError): + logger.error(error_msg) + + logger.debug("Outer loop error in API call #%d", api_call_count, exc_info=True) + + # If an assistant message with tool_calls was already appended, + # the API expects a role="tool" result for every tool_call_id. + # Fill in error results for any that weren't answered yet. + for idx in range(len(messages) - 1, -1, -1): + msg = messages[idx] + if not isinstance(msg, dict): + break + if msg.get("role") == "tool": + continue + if msg.get("role") == "assistant" and msg.get("tool_calls"): + answered_ids = { + m["tool_call_id"] + for m in messages[idx + 1:] + if isinstance(m, dict) and m.get("role") == "tool" + } + for tc in msg["tool_calls"]: + if not tc or not isinstance(tc, dict): continue + if tc["id"] not in answered_ids: + err_msg = { + "role": "tool", + "name": _ra().AIAgent._get_tool_call_name_static(tc), + "tool_call_id": tc["id"], + "content": f"Error executing tool: {error_msg}", + } + messages.append(err_msg) + break + + # Non-tool errors don't need a synthetic message injected. + # The error is already printed to the user (line above), and + # the retry loop continues. Injecting a fake user/assistant + # message pollutes history, burns tokens, and risks violating + # role-alternation invariants. + + # If we're near the limit, break to avoid infinite loops + if api_call_count >= agent.max_iterations - 1: + _turn_exit_reason = f"error_near_max_iterations({error_msg[:80]})" + final_response = f"I apologize, but I encountered repeated errors: {error_msg}" + # Append as assistant so the history stays valid for + # session resume (avoids consecutive user messages). + messages.append({"role": "assistant", "content": final_response}) + break + + if final_response is None and ( + api_call_count >= agent.max_iterations + or agent.iteration_budget.remaining <= 0 + ): + # Budget exhausted — ask the model for a summary via one extra + # API call with tools stripped. _handle_max_iterations injects a + # user message and makes a single toolless request. + _turn_exit_reason = f"max_iterations_reached({api_call_count}/{agent.max_iterations})" + agent._emit_status( + f"⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) " + "— asking model to summarise" + ) + if not agent.quiet_mode: + agent._safe_print( + f"\n⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) " + "— requesting summary..." + ) + final_response = agent._handle_max_iterations(messages, api_call_count) + + # If running as a kanban worker, block the task so the dispatcher + # knows the worker could not complete (rather than treating it as a + # protocol violation). The agent loop strips tools before calling + # _handle_max_iterations, so the model cannot call kanban_block + # itself — we must do it on its behalf. + _kanban_task = os.environ.get("HERMES_KANBAN_TASK") + if _kanban_task: + try: + _ra().handle_function_call( + "kanban_block", + { + "task_id": _kanban_task, + "reason": ( + f"Iteration budget exhausted " + f"({api_call_count}/{agent.max_iterations}) — " + "task could not complete within the allowed " + "iterations" + ), + }, + task_id=effective_task_id, + ) + logger.info( + "kanban_block called for task %s after iteration " + "exhaustion (%d/%d)", + _kanban_task, api_call_count, agent.max_iterations, + ) + except Exception: + logger.warning( + "Failed to call kanban_block after iteration " + "exhaustion for task %s", + _kanban_task, + exc_info=True, + ) + + # Determine if conversation completed successfully + completed = final_response is not None and api_call_count < agent.max_iterations + + # Save trajectory if enabled. ``user_message`` may be a multimodal + # list of parts; the trajectory format wants a plain string. + agent._save_trajectory(messages, _summarize_user_message_for_log(user_message), completed) + + # Clean up VM and browser for this task after conversation completes + agent._cleanup_task_resources(effective_task_id) + + # Persist session to both JSON log and SQLite only after private retry + # scaffolding has been removed. Otherwise a later user "continue" turn + # can replay assistant("(empty)") / recovery nudges and fall into the + # same empty-response loop again. + agent._drop_trailing_empty_response_scaffolding(messages) + agent._persist_session(messages, conversation_history) + + # ── Turn-exit diagnostic log ───────────────────────────────────── + # Always logged at INFO so agent.log captures WHY every turn ended. + # When the last message is a tool result (agent was mid-work), log + # at WARNING — this is the "just stops" scenario users report. + _last_msg_role = messages[-1].get("role") if messages else None + _last_tool_name = None + if _last_msg_role == "tool": + # Walk back to find the assistant message with the tool call + for _m in reversed(messages): + if _m.get("role") == "assistant" and _m.get("tool_calls"): + _tcs = _m["tool_calls"] + if _tcs and isinstance(_tcs[0], dict): + _last_tool_name = _tcs[-1].get("function", {}).get("name") + break + + _turn_tool_count = sum( + 1 for m in messages + if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls") + ) + _resp_len = len(final_response) if final_response else 0 + _budget_used = agent.iteration_budget.used if agent.iteration_budget else 0 + _budget_max = agent.iteration_budget.max_total if agent.iteration_budget else 0 + + _diag_msg = ( + "Turn ended: reason=%s model=%s api_calls=%d/%d budget=%d/%d " + "tool_turns=%d last_msg_role=%s response_len=%d session=%s" + ) + _diag_args = ( + _turn_exit_reason, agent.model, api_call_count, agent.max_iterations, + _budget_used, _budget_max, + _turn_tool_count, _last_msg_role, _resp_len, + agent.session_id or "none", + ) + + if _last_msg_role == "tool" and not interrupted: + # Agent was mid-work — this is the "just stops" case. + logger.warning( + "Turn ended with pending tool result (agent may appear stuck). " + + _diag_msg + " last_tool=%s", + *_diag_args, _last_tool_name, + ) + else: + logger.info(_diag_msg, *_diag_args) + + # File-mutation verifier footer. + # If one or more ``write_file`` / ``patch`` calls failed during this + # turn and were never superseded by a successful write to the same + # path, append an advisory footer to the assistant response. This + # catches the specific case — reported by Ben Eng (#15524-adjacent) + # — where a model issues a batch of parallel patches, half of them + # fail with "Could not find old_string", and the model summarises + # the turn claiming every file was edited. The user then has to + # manually run ``git status`` to catch the lie. With this footer + # the truth is surfaced on every turn, so over-claiming is + # structurally impossible past the model. + # + # Gate: only applied when a real text response exists for this + # turn and the user didn't interrupt. Empty/interrupted turns + # already have other surface text that shouldn't be augmented. + if final_response and not interrupted: + try: + _failed = getattr(agent, "_turn_failed_file_mutations", None) or {} + if _failed and agent._file_mutation_verifier_enabled(): + footer = agent._format_file_mutation_failure_footer(_failed) + if footer: + final_response = final_response.rstrip() + "\n\n" + footer + except Exception as _ver_err: + logger.debug("file-mutation verifier footer failed: %s", _ver_err) + + # Plugin hook: transform_llm_output + # Fired once per turn after the tool-calling loop completes. + # Plugins can transform the LLM's output text before it's returned. + # First hook to return a string wins; None/empty return leaves text unchanged. + if final_response and not interrupted: + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _transform_results = _invoke_hook( + "transform_llm_output", + response_text=final_response, + session_id=agent.session_id or "", + model=agent.model, + platform=getattr(agent, "platform", None) or "", + ) + for _hook_result in _transform_results: + if isinstance(_hook_result, str) and _hook_result: + final_response = _hook_result + break # First non-empty string wins + except Exception as exc: + logger.warning("transform_llm_output hook failed: %s", exc) + + # Plugin hook: post_llm_call + # Fired once per turn after the tool-calling loop completes. + # Plugins can use this to persist conversation data (e.g. sync + # to an external memory system). + if final_response and not interrupted: + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _invoke_hook( + "post_llm_call", + session_id=agent.session_id, + user_message=original_user_message, + assistant_response=final_response, + conversation_history=list(messages), + model=agent.model, + platform=getattr(agent, "platform", None) or "", + ) + except Exception as exc: + logger.warning("post_llm_call hook failed: %s", exc) + + # Extract reasoning from the CURRENT turn only. Walk backwards + # but stop at the user message that started this turn — anything + # earlier is from a prior turn and must not leak into the reasoning + # box (confusing stale display; #17055). Within the current turn + # we still want the *most recent* non-empty reasoning: many + # providers (Claude thinking, DeepSeek v4, Codex Responses) emit + # reasoning on the tool-call step and leave the final-answer step + # with reasoning=None, so picking only the last assistant would + # silently drop legitimate same-turn reasoning. + last_reasoning = None + for msg in reversed(messages): + if msg.get("role") == "user": + break # turn boundary — don't cross into prior turns + if msg.get("role") == "assistant" and msg.get("reasoning"): + last_reasoning = msg["reasoning"] + break + + # Build result with interrupt info if applicable + result = { + "final_response": final_response, + "last_reasoning": last_reasoning, + "messages": messages, + "api_calls": api_call_count, + "completed": completed, + "turn_exit_reason": _turn_exit_reason, + "partial": False, # True only when stopped due to invalid tool calls + "interrupted": interrupted, + "response_previewed": getattr(agent, "_response_was_previewed", False), + "model": agent.model, + "provider": agent.provider, + "base_url": agent.base_url, + "input_tokens": agent.session_input_tokens, + "output_tokens": agent.session_output_tokens, + "cache_read_tokens": agent.session_cache_read_tokens, + "cache_write_tokens": agent.session_cache_write_tokens, + "reasoning_tokens": agent.session_reasoning_tokens, + "prompt_tokens": agent.session_prompt_tokens, + "completion_tokens": agent.session_completion_tokens, + "total_tokens": agent.session_total_tokens, + "last_prompt_tokens": getattr(agent.context_compressor, "last_prompt_tokens", 0) or 0, + "estimated_cost_usd": agent.session_estimated_cost_usd, + "cost_status": agent.session_cost_status, + "cost_source": agent.session_cost_source, + } + if agent._tool_guardrail_halt_decision is not None: + result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata() + # If a /steer landed after the final assistant turn (no more tool + # batches to drain into), hand it back to the caller so it can be + # delivered as the next user turn instead of being silently lost. + _leftover_steer = agent._drain_pending_steer() + if _leftover_steer: + result["pending_steer"] = _leftover_steer + agent._response_was_previewed = False + + # Include interrupt message if one triggered the interrupt + if interrupted and agent._interrupt_message: + result["interrupt_message"] = agent._interrupt_message + + # Clear interrupt state after handling + agent.clear_interrupt() + + # Clear stream callback so it doesn't leak into future calls + agent._stream_callback = None + + # Check skill trigger NOW — based on how many tool iterations THIS turn used. + _should_review_skills = False + if (agent._skill_nudge_interval > 0 + and agent._iters_since_skill >= agent._skill_nudge_interval + and "skill_manage" in agent.valid_tool_names): + _should_review_skills = True + agent._iters_since_skill = 0 + + # External memory provider: sync the completed turn + queue next prefetch. + agent._sync_external_memory_for_turn( + original_user_message=original_user_message, + final_response=final_response, + interrupted=interrupted, + ) + + # Background memory/skill review — runs AFTER the response is delivered + # so it never competes with the user's task for model attention. + if final_response and not interrupted and (_should_review_memory or _should_review_skills): + try: + agent._spawn_background_review( + messages_snapshot=list(messages), + review_memory=_should_review_memory, + review_skills=_should_review_skills, + ) + except Exception: + pass # Background review is best-effort + + # Note: Memory provider on_session_end() + shutdown_all() are NOT + # called here — run_conversation() is called once per user message in + # multi-turn sessions. Shutting down after every turn would kill the + # provider before the second message. Actual session-end cleanup is + # handled by the CLI (atexit / /reset) and gateway (session expiry / + # _reset_session). + + # Plugin hook: on_session_end + # Fired at the very end of every run_conversation call. + # Plugins can use this for cleanup, flushing buffers, etc. + try: + from hermes_cli.plugins import invoke_hook as _invoke_hook + _invoke_hook( + "on_session_end", + session_id=agent.session_id, + completed=completed, + interrupted=interrupted, + model=agent.model, + platform=getattr(agent, "platform", None) or "", + ) + except Exception as exc: + logger.warning("on_session_end hook failed: %s", exc) + + return result + + + +__all__ = ["run_conversation"] diff --git a/run_agent.py b/run_agent.py index 8ea73167ac9..b13eb851175 100644 --- a/run_agent.py +++ b/run_agent.py @@ -5694,3873 +5694,9 @@ class AIAgent: stream_callback: Optional[callable] = None, persist_user_message: Optional[str] = None, ) -> Dict[str, Any]: - """ - Run a complete conversation with tool calling until completion. - - Args: - user_message (str): The user's message/question - system_message (str): Custom system message (optional, overrides ephemeral_system_prompt if provided) - conversation_history (List[Dict]): Previous conversation messages (optional) - task_id (str): Unique identifier for this task to isolate VMs between concurrent tasks (optional, auto-generated if not provided) - stream_callback: Optional callback invoked with each text delta during streaming. - Used by the TTS pipeline to start audio generation before the full response. - When None (default), API calls use the standard non-streaming path. - persist_user_message: Optional clean user message to store in - transcripts/history when user_message contains API-only - synthetic prefixes. - or queuing follow-up prefetch work. - - Returns: - Dict: Complete conversation result with final response and message history - """ - # Guard stdio against OSError from broken pipes (systemd/headless/daemon). - # Installed once, transparent when streams are healthy, prevents crash on write. - _install_safe_stdio() - - self._ensure_db_session() - - # Tell auxiliary_client what the live main provider/model are for - # this turn. Used by tools whose behaviour depends on the active - # main model (e.g. vision_analyze's native fast path) so they see - # the CLI/gateway override instead of the stale config.yaml - # default. Idempotent — fine to call every turn. - try: - from agent.auxiliary_client import set_runtime_main - set_runtime_main( - getattr(self, "provider", "") or "", - getattr(self, "model", "") or "", - ) - except Exception: - pass - - # Tag all log records on this thread with the session ID so - # ``hermes logs --session `` can filter a single conversation. - from hermes_logging import set_session_context - set_session_context(self.session_id) - - # Bind the skill write-origin ContextVar for this thread so tool - # handlers (e.g. skill_manage create) can tell whether they are - # running inside the background self-improvement review fork vs. - # a foreground user-directed turn. Set at the top of each call; - # the review fork runs on its own thread with a fresh context, - # so the foreground value here does not leak into it. - from tools.skill_provenance import set_current_write_origin - set_current_write_origin(getattr(self, "_memory_write_origin", "assistant_tool")) - - # If the previous turn activated fallback, restore the primary - # runtime so this turn gets a fresh attempt with the preferred model. - # No-op when _fallback_activated is False (gateway, first turn, etc.). - self._restore_primary_runtime() - - # Sanitize surrogate characters from user input. Clipboard paste from - # rich-text editors (Google Docs, Word, etc.) can inject lone surrogates - # that are invalid UTF-8 and crash JSON serialization in the OpenAI SDK. - if isinstance(user_message, str): - user_message = _sanitize_surrogates(user_message) - if isinstance(persist_user_message, str): - persist_user_message = _sanitize_surrogates(persist_user_message) - - # Store stream callback for _interruptible_api_call to pick up - self._stream_callback = stream_callback - self._persist_user_message_idx = None - self._persist_user_message_override = persist_user_message - # Generate unique task_id if not provided to isolate VMs between concurrent tasks - effective_task_id = task_id or str(uuid.uuid4()) - # Expose the active task_id so tools running mid-turn (e.g. delegate_task - # in delegate_tool.py) can identify this agent for the cross-agent file - # state registry. Set BEFORE any tool dispatch so snapshots taken at - # child-launch time see the parent's real id, not None. - self._current_task_id = effective_task_id - - # Reset retry counters and iteration budget at the start of each turn - # so subagent usage from a previous turn doesn't eat into the next one. - self._invalid_tool_retries = 0 - self._invalid_json_retries = 0 - self._empty_content_retries = 0 - self._incomplete_scratchpad_retries = 0 - self._codex_incomplete_retries = 0 - self._thinking_prefill_retries = 0 - self._post_tool_empty_retried = False - self._last_content_with_tools = None - self._last_content_tools_all_housekeeping = False - self._mute_post_response = False - self._unicode_sanitization_passes = 0 - self._tool_guardrails.reset_for_turn() - self._tool_guardrail_halt_decision = None - # True until the server rejects an image_url content part with an error - # like "Only 'text' content type is supported." Set to False on first - # rejection and kept False for the rest of the session so we never re-send - # images to a text-only endpoint. Scoped per `_run()` call, not per instance. - self._vision_supported = True - - # Pre-turn connection health check: detect and clean up dead TCP - # connections left over from provider outages or dropped streams. - # This prevents the next API call from hanging on a zombie socket. - if self.api_mode != "anthropic_messages": - try: - if self._cleanup_dead_connections(): - self._emit_status( - "🔌 Detected stale connections from a previous provider " - "issue — cleaned up automatically. Proceeding with fresh " - "connection." - ) - except Exception: - pass - # Replay compression warning through status_callback for gateway - # platforms (the callback was not wired during __init__). - if self._compression_warning: - self._replay_compression_warning() - self._compression_warning = None # send once - - # NOTE: _turns_since_memory and _iters_since_skill are NOT reset here. - # They are initialized in __init__ and must persist across run_conversation - # calls so that nudge logic accumulates correctly in CLI mode. - self.iteration_budget = IterationBudget(self.max_iterations) - - # Log conversation turn start for debugging/observability - _preview_text = _summarize_user_message_for_log(user_message) - _msg_preview = (_preview_text[:80] + "...") if len(_preview_text) > 80 else _preview_text - _msg_preview = _msg_preview.replace("\n", " ") - logger.info( - "conversation turn: session=%s model=%s provider=%s platform=%s history=%d msg=%r", - self.session_id or "none", self.model, self.provider or "unknown", - self.platform or "unknown", len(conversation_history or []), - _msg_preview, - ) - - # Initialize conversation (copy to avoid mutating the caller's list) - messages = list(conversation_history) if conversation_history else [] - - # Hydrate todo store from conversation history (gateway creates a fresh - # AIAgent per message, so the in-memory store is empty -- we need to - # recover the todo state from the most recent todo tool response in history) - if conversation_history and not self._todo_store.has_items(): - self._hydrate_todo_store(conversation_history) - - # Hydrate per-session nudge counters from persisted history. - # Gateway creates a fresh AIAgent per inbound message (cache miss / - # 1h idle eviction / config-signature mismatch / process restart), so - # _turns_since_memory and _user_turn_count start at 0 every turn and - # the memory.nudge_interval trigger may never be reached. Reconstruct - # an effective count from prior user turns in conversation_history. - # Idempotent: a cached agent that already accumulated counters keeps - # them; only a freshly-built agent with empty in-memory state hydrates. - # See issue #22357. - if conversation_history and self._user_turn_count == 0: - prior_user_turns = sum( - 1 for m in conversation_history if m.get("role") == "user" - ) - if prior_user_turns > 0: - self._user_turn_count = prior_user_turns - if self._memory_nudge_interval > 0 and self._turns_since_memory == 0: - # % preserves original 1-in-N cadence rather than firing a - # review immediately on resume (which would surprise users - # whose session happened to land just past a multiple of N). - self._turns_since_memory = prior_user_turns % self._memory_nudge_interval - - - # Prefill messages (few-shot priming) are injected at API-call time only, - # never stored in the messages list. This keeps them ephemeral: they won't - # be saved to session DB, session logs, or batch trajectories, but they're - # automatically re-applied on every API call (including session continuations). - - # Track user turns for memory flush and periodic nudge logic - self._user_turn_count += 1 - - # Reset the streaming context scrubber at the top of each turn so a - # hung span from a prior interrupted stream can't taint this turn's - # output. - scrubber = getattr(self, "_stream_context_scrubber", None) - if scrubber is not None: - scrubber.reset() - # Reset the think scrubber for the same reason — an interrupted - # prior stream may have left us inside an unterminated block. - think_scrubber = getattr(self, "_stream_think_scrubber", None) - if think_scrubber is not None: - think_scrubber.reset() - - # Preserve the original user message (no nudge injection). - original_user_message = persist_user_message if persist_user_message is not None else user_message - - # Track memory nudge trigger (turn-based, checked here). - # Skill trigger is checked AFTER the agent loop completes, based on - # how many tool iterations THIS turn used. - _should_review_memory = False - if (self._memory_nudge_interval > 0 - and "memory" in self.valid_tool_names - and self._memory_store): - self._turns_since_memory += 1 - if self._turns_since_memory >= self._memory_nudge_interval: - _should_review_memory = True - self._turns_since_memory = 0 - - # Add user message - user_msg = {"role": "user", "content": user_message} - messages.append(user_msg) - current_turn_user_idx = len(messages) - 1 - self._persist_user_message_idx = current_turn_user_idx - - if not self.quiet_mode: - _print_preview = _summarize_user_message_for_log(user_message) - self._safe_print(f"💬 Starting conversation: '{_print_preview[:60]}{'...' if len(_print_preview) > 60 else ''}'") - - # ── System prompt (cached per session for prefix caching) ── - # Built once on first call, reused for all subsequent calls. - # Only rebuilt after context compression events (which invalidate - # the cache and reload memory from disk). - # - # For continuing sessions (gateway creates a fresh AIAgent per - # message), we load the stored system prompt from the session DB - # instead of rebuilding. Rebuilding would pick up memory changes - # from disk that the model already knows about (it wrote them!), - # producing a different system prompt and breaking the Anthropic - # prefix cache. - if self._cached_system_prompt is None: - stored_prompt = None - if conversation_history and self._session_db: - try: - session_row = self._session_db.get_session(self.session_id) - if session_row: - stored_prompt = session_row.get("system_prompt") or None - except Exception: - pass # Fall through to build fresh - - if stored_prompt: - # Continuing session — reuse the exact system prompt from - # the previous turn so the Anthropic cache prefix matches. - self._cached_system_prompt = stored_prompt - else: - # First turn of a new session — build from scratch. - self._cached_system_prompt = self._build_system_prompt(system_message) - # Plugin hook: on_session_start - # Fired once when a brand-new session is created (not on - # continuation). Plugins can use this to initialise - # session-scoped state (e.g. warm a memory cache). - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_start", - session_id=self.session_id, - model=self.model, - platform=getattr(self, "platform", None) or "", - ) - except Exception as exc: - logger.warning("on_session_start hook failed: %s", exc) - - # Store the system prompt snapshot in SQLite - if self._session_db: - try: - self._session_db.update_system_prompt(self.session_id, self._cached_system_prompt) - except Exception as e: - logger.debug("Session DB update_system_prompt failed: %s", e) - - active_system_prompt = self._cached_system_prompt - - # ── Preflight context compression ── - # Before entering the main loop, check if the loaded conversation - # history already exceeds the model's context threshold. This handles - # cases where a user switches to a model with a smaller context window - # while having a large existing session — compress proactively rather - # than waiting for an API error (which might be caught as a non-retryable - # 4xx and abort the request entirely). - if ( - self.compression_enabled - and len(messages) > self.context_compressor.protect_first_n - + self.context_compressor.protect_last_n + 1 - ): - # Include tool schema tokens — with many tools these can add - # 20-30K+ tokens that the old sys+msg estimate missed entirely. - _preflight_tokens = estimate_request_tokens_rough( - messages, - system_prompt=active_system_prompt or "", - tools=self.tools or None, - ) - - if _preflight_tokens >= self.context_compressor.threshold_tokens: - logger.info( - "Preflight compression: ~%s tokens >= %s threshold (model %s, ctx %s)", - f"{_preflight_tokens:,}", - f"{self.context_compressor.threshold_tokens:,}", - self.model, - f"{self.context_compressor.context_length:,}", - ) - self._emit_status( - f"📦 Preflight compression: ~{_preflight_tokens:,} tokens " - f">= {self.context_compressor.threshold_tokens:,} threshold. " - "This may take a moment." - ) - # May need multiple passes for very large sessions with small - # context windows (each pass summarises the middle N turns). - for _pass in range(3): - _orig_len = len(messages) - messages, active_system_prompt = self._compress_context( - messages, system_message, approx_tokens=_preflight_tokens, - task_id=effective_task_id, - ) - if len(messages) >= _orig_len: - break # Cannot compress further - # Compression created a new session — clear the history - # reference so _flush_messages_to_session_db writes ALL - # compressed messages to the new session's SQLite, not - # skipping them because conversation_history is still the - # pre-compression length. - conversation_history = None - # Fix: reset retry counters after compression so the model - # gets a fresh budget on the compressed context. Without - # this, pre-compression retries carry over and the model - # hits "(empty)" immediately after compression-induced - # context loss. - self._empty_content_retries = 0 - self._thinking_prefill_retries = 0 - self._last_content_with_tools = None - self._last_content_tools_all_housekeeping = False - self._mute_post_response = False - # Re-estimate after compression - _preflight_tokens = estimate_request_tokens_rough( - messages, - system_prompt=active_system_prompt or "", - tools=self.tools or None, - ) - if _preflight_tokens < self.context_compressor.threshold_tokens: - break # Under threshold - - # Plugin hook: pre_llm_call - # Fired once per turn before the tool-calling loop. Plugins can - # return a dict with a ``context`` key (or a plain string) whose - # value is appended to the current turn's user message. - # - # Context is ALWAYS injected into the user message, never the - # system prompt. This preserves the prompt cache prefix — the - # system prompt stays identical across turns so cached tokens - # are reused. The system prompt is Hermes's territory; plugins - # contribute context alongside the user's input. - # - # All injected context is ephemeral (not persisted to session DB). - _plugin_user_context = "" - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _pre_results = _invoke_hook( - "pre_llm_call", - session_id=self.session_id, - user_message=original_user_message, - conversation_history=list(messages), - is_first_turn=(not bool(conversation_history)), - model=self.model, - platform=getattr(self, "platform", None) or "", - sender_id=getattr(self, "_user_id", None) or "", - ) - _ctx_parts: list[str] = [] - for r in _pre_results: - if isinstance(r, dict) and r.get("context"): - _ctx_parts.append(str(r["context"])) - elif isinstance(r, str) and r.strip(): - _ctx_parts.append(r) - if _ctx_parts: - _plugin_user_context = "\n\n".join(_ctx_parts) - except Exception as exc: - logger.warning("pre_llm_call hook failed: %s", exc) - - # Main conversation loop - api_call_count = 0 - final_response = None - interrupted = False - codex_ack_continuations = 0 - length_continue_retries = 0 - truncated_tool_call_retries = 0 - truncated_response_prefix = "" - compression_attempts = 0 - _turn_exit_reason = "unknown" # Diagnostic: why the loop ended - - # Per-turn file-mutation verifier state. Keyed by resolved path; - # each failed ``write_file`` / ``patch`` call records the error - # preview. Later successful writes to the same path remove the - # entry (the model recovered). At end-of-turn, any entries still - # present are surfaced in an advisory footer so the model cannot - # over-claim success while the file is actually unchanged on disk. - self._turn_failed_file_mutations: Dict[str, Dict[str, Any]] = {} - - # Record the execution thread so interrupt()/clear_interrupt() can - # scope the tool-level interrupt signal to THIS agent's thread only. - # Must be set before any thread-scoped interrupt syncing. - self._execution_thread_id = threading.current_thread().ident - - # Always clear stale per-thread state from a previous turn. If an - # interrupt arrived before startup finished, preserve it and bind it - # to this execution thread now instead of dropping it on the floor. - _set_interrupt(False, self._execution_thread_id) - if self._interrupt_requested: - _set_interrupt(True, self._execution_thread_id) - self._interrupt_thread_signal_pending = False - else: - self._interrupt_message = None - self._interrupt_thread_signal_pending = False - - # Notify memory providers of the new turn so cadence tracking works. - # Must happen BEFORE prefetch_all() so providers know which turn it is - # and can gate context/dialectic refresh via contextCadence/dialecticCadence. - if self._memory_manager: - try: - _turn_msg = original_user_message if isinstance(original_user_message, str) else "" - self._memory_manager.on_turn_start(self._user_turn_count, _turn_msg) - except Exception: - pass - - # External memory provider: prefetch once before the tool loop. - # Reuse the cached result on every iteration to avoid re-calling - # prefetch_all() on each tool call (10 tool calls = 10x latency + cost). - # Use original_user_message (clean input) — user_message may contain - # injected skill content that bloats / breaks provider queries. - _ext_prefetch_cache = "" - if self._memory_manager: - try: - _query = original_user_message if isinstance(original_user_message, str) else "" - _ext_prefetch_cache = self._memory_manager.prefetch_all(_query) or "" - except Exception: - pass - - # Optional opt-in runtime: if api_mode == codex_app_server, hand the - # turn to the codex app-server subprocess (terminal/file ops/patching - # all run inside Codex). Default Hermes path is bypassed entirely. - # See agent/transports/codex_app_server_session.py for the adapter - # and references/codex-app-server-runtime.md for the rationale. - if self.api_mode == "codex_app_server": - return self._run_codex_app_server_turn( - user_message=user_message, - original_user_message=original_user_message, - messages=messages, - effective_task_id=effective_task_id, - should_review_memory=_should_review_memory, - ) - - while (api_call_count < self.max_iterations and self.iteration_budget.remaining > 0) or self._budget_grace_call: - # Reset per-turn checkpoint dedup so each iteration can take one snapshot - self._checkpoint_mgr.new_turn() - - # Check for interrupt request (e.g., user sent new message) - if self._interrupt_requested: - interrupted = True - _turn_exit_reason = "interrupted_by_user" - if not self.quiet_mode: - self._safe_print("\n⚡ Breaking out of tool loop due to interrupt...") - break - - api_call_count += 1 - self._api_call_count = api_call_count - self._touch_activity(f"starting API call #{api_call_count}") - - # Grace call: the budget is exhausted but we gave the model one - # more chance. Consume the grace flag so the loop exits after - # this iteration regardless of outcome. - if self._budget_grace_call: - self._budget_grace_call = False - elif not self.iteration_budget.consume(): - _turn_exit_reason = "budget_exhausted" - if not self.quiet_mode: - self._safe_print(f"\n⚠️ Iteration budget exhausted ({self.iteration_budget.used}/{self.iteration_budget.max_total} iterations used)") - break - - # Fire step_callback for gateway hooks (agent:step event) - if self.step_callback is not None: - try: - prev_tools = [] - for _idx, _m in enumerate(reversed(messages)): - if _m.get("role") == "assistant" and _m.get("tool_calls"): - _fwd_start = len(messages) - _idx - _results_by_id = {} - for _tm in messages[_fwd_start:]: - if _tm.get("role") != "tool": - break - _tcid = _tm.get("tool_call_id") - if _tcid: - _results_by_id[_tcid] = _tm.get("content", "") - prev_tools = [ - { - "name": tc["function"]["name"], - "result": _results_by_id.get(tc.get("id")), - "arguments": tc["function"].get("arguments"), - } - for tc in _m["tool_calls"] - if isinstance(tc, dict) - ] - break - self.step_callback(api_call_count, prev_tools) - except Exception as _step_err: - logger.debug("step_callback error (iteration %s): %s", api_call_count, _step_err) - - # Track tool-calling iterations for skill nudge. - # Counter resets whenever skill_manage is actually used. - if (self._skill_nudge_interval > 0 - and "skill_manage" in self.valid_tool_names): - self._iters_since_skill += 1 - - # ── Pre-API-call /steer drain ────────────────────────────────── - # If a /steer arrived during the previous API call (while the model - # was thinking), drain it now — before we build api_messages — so - # the model sees the steer text on THIS iteration. Without this, - # steers sent during an API call only land after the NEXT tool batch, - # which may never come if the model returns a final response. - # - # We scan backwards for the last tool-role message in the messages - # list. If found, the steer is appended there. If not (first - # iteration, no tools yet), the steer stays pending for the next - # tool batch — injecting into a user message would break role - # alternation, and there's no tool output to piggyback on. - _pre_api_steer = self._drain_pending_steer() - if _pre_api_steer: - _injected = False - for _si in range(len(messages) - 1, -1, -1): - _sm = messages[_si] - if isinstance(_sm, dict) and _sm.get("role") == "tool": - marker = f"\n\nUser guidance: {_pre_api_steer}" - existing = _sm.get("content", "") - if isinstance(existing, str): - _sm["content"] = existing + marker - else: - # Multimodal content blocks — append text block - try: - blocks = list(existing) if existing else [] - blocks.append({"type": "text", "text": marker}) - _sm["content"] = blocks - except Exception: - pass - _injected = True - logger.debug( - "Pre-API-call steer drain: injected into tool msg at index %d", - _si, - ) - break - if not _injected: - # No tool message to inject into — put it back so - # the post-tool-execution drain picks it up later. - _lock = getattr(self, "_pending_steer_lock", None) - if _lock is not None: - with _lock: - if self._pending_steer: - self._pending_steer = self._pending_steer + "\n" + _pre_api_steer - else: - self._pending_steer = _pre_api_steer - else: - existing = getattr(self, "_pending_steer", None) - self._pending_steer = (existing + "\n" + _pre_api_steer) if existing else _pre_api_steer - - # Prepare messages for API call - # If we have an ephemeral system prompt, prepend it to the messages - # Note: Reasoning is embedded in content via tags for trajectory storage. - # However, providers like Moonshot AI require a separate 'reasoning_content' field - # on assistant messages with tool_calls. We handle both cases here. - request_logger = getattr(self, "logger", None) or logging.getLogger(__name__) - repaired_tool_calls = self._sanitize_tool_call_arguments( - messages, - logger=request_logger, - session_id=self.session_id, - ) - if repaired_tool_calls > 0: - request_logger.info( - "Sanitized %s corrupted tool_call arguments before request (session=%s)", - repaired_tool_calls, - self.session_id or "-", - ) - - # Defensive: repair malformed role-alternation before API call. - # Catches cases where the history got wedged into a - # ``tool → user`` or ``user → user`` tail (e.g. after empty- - # response scaffolding was stripped and a new user message - # landed after an orphan tool result). Most providers return - # empty content on malformed sequences, which would otherwise - # retrigger the empty-retry loop indefinitely. - repaired_seq = self._repair_message_sequence(messages) - if repaired_seq > 0: - request_logger.info( - "Repaired %s message-alternation violations before request (session=%s)", - repaired_seq, - self.session_id or "-", - ) - - api_messages = [] - for idx, msg in enumerate(messages): - api_msg = msg.copy() - - # Inject ephemeral context into the current turn's user message. - # Sources: memory manager prefetch + plugin pre_llm_call hooks - # with target="user_message" (the default). Both are - # API-call-time only — the original message in `messages` is - # never mutated, so nothing leaks into session persistence. - if idx == current_turn_user_idx and msg.get("role") == "user": - _injections = [] - if _ext_prefetch_cache: - _fenced = build_memory_context_block(_ext_prefetch_cache) - if _fenced: - _injections.append(_fenced) - if _plugin_user_context: - _injections.append(_plugin_user_context) - if _injections: - _base = api_msg.get("content", "") - if isinstance(_base, str): - api_msg["content"] = _base + "\n\n" + "\n\n".join(_injections) - - # For ALL assistant messages, pass reasoning back to the API - # This ensures multi-turn reasoning context is preserved - self._copy_reasoning_content_for_api(msg, api_msg) - - # Remove 'reasoning' field - it's for trajectory storage only - # We've copied it to 'reasoning_content' for the API above - if "reasoning" in api_msg: - api_msg.pop("reasoning") - # Remove finish_reason - not accepted by strict APIs (e.g. Mistral) - if "finish_reason" in api_msg: - api_msg.pop("finish_reason") - # Strip internal thinking-prefill marker - api_msg.pop("_thinking_prefill", None) - # Strip Codex Responses API fields (call_id, response_item_id) for - # strict providers like Mistral, Fireworks, etc. that reject unknown fields. - # Uses new dicts so the internal messages list retains the fields - # for Codex Responses compatibility. - if self._should_sanitize_tool_calls(): - self._sanitize_tool_calls_for_strict_api(api_msg) - # Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context - # The signature field helps maintain reasoning continuity - api_messages.append(api_msg) - - # Build the final system message: cached prompt + ephemeral system prompt. - # Ephemeral additions are API-call-time only (not persisted to session DB). - # External recall context is injected into the user message, not the system - # prompt, so the stable cache prefix remains unchanged. - # - # NOTE: Plugin context from pre_llm_call hooks is injected into the - # user message (see injection block above), NOT the system prompt. - # This is intentional — system prompt modifications break the prompt - # cache prefix. The system prompt is reserved for Hermes internals. - # - # Hermes invariant: the system prompt is built ONCE per session - # (cached on ``_cached_system_prompt``) and replayed verbatim on - # every turn. We send it as a single content string so the - # bytes are byte-stable across turns and upstream prompt caches - # stay warm. - effective_system = active_system_prompt or "" - if self.ephemeral_system_prompt: - effective_system = (effective_system + "\n\n" + self.ephemeral_system_prompt).strip() - if effective_system: - api_messages = [{"role": "system", "content": effective_system}] + api_messages - - # Inject ephemeral prefill messages right after the system prompt - # but before conversation history. Same API-call-time-only pattern. - if self.prefill_messages: - sys_offset = 1 if (api_messages and api_messages[0].get("role") == "system") else 0 - for idx, pfm in enumerate(self.prefill_messages): - api_messages.insert(sys_offset + idx, pfm.copy()) - - # Apply Anthropic prompt caching for Claude models on native - # Anthropic, OpenRouter, and third-party Anthropic-compatible - # gateways. Auto-detected: if ``_use_prompt_caching`` is set, - # inject cache_control breakpoints (system + last 3 messages) - # to reduce input token costs by ~75% on multi-turn - # conversations. - if self._use_prompt_caching: - api_messages = apply_anthropic_cache_control( - api_messages, - cache_ttl=self._cache_ttl, - native_anthropic=self._use_native_cache_layout, - ) - - # Safety net: strip orphaned tool results / add stubs for missing - # results before sending to the API. Runs unconditionally — not - # gated on context_compressor — so orphans from session loading or - # manual message manipulation are always caught. - api_messages = self._sanitize_api_messages(api_messages) - - # Drop thinking-only assistant turns (reasoning but no visible - # output and no tool_calls) and merge any adjacent user messages - # left behind. Prevents Anthropic 400s ("The final block in an - # assistant message cannot be `thinking`.") and equivalent errors - # from third-party Anthropic-compatible gateways that can't replay - # a thinking-only turn. Runs on the per-call copy only — the - # stored conversation history keeps the reasoning block for the - # UI transcript and session persistence. - api_messages = self._drop_thinking_only_and_merge_users(api_messages) - - # Normalize message whitespace and tool-call JSON for consistent - # prefix matching. Ensures bit-perfect prefixes across turns, - # which enables KV cache reuse on local inference servers - # (llama.cpp, vLLM, Ollama) and improves cache hit rates for - # cloud providers. Operates on api_messages (the API copy) so - # the original conversation history in `messages` is untouched. - for am in api_messages: - if isinstance(am.get("content"), str): - am["content"] = am["content"].strip() - for am in api_messages: - tcs = am.get("tool_calls") - if not tcs: - continue - new_tcs = [] - for tc in tcs: - if isinstance(tc, dict) and "function" in tc: - try: - args_obj = json.loads(tc["function"]["arguments"]) - tc = {**tc, "function": { - **tc["function"], - "arguments": json.dumps( - args_obj, separators=(",", ":"), - sort_keys=True, - ), - }} - except Exception: - tc["function"]["arguments"] = _repair_tool_call_arguments( - tc["function"]["arguments"], - tc["function"].get("name", "?"), - ) - new_tcs.append(tc) - am["tool_calls"] = new_tcs - - # Proactively strip any surrogate characters before the API call. - # Models served via Ollama (Kimi K2.5, GLM-5, Qwen) can return - # lone surrogates (U+D800-U+DFFF) that crash json.dumps() inside - # the OpenAI SDK. Sanitizing here prevents the 3-retry cycle. - _sanitize_messages_surrogates(api_messages) - - # Calculate approximate request size for logging - total_chars = sum(len(str(msg)) for msg in api_messages) - approx_tokens = estimate_messages_tokens_rough(api_messages) - - # Thinking spinner for quiet mode (animated during API call) - thinking_spinner = None - - if not self.quiet_mode: - self._vprint(f"\n{self.log_prefix}🔄 Making API call #{api_call_count}/{self.max_iterations}...") - self._vprint(f"{self.log_prefix} 📊 Request size: {len(api_messages)} messages, ~{approx_tokens:,} tokens (~{total_chars:,} chars)") - self._vprint(f"{self.log_prefix} 🔧 Available tools: {len(self.tools) if self.tools else 0}") - else: - # Animated thinking spinner in quiet mode - face = random.choice(KawaiiSpinner.get_thinking_faces()) - verb = random.choice(KawaiiSpinner.get_thinking_verbs()) - if self.thinking_callback: - # CLI TUI mode: use prompt_toolkit widget instead of raw spinner - # (works in both streaming and non-streaming modes) - self.thinking_callback(f"{face} {verb}...") - elif not self._has_stream_consumers() and self._should_start_quiet_spinner(): - # Raw KawaiiSpinner only when no streaming consumers and the - # spinner output has a safe sink. - spinner_type = random.choice(['brain', 'sparkle', 'pulse', 'moon', 'star']) - thinking_spinner = KawaiiSpinner(f"{face} {verb}...", spinner_type=spinner_type, print_fn=self._print_fn) - thinking_spinner.start() - - # Log request details if verbose - if self.verbose_logging: - logging.debug(f"API Request - Model: {self.model}, Messages: {len(messages)}, Tools: {len(self.tools) if self.tools else 0}") - logging.debug(f"Last message role: {messages[-1]['role'] if messages else 'none'}") - logging.debug(f"Total message size: ~{approx_tokens:,} tokens") - - api_start_time = time.time() - retry_count = 0 - max_retries = self._api_max_retries - primary_recovery_attempted = False - max_compression_attempts = 3 - codex_auth_retry_attempted=False - anthropic_auth_retry_attempted=False - nous_auth_retry_attempted=False - copilot_auth_retry_attempted=False - thinking_sig_retry_attempted = False - image_shrink_retry_attempted = False - oauth_1m_beta_retry_attempted = False - llama_cpp_grammar_retry_attempted = False - has_retried_429 = False - restart_with_compressed_messages = False - restart_with_length_continuation = False - - finish_reason = "stop" - response = None # Guard against UnboundLocalError if all retries fail - api_kwargs = None # Guard against UnboundLocalError in except handler - - while retry_count < max_retries: - # ── Nous Portal rate limit guard ────────────────────── - # If another session already recorded that Nous is rate- - # limited, skip the API call entirely. Each attempt - # (including SDK-level retries) counts against RPH and - # deepens the rate limit hole. - if self.provider == "nous": - try: - from agent.nous_rate_guard import ( - nous_rate_limit_remaining, - format_remaining as _fmt_nous_remaining, - ) - _nous_remaining = nous_rate_limit_remaining() - if _nous_remaining is not None and _nous_remaining > 0: - _nous_msg = ( - f"Nous Portal rate limit active — " - f"resets in {_fmt_nous_remaining(_nous_remaining)}." - ) - self._vprint( - f"{self.log_prefix}⏳ {_nous_msg} Trying fallback...", - force=True, - ) - self._emit_status(f"⏳ {_nous_msg}") - if self._try_activate_fallback(): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - # No fallback available — return with clear message - self._persist_session(messages, conversation_history) - return { - "final_response": ( - f"⏳ {_nous_msg}\n\n" - "No fallback provider available. " - "Try again after the reset, or add a " - "fallback provider in config.yaml." - ), - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "failed": True, - "error": _nous_msg, - } - except ImportError: - pass - except Exception: - pass # Never let rate guard break the agent loop - - try: - self._reset_stream_delivery_tracking() - api_kwargs = self._build_api_kwargs(api_messages) - if self._force_ascii_payload: - _sanitize_structure_non_ascii(api_kwargs) - if self.api_mode == "codex_responses": - api_kwargs = self._get_transport().preflight_kwargs(api_kwargs, allow_stream=False) - - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "pre_api_request", - task_id=effective_task_id, - session_id=self.session_id or "", - platform=self.platform or "", - model=self.model, - provider=self.provider, - base_url=self.base_url, - api_mode=self.api_mode, - api_call_count=api_call_count, - message_count=len(api_messages), - tool_count=len(self.tools or []), - approx_input_tokens=approx_tokens, - request_char_count=total_chars, - max_tokens=self.max_tokens, - ) - except Exception: - pass - - if env_var_enabled("HERMES_DUMP_REQUESTS"): - self._dump_api_request_debug(api_kwargs, reason="preflight") - - # Always prefer the streaming path — even without stream - # consumers. Streaming gives us fine-grained health - # checking (90s stale-stream detection, 60s read timeout) - # that the non-streaming path lacks. Without this, - # subagents and other quiet-mode callers can hang - # indefinitely when the provider keeps the connection - # alive with SSE pings but never delivers a response. - # The streaming path is a no-op for callbacks when no - # consumers are registered, and falls back to non- - # streaming automatically if the provider doesn't - # support it. - def _stop_spinner(): - nonlocal thinking_spinner - if thinking_spinner: - thinking_spinner.stop("") - thinking_spinner = None - if self.thinking_callback: - self.thinking_callback("") - - _use_streaming = True - # Provider signaled "stream not supported" on a previous - # attempt — switch to non-streaming for the rest of this - # session instead of re-failing every retry. - if getattr(self, "_disable_streaming", False): - _use_streaming = False - # CopilotACPClient communicates via subprocess stdio and - # returns a plain SimpleNamespace — not an iterable - # stream. Mirror the ACP exclusion used for Responses - # API upgrade (lines ~1083-1085). - elif ( - self.provider == "copilot-acp" - or str(self.base_url or "").lower().startswith("acp://copilot") - or str(self.base_url or "").lower().startswith("acp+tcp://") - ): - _use_streaming = False - elif not self._has_stream_consumers(): - # No display/TTS consumer. Still prefer streaming for - # health checking, but skip for Mock clients in tests - # (mocks return SimpleNamespace, not stream iterators). - from unittest.mock import Mock - if isinstance(getattr(self, "client", None), Mock): - _use_streaming = False - - if _use_streaming: - response = self._interruptible_streaming_api_call( - api_kwargs, on_first_delta=_stop_spinner - ) - else: - response = self._interruptible_api_call(api_kwargs) - - api_duration = time.time() - api_start_time - - # Stop thinking spinner silently -- the response box or tool - # execution messages that follow are more informative. - if thinking_spinner: - thinking_spinner.stop("") - thinking_spinner = None - if self.thinking_callback: - self.thinking_callback("") - - if not self.quiet_mode: - self._vprint(f"{self.log_prefix}⏱️ API call completed in {api_duration:.2f}s") - - if self.verbose_logging: - # Log response with provider info if available - resp_model = getattr(response, 'model', 'N/A') if response else 'N/A' - logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}") - - # Validate response shape before proceeding - response_invalid = False - error_details = [] - if self.api_mode == "codex_responses": - _ct_v = self._get_transport() - if not _ct_v.validate_response(response): - if response is None: - response_invalid = True - error_details.append("response is None") - else: - # Provider returned a terminal failure (e.g. quota exhaustion). - # Treat as invalid so the fallback chain is triggered instead of - # letting the error bubble up outside the retry/fallback loop. - _codex_resp_status = str(getattr(response, "status", "") or "").strip().lower() - if _codex_resp_status in {"failed", "cancelled"}: - _codex_error_obj = getattr(response, "error", None) - _codex_error_msg = ( - _codex_error_obj.get("message") if isinstance(_codex_error_obj, dict) - else str(_codex_error_obj) if _codex_error_obj - else f"Responses API returned status '{_codex_resp_status}'" - ) - logging.warning( - "Codex response status='%s' (error=%s). Routing to fallback. %s", - _codex_resp_status, _codex_error_msg, - self._client_log_context(), - ) - response_invalid = True - error_details.append(f"response.status={_codex_resp_status}: {_codex_error_msg}") - else: - # output_text fallback: stream backfill may have failed - # but normalize can still recover from output_text - _out_text = getattr(response, "output_text", None) - _out_text_stripped = _out_text.strip() if isinstance(_out_text, str) else "" - if _out_text_stripped: - logger.debug( - "Codex response.output is empty but output_text is present " - "(%d chars); deferring to normalization.", - len(_out_text_stripped), - ) - else: - _resp_status = getattr(response, "status", None) - _resp_incomplete = getattr(response, "incomplete_details", None) - logger.warning( - "Codex response.output is empty after stream backfill " - "(status=%s, incomplete_details=%s, model=%s). %s", - _resp_status, _resp_incomplete, - getattr(response, "model", None), - f"api_mode={self.api_mode} provider={self.provider}", - ) - response_invalid = True - error_details.append("response.output is empty") - elif self.api_mode == "anthropic_messages": - _tv = self._get_transport() - if not _tv.validate_response(response): - response_invalid = True - if response is None: - error_details.append("response is None") - else: - error_details.append("response.content invalid (not a non-empty list)") - elif self.api_mode == "bedrock_converse": - _btv = self._get_transport() - if not _btv.validate_response(response): - response_invalid = True - if response is None: - error_details.append("response is None") - else: - error_details.append("Bedrock response invalid (no output or choices)") - else: - _ctv = self._get_transport() - if not _ctv.validate_response(response): - response_invalid = True - if response is None: - error_details.append("response is None") - elif not hasattr(response, 'choices'): - error_details.append("response has no 'choices' attribute") - elif response.choices is None: - error_details.append("response.choices is None") - else: - error_details.append("response.choices is empty") - - if response_invalid: - # Stop spinner before printing error messages - if thinking_spinner: - thinking_spinner.stop("(´;ω;`) oops, retrying...") - thinking_spinner = None - if self.thinking_callback: - self.thinking_callback("") - - # Invalid response — could be rate limiting, provider timeout, - # upstream server error, or malformed response. - retry_count += 1 - - # Eager fallback: empty/malformed responses are a common - # rate-limit symptom. Switch to fallback immediately - # rather than retrying with extended backoff. - if self._fallback_index < len(self._fallback_chain): - self._emit_status("⚠️ Empty/malformed response — switching to fallback...") - if self._try_activate_fallback(): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - - # Check for error field in response (some providers include this) - error_msg = "Unknown" - provider_name = "Unknown" - if response and hasattr(response, 'error') and response.error: - error_msg = str(response.error) - # Try to extract provider from error metadata - if hasattr(response.error, 'metadata') and response.error.metadata: - provider_name = response.error.metadata.get('provider_name', 'Unknown') - elif response and hasattr(response, 'message') and response.message: - error_msg = str(response.message) - - # Try to get provider from model field (OpenRouter often returns actual model used) - if provider_name == "Unknown" and response and hasattr(response, 'model') and response.model: - provider_name = f"model={response.model}" - - # Check for x-openrouter-provider or similar metadata - if provider_name == "Unknown" and response: - # Log all response attributes for debugging - resp_attrs = {k: str(v)[:100] for k, v in vars(response).items() if not k.startswith('_')} - if self.verbose_logging: - logging.debug(f"Response attributes for invalid response: {resp_attrs}") - - # Extract error code from response for contextual diagnostics - _resp_error_code = None - if response and hasattr(response, 'error') and response.error: - _code_raw = getattr(response.error, 'code', None) - if _code_raw is None and isinstance(response.error, dict): - _code_raw = response.error.get('code') - if _code_raw is not None: - try: - _resp_error_code = int(_code_raw) - except (TypeError, ValueError): - pass - - # Build a human-readable failure hint from the error code - # and response time, instead of always assuming rate limiting. - if _resp_error_code == 524: - _failure_hint = f"upstream provider timed out (Cloudflare 524, {api_duration:.0f}s)" - elif _resp_error_code == 504: - _failure_hint = f"upstream gateway timeout (504, {api_duration:.0f}s)" - elif _resp_error_code == 429: - _failure_hint = f"rate limited by upstream provider (429)" - elif _resp_error_code in {500, 502}: - _failure_hint = f"upstream server error ({_resp_error_code}, {api_duration:.0f}s)" - elif _resp_error_code in {503, 529}: - _failure_hint = f"upstream provider overloaded ({_resp_error_code})" - elif _resp_error_code is not None: - _failure_hint = f"upstream error (code {_resp_error_code}, {api_duration:.0f}s)" - elif api_duration < 10: - _failure_hint = f"fast response ({api_duration:.1f}s) — likely rate limited" - elif api_duration > 60: - _failure_hint = f"slow response ({api_duration:.0f}s) — likely upstream timeout" - else: - _failure_hint = f"response time {api_duration:.1f}s" - - self._vprint(f"{self.log_prefix}⚠️ Invalid API response (attempt {retry_count}/{max_retries}): {', '.join(error_details)}", force=True) - self._vprint(f"{self.log_prefix} 🏢 Provider: {provider_name}", force=True) - cleaned_provider_error = self._clean_error_message(error_msg) - self._vprint(f"{self.log_prefix} 📝 Provider message: {cleaned_provider_error}", force=True) - self._vprint(f"{self.log_prefix} ⏱️ {_failure_hint}", force=True) - - if retry_count >= max_retries: - # Try fallback before giving up - self._emit_status(f"⚠️ Max retries ({max_retries}) for invalid responses — trying fallback...") - if self._try_activate_fallback(): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - self._emit_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.") - logging.error(f"{self.log_prefix}Invalid API response after {max_retries} retries.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": f"Invalid API response after {max_retries} retries: {_failure_hint}", - "failed": True # Mark as failure for filtering - } - - # Backoff before retry — jittered exponential: 5s base, 120s cap - wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0) - self._vprint(f"{self.log_prefix}⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...", force=True) - logging.warning(f"Invalid API response (retry {retry_count}/{max_retries}): {', '.join(error_details)} | Provider: {provider_name}") - - # Sleep in small increments to stay responsive to interrupts - sleep_end = time.time() + wait_time - _backoff_touch_counter = 0 - while time.time() < sleep_end: - if self._interrupt_requested: - self._vprint(f"{self.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) - self._persist_session(messages, conversation_history) - self.clear_interrupt() - return { - "final_response": f"Operation interrupted during retry ({_failure_hint}, attempt {retry_count}/{max_retries}).", - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "interrupted": True, - } - time.sleep(0.2) - # Touch activity every ~30s so the gateway's inactivity - # monitor knows we're alive during backoff waits. - _backoff_touch_counter += 1 - if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s - self._touch_activity( - f"retry backoff ({retry_count}/{max_retries}), " - f"{int(sleep_end - time.time())}s remaining" - ) - continue # Retry the API call - - # Check finish_reason before proceeding - if self.api_mode == "codex_responses": - status = getattr(response, "status", None) - incomplete_details = getattr(response, "incomplete_details", None) - incomplete_reason = None - if isinstance(incomplete_details, dict): - incomplete_reason = incomplete_details.get("reason") - else: - incomplete_reason = getattr(incomplete_details, "reason", None) - if status == "incomplete" and incomplete_reason in {"max_output_tokens", "length"}: - finish_reason = "length" - else: - finish_reason = "stop" - elif self.api_mode == "anthropic_messages": - _tfr = self._get_transport() - finish_reason = _tfr.map_finish_reason(response.stop_reason) - elif self.api_mode == "bedrock_converse": - # Bedrock response already normalized at dispatch — use transport - _bt_fr = self._get_transport() - _bedrock_result = _bt_fr.normalize_response(response) - finish_reason = _bedrock_result.finish_reason - else: - _cc_fr = self._get_transport() - _finish_result = _cc_fr.normalize_response(response) - finish_reason = _finish_result.finish_reason - assistant_message = _finish_result - if self._should_treat_stop_as_truncated( - finish_reason, - assistant_message, - messages, - ): - self._vprint( - f"{self.log_prefix}⚠️ Treating suspicious Ollama/GLM stop response as truncated", - force=True, - ) - finish_reason = "length" - - if finish_reason == "length": - self._vprint(f"{self.log_prefix}⚠️ Response truncated (finish_reason='length') - model hit max output tokens", force=True) - - # Normalize the truncated response to a single OpenAI-style - # message shape so text-continuation and tool-call retry - # work uniformly across chat_completions, bedrock_converse, - # and anthropic_messages. For Anthropic we use the same - # adapter the agent loop already relies on so the rebuilt - # interim assistant message is byte-identical to what - # would have been appended in the non-truncated path. - _trunc_msg = None - _trunc_transport = self._get_transport() - if self.api_mode == "anthropic_messages": - _trunc_result = _trunc_transport.normalize_response( - response, strip_tool_prefix=self._is_anthropic_oauth - ) - else: - _trunc_result = _trunc_transport.normalize_response(response) - _trunc_msg = _trunc_result - - _trunc_content = getattr(_trunc_msg, "content", None) if _trunc_msg else None - _trunc_has_tool_calls = bool(getattr(_trunc_msg, "tool_calls", None)) if _trunc_msg else False - - # ── Detect thinking-budget exhaustion ────────────── - # When the model spends ALL output tokens on reasoning - # and has none left for the response, continuation - # retries are pointless. Detect this early and give a - # targeted error instead of wasting 3 API calls. - # A response is "thinking exhausted" only when the model - # actually produced reasoning blocks but no visible text after - # them. Models that do not use tags (e.g. GLM-4.7 on - # NVIDIA Build, minimax) may return content=None or an empty - # string for unrelated reasons — treat those as normal - # truncations that deserve continuation retries, not as - # thinking-budget exhaustion. - _has_think_tags = bool( - _trunc_content and re.search( - r'<(?:think|thinking|reasoning|REASONING_SCRATCHPAD)[^>]*>', - _trunc_content, - re.IGNORECASE, - ) - ) - _thinking_exhausted = ( - not _trunc_has_tool_calls - and _has_think_tags - and ( - (_trunc_content is not None and not self._has_content_after_think_block(_trunc_content)) - or _trunc_content is None - ) - ) - - if _thinking_exhausted: - _exhaust_error = ( - "Model used all output tokens on reasoning with none left " - "for the response. Try lowering reasoning effort or " - "increasing max_tokens." - ) - self._vprint( - f"{self.log_prefix}💭 Reasoning exhausted the output token budget — " - f"no visible response was produced.", - force=True, - ) - # Return a user-friendly message as the response so - # CLI (response box) and gateway (chat message) both - # display it naturally instead of a suppressed error. - _exhaust_response = ( - "⚠️ **Thinking Budget Exhausted**\n\n" - "The model used all its output tokens on reasoning " - "and had none left for the actual response.\n\n" - "To fix this:\n" - "→ Lower reasoning effort: `/thinkon low` or `/thinkon minimal`\n" - "→ Or switch to a larger/non-reasoning model with `/model`" - ) - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - return { - "final_response": _exhaust_response, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": _exhaust_error, - } - - if self.api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"}: - assistant_message = _trunc_msg - if assistant_message is not None and not _trunc_has_tool_calls: - length_continue_retries += 1 - interim_msg = self._build_assistant_message(assistant_message, finish_reason) - messages.append(interim_msg) - if assistant_message.content: - truncated_response_prefix += assistant_message.content - - if length_continue_retries < 3: - self._vprint( - f"{self.log_prefix}↻ Requesting continuation " - f"({length_continue_retries}/3)..." - ) - continue_msg = { - "role": "user", - "content": ( - "[System: Your previous response was truncated by the output " - "length limit. Continue exactly where you left off. Do not " - "restart or repeat prior text. Finish the answer directly.]" - ), - } - messages.append(continue_msg) - self._session_messages = messages - self._save_session_log(messages) - restart_with_length_continuation = True - break - - partial_response = self._strip_think_blocks(truncated_response_prefix).strip() - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - return { - "final_response": partial_response or None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Response remained truncated after 3 continuation attempts", - } - - if self.api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"}: - assistant_message = _trunc_msg - if assistant_message is not None and _trunc_has_tool_calls: - if truncated_tool_call_retries < 1: - truncated_tool_call_retries += 1 - self._vprint( - f"{self.log_prefix}⚠️ Truncated tool call detected — retrying API call...", - force=True, - ) - # Don't append the broken response to messages; - # just re-run the same API call from the current - # message state, giving the model another chance. - continue - self._vprint( - f"{self.log_prefix}⚠️ Truncated tool call response detected again — refusing to execute incomplete tool arguments.", - force=True, - ) - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Response truncated due to output length limit", - } - - # If we have prior messages, roll back to last complete state - if len(messages) > 1: - self._vprint(f"{self.log_prefix} ⏪ Rolling back to last complete assistant turn") - rolled_back_messages = self._get_messages_up_to_last_assistant(messages) - - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - - return { - "final_response": None, - "messages": rolled_back_messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Response truncated due to output length limit" - } - else: - # First message was truncated - mark as failed - self._vprint(f"{self.log_prefix}❌ First response truncated - cannot recover", force=True) - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "failed": True, - "error": "First response truncated due to output length limit" - } - - # Track actual token usage from response for context management - if hasattr(response, 'usage') and response.usage: - canonical_usage = normalize_usage( - response.usage, - provider=self.provider, - api_mode=self.api_mode, - ) - prompt_tokens = canonical_usage.prompt_tokens - completion_tokens = canonical_usage.output_tokens - total_tokens = canonical_usage.total_tokens - usage_dict = { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": total_tokens, - } - self.context_compressor.update_from_response(usage_dict) - - # Cache discovered context length after successful call. - # Only persist limits confirmed by the provider (parsed - # from the error message), not guessed probe tiers. - if getattr(self.context_compressor, "_context_probed", False): - ctx = self.context_compressor.context_length - if getattr(self.context_compressor, "_context_probe_persistable", False): - save_context_length(self.model, self.base_url, ctx) - self._safe_print(f"{self.log_prefix}💾 Cached context length: {ctx:,} tokens for {self.model}") - self.context_compressor._context_probed = False - self.context_compressor._context_probe_persistable = False - - self.session_prompt_tokens += prompt_tokens - self.session_completion_tokens += completion_tokens - self.session_total_tokens += total_tokens - self.session_api_calls += 1 - self.session_input_tokens += canonical_usage.input_tokens - self.session_output_tokens += canonical_usage.output_tokens - self.session_cache_read_tokens += canonical_usage.cache_read_tokens - self.session_cache_write_tokens += canonical_usage.cache_write_tokens - self.session_reasoning_tokens += canonical_usage.reasoning_tokens - - # Log API call details for debugging/observability - _cache_pct = "" - if canonical_usage.cache_read_tokens and prompt_tokens: - _cache_pct = f" cache={canonical_usage.cache_read_tokens}/{prompt_tokens} ({100*canonical_usage.cache_read_tokens/prompt_tokens:.0f}%)" - logger.info( - "API call #%d: model=%s provider=%s in=%d out=%d total=%d latency=%.1fs%s", - self.session_api_calls, self.model, self.provider or "unknown", - prompt_tokens, completion_tokens, total_tokens, - api_duration, _cache_pct, - ) - - cost_result = estimate_usage_cost( - self.model, - canonical_usage, - provider=self.provider, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - ) - if cost_result.amount_usd is not None: - self.session_estimated_cost_usd += float(cost_result.amount_usd) - self.session_cost_status = cost_result.status - self.session_cost_source = cost_result.source - - # Persist token counts to session DB for /insights. - # Do this for every platform with a session_id so non-CLI - # sessions (gateway, cron, delegated runs) cannot lose - # token/accounting data if a higher-level persistence path - # is skipped or fails. Gateway/session-store writes use - # absolute totals, so they safely overwrite these per-call - # deltas instead of double-counting them. - if self._session_db and self.session_id: - try: - # Ensure the session row exists before attempting UPDATE. - # Under concurrent load (cron/kanban), the initial - # _ensure_db_session() may have failed due to SQLite - # locking. Retry here so per-call token deltas are - # not silently lost (UPDATE on a non-existent row - # affects 0 rows without error). - if not self._session_db_created: - self._ensure_db_session() - self._session_db.update_token_counts( - self.session_id, - input_tokens=canonical_usage.input_tokens, - output_tokens=canonical_usage.output_tokens, - cache_read_tokens=canonical_usage.cache_read_tokens, - cache_write_tokens=canonical_usage.cache_write_tokens, - reasoning_tokens=canonical_usage.reasoning_tokens, - estimated_cost_usd=float(cost_result.amount_usd) - if cost_result.amount_usd is not None else None, - cost_status=cost_result.status, - cost_source=cost_result.source, - billing_provider=self.provider, - billing_base_url=self.base_url, - billing_mode="subscription_included" - if cost_result.status == "included" else None, - model=self.model, - api_call_count=1, - ) - except Exception as e: - # Log token persistence failures so they're - # visible in agent.log — silent loss here is - # the root cause of undercounted analytics. - logger.debug( - "Token persistence failed (session=%s, tokens=%d): %s", - self.session_id, total_tokens, e, - ) - - if self.verbose_logging: - logging.debug(f"Token usage: prompt={usage_dict['prompt_tokens']:,}, completion={usage_dict['completion_tokens']:,}, total={usage_dict['total_tokens']:,}") - - # Surface cache hit stats for any provider that reports - # them — not just those where we inject cache_control - # markers. OpenAI/Kimi/DeepSeek/Qwen all do automatic - # server-side prefix caching and return - # ``prompt_tokens_details.cached_tokens``; users - # previously could not see their cache % because this - # line was gated on ``_use_prompt_caching``, which is - # only True for Anthropic-style marker injection. - # ``canonical_usage`` is already normalised from all - # three API shapes (Anthropic / Codex / OpenAI-chat) - # so we can rely on its values directly. - cached = canonical_usage.cache_read_tokens - written = canonical_usage.cache_write_tokens - prompt = usage_dict["prompt_tokens"] - if (cached or written) and not self.quiet_mode: - hit_pct = (cached / prompt * 100) if prompt > 0 else 0 - self._vprint( - f"{self.log_prefix} 💾 Cache: " - f"{cached:,}/{prompt:,} tokens " - f"({hit_pct:.0f}% hit, {written:,} written)" - ) - - has_retried_429 = False # Reset on success - # Clear Nous rate limit state on successful request — - # proves the limit has reset and other sessions can - # resume hitting Nous. - if self.provider == "nous": - try: - from agent.nous_rate_guard import clear_nous_rate_limit - clear_nous_rate_limit() - except Exception: - pass - self._touch_activity(f"API call #{api_call_count} completed") - break # Success, exit retry loop - - except InterruptedError: - if thinking_spinner: - thinking_spinner.stop("") - thinking_spinner = None - if self.thinking_callback: - self.thinking_callback("") - api_elapsed = time.time() - api_start_time - self._vprint(f"{self.log_prefix}⚡ Interrupted during API call.", force=True) - self._persist_session(messages, conversation_history) - interrupted = True - final_response = f"Operation interrupted: waiting for model response ({api_elapsed:.1f}s elapsed)." - break - - except Exception as api_error: - # Stop spinner before printing error messages - if thinking_spinner: - thinking_spinner.stop("(╥_╥) error, retrying...") - thinking_spinner = None - if self.thinking_callback: - self.thinking_callback("") - - # ----------------------------------------------------------- - # UnicodeEncodeError recovery. Two common causes: - # 1. Lone surrogates (U+D800..U+DFFF) from clipboard paste - # (Google Docs, rich-text editors) — sanitize and retry. - # 2. ASCII codec on systems with LANG=C or non-UTF-8 locale - # (e.g. Chromebooks) — any non-ASCII character fails. - # Detect via the error message mentioning 'ascii' codec. - # We sanitize messages in-place and may retry twice: - # first to strip surrogates, then once more for pure - # ASCII-only locale sanitization if needed. - # ----------------------------------------------------------- - if isinstance(api_error, UnicodeEncodeError) and getattr(self, '_unicode_sanitization_passes', 0) < 2: - _err_str = str(api_error).lower() - _is_ascii_codec = "'ascii'" in _err_str or "ascii" in _err_str - # Detect surrogate errors — utf-8 codec refusing to - # encode U+D800..U+DFFF. The error text is: - # "'utf-8' codec can't encode characters in position - # N-M: surrogates not allowed" - _is_surrogate_error = ( - "surrogate" in _err_str - or ("'utf-8'" in _err_str and not _is_ascii_codec) - ) - # Sanitize surrogates from both the canonical `messages` - # list AND `api_messages` (the API-copy, which may carry - # `reasoning_content`/`reasoning_details` transformed - # from `reasoning` — fields the canonical list doesn't - # have directly). Also clean `api_kwargs` if built and - # `prefill_messages` if present. Mirrors the ASCII - # codec recovery below. - _surrogates_found = _sanitize_messages_surrogates(messages) - if isinstance(api_messages, list): - if _sanitize_messages_surrogates(api_messages): - _surrogates_found = True - if isinstance(api_kwargs, dict): - if _sanitize_structure_surrogates(api_kwargs): - _surrogates_found = True - if isinstance(getattr(self, "prefill_messages", None), list): - if _sanitize_messages_surrogates(self.prefill_messages): - _surrogates_found = True - # Gate the retry on the error type, not on whether we - # found anything — _force_ascii_payload / the extended - # surrogate walker above cover all known paths, but a - # new transformed field could still slip through. If - # the error was a surrogate encode failure, always let - # the retry run; the proactive sanitizer at line ~8781 - # runs again on the next iteration. Bounded by - # _unicode_sanitization_passes < 2 (outer guard). - if _surrogates_found or _is_surrogate_error: - self._unicode_sanitization_passes += 1 - if _surrogates_found: - self._vprint( - f"{self.log_prefix}⚠️ Stripped invalid surrogate characters from messages. Retrying...", - force=True, - ) - else: - self._vprint( - f"{self.log_prefix}⚠️ Surrogate encoding error — retrying after full-payload sanitization...", - force=True, - ) - continue - if _is_ascii_codec: - self._force_ascii_payload = True - # ASCII codec: the system encoding can't handle - # non-ASCII characters at all. Sanitize all - # non-ASCII content from messages/tool schemas and retry. - # Sanitize both the canonical `messages` list and - # `api_messages` (the API-copy built before the retry - # loop, which may contain extra fields like - # reasoning_content that are not in `messages`). - _messages_sanitized = _sanitize_messages_non_ascii(messages) - if isinstance(api_messages, list): - _sanitize_messages_non_ascii(api_messages) - # Also sanitize the last api_kwargs if already built, - # so a leftover non-ASCII value in a transformed field - # (e.g. extra_body, reasoning_content) doesn't survive - # into the next attempt via _build_api_kwargs cache paths. - if isinstance(api_kwargs, dict): - _sanitize_structure_non_ascii(api_kwargs) - _prefill_sanitized = False - if isinstance(getattr(self, "prefill_messages", None), list): - _prefill_sanitized = _sanitize_messages_non_ascii(self.prefill_messages) - - _tools_sanitized = False - if isinstance(getattr(self, "tools", None), list): - _tools_sanitized = _sanitize_tools_non_ascii(self.tools) - - _system_sanitized = False - if isinstance(active_system_prompt, str): - _sanitized_system = _strip_non_ascii(active_system_prompt) - if _sanitized_system != active_system_prompt: - active_system_prompt = _sanitized_system - self._cached_system_prompt = _sanitized_system - _system_sanitized = True - if isinstance(getattr(self, "ephemeral_system_prompt", None), str): - _sanitized_ephemeral = _strip_non_ascii(self.ephemeral_system_prompt) - if _sanitized_ephemeral != self.ephemeral_system_prompt: - self.ephemeral_system_prompt = _sanitized_ephemeral - _system_sanitized = True - - _headers_sanitized = False - _default_headers = ( - self._client_kwargs.get("default_headers") - if isinstance(getattr(self, "_client_kwargs", None), dict) - else None - ) - if isinstance(_default_headers, dict): - _headers_sanitized = _sanitize_structure_non_ascii(_default_headers) - - # Sanitize the API key — non-ASCII characters in - # credentials (e.g. ʋ instead of v from a bad - # copy-paste) cause httpx to fail when encoding - # the Authorization header as ASCII. This is the - # most common cause of persistent UnicodeEncodeError - # that survives message/tool sanitization (#6843). - _credential_sanitized = False - _raw_key = getattr(self, "api_key", None) or "" - if _raw_key: - _clean_key = _strip_non_ascii(_raw_key) - if _clean_key != _raw_key: - self.api_key = _clean_key - if isinstance(getattr(self, "_client_kwargs", None), dict): - self._client_kwargs["api_key"] = _clean_key - # Also update the live client — it holds its - # own copy of api_key which auth_headers reads - # dynamically on every request. - if getattr(self, "client", None) is not None and hasattr(self.client, "api_key"): - self.client.api_key = _clean_key - _credential_sanitized = True - self._vprint( - f"{self.log_prefix}⚠️ API key contained non-ASCII characters " - f"(bad copy-paste?) — stripped them. If auth fails, " - f"re-copy the key from your provider's dashboard.", - force=True, - ) - - # Always retry on ASCII codec detection — - # _force_ascii_payload guarantees the full - # api_kwargs payload is sanitized on the - # next iteration (line ~8475). Even when - # per-component checks above find nothing - # (e.g. non-ASCII only in api_messages' - # reasoning_content), the flag catches it. - # Bounded by _unicode_sanitization_passes < 2. - self._unicode_sanitization_passes += 1 - _any_sanitized = ( - _messages_sanitized - or _prefill_sanitized - or _tools_sanitized - or _system_sanitized - or _headers_sanitized - or _credential_sanitized - ) - if _any_sanitized: - self._vprint( - f"{self.log_prefix}⚠️ System encoding is ASCII — stripped non-ASCII characters from request payload. Retrying...", - force=True, - ) - else: - self._vprint( - f"{self.log_prefix}⚠️ System encoding is ASCII — enabling full-payload sanitization for retry...", - force=True, - ) - continue - - # ── Image-rejection recovery ────────────────────────────── - # Some providers (mlx-lm, text-only endpoints, text-only - # fallbacks on multimodal models) reject any message that - # contains image_url content with a 4xx error like - # "Only 'text' content type is supported." On first hit, - # strip all images from the message list, mark the session - # as vision-unsupported, and retry with text only. - # - # Detection is best-effort English phrase matching — a - # locale-translated or heavily-reworded upstream error - # will bypass this guard and fall through to the normal - # error handler. Expand the phrase list when new - # provider wordings are observed in the wild. - _err_body = "" - try: - _err_body = str(getattr(api_error, "body", None) or - getattr(api_error, "message", None) or - str(api_error)) - except Exception: - pass - _err_status = getattr(api_error, "status_code", None) - _IMAGE_REJECTION_PHRASES = ( - "only 'text' content type is supported", - "only text content type is supported", - "image_url is not supported", - "image content is not supported", - "multimodal is not supported", - "multimodal content is not supported", - "multimodal input is not supported", - "vision is not supported", - "vision input is not supported", - "does not support images", - "does not support image input", - "does not support multimodal", - "does not support vision", - "model does not support image", - # ChatGPT-account Codex backend - # (https://chatgpt.com/backend-api/codex) rejects - # data:image/...base64 URLs in input_image fields - # with HTTP 400 "Invalid 'input[N].content[K].image_url'. - # Expected a valid URL, but got a value with an - # invalid format." The OpenAI Responses API on the - # public endpoint accepts data URLs, but the - # ChatGPT-account variant does not. Without this - # phrase the agent cascaded into compression / - # context-too-large recovery instead of just - # stripping the images. Match is narrow on - # purpose — keyed on the field-path apostrophe so - # we don't false-trip on other URL validation - # errors. (issue #23570) - "image_url'. expected", - # DeepSeek's OpenAI-compatible API reports text-only - # request-body variants as: - # "unknown variant `image_url`, expected `text`". - "unknown variant `image_url`, expected `text`", - "unknown variant image_url, expected text", - ) - _err_lower = _err_body.lower() - _looks_like_image_rejection = any( - p in _err_lower for p in _IMAGE_REJECTION_PHRASES - ) - # 4xx-only gate: never interpret 5xx/timeout as "server - # said no to images" — those are transient and must - # route to the normal retry path. - _status_ok = _err_status is None or (400 <= int(_err_status) < 500) - if ( - getattr(self, "_vision_supported", True) - and _looks_like_image_rejection - and _status_ok - ): - self._vision_supported = False - _imgs_removed = _strip_images_from_messages(messages) - if isinstance(api_messages, list): - _strip_images_from_messages(api_messages) - self._vprint( - f"{self.log_prefix}⚠️ Server rejected image content — " - f"switching to text-only mode for this session" - + (". Stripped images from history and retrying." if _imgs_removed else "."), - force=True, - ) - continue - - status_code = getattr(api_error, "status_code", None) - error_context = self._extract_api_error_context(api_error) - - # ── Classify the error for structured recovery decisions ── - _compressor = getattr(self, "context_compressor", None) - _ctx_len = getattr(_compressor, "context_length", 200000) if _compressor else 200000 - classified = classify_api_error( - api_error, - provider=getattr(self, "provider", "") or "", - model=getattr(self, "model", "") or "", - approx_tokens=approx_tokens, - context_length=_ctx_len, - num_messages=len(api_messages) if api_messages else 0, - ) - logger.debug( - "Error classified: reason=%s status=%s retryable=%s compress=%s rotate=%s fallback=%s", - classified.reason.value, classified.status_code, - classified.retryable, classified.should_compress, - classified.should_rotate_credential, classified.should_fallback, - ) - - recovered_with_pool, has_retried_429 = self._recover_with_credential_pool( - status_code=status_code, - has_retried_429=has_retried_429, - classified_reason=classified.reason, - error_context=error_context, - ) - if recovered_with_pool: - continue - - # Image-too-large recovery: shrink oversized native image - # parts in-place and retry once. Triggered by Anthropic's - # per-image 5 MB ceiling (400 with "image exceeds 5 MB - # maximum") or any other provider that complains about - # image size. If shrink fails or a second attempt still - # fails, fall through to normal error handling. - if ( - classified.reason == FailoverReason.image_too_large - and not image_shrink_retry_attempted - ): - image_shrink_retry_attempted = True - if self._try_shrink_image_parts_in_messages(api_messages): - self._vprint( - f"{self.log_prefix}📐 Image(s) exceeded provider size limit — " - f"shrank and retrying...", - force=True, - ) - continue - else: - logger.info( - "image-shrink recovery: no data-URL image parts found " - "or shrink didn't reduce size; surfacing original error." - ) - - # Anthropic OAuth subscription rejected the 1M-context beta - # header ("long context beta is not yet available for this - # subscription"). Disable the beta for the rest of this - # session, rebuild the client, and retry once. 1M-capable - # subscriptions never hit this branch — they accept the - # beta and keep full 1M context. See PR #17680 for the - # original report (we chose reactive recovery over the - # proposed unconditional omit so capable subscriptions - # don't silently lose the capability). - if ( - classified.reason == FailoverReason.oauth_long_context_beta_forbidden - and self.api_mode == "anthropic_messages" - and self._is_anthropic_oauth - and not oauth_1m_beta_retry_attempted - ): - oauth_1m_beta_retry_attempted = True - if not getattr(self, "_oauth_1m_beta_disabled", False): - self._oauth_1m_beta_disabled = True - try: - self._anthropic_client.close() - except Exception: - pass - self._rebuild_anthropic_client() - self._vprint( - f"{self.log_prefix}🔕 OAuth subscription doesn't support " - f"the 1M-context beta — disabled for this session and retrying...", - force=True, - ) - continue - - if ( - self.api_mode == "codex_responses" - and self.provider == "openai-codex" - and status_code == 401 - and not codex_auth_retry_attempted - ): - codex_auth_retry_attempted = True - if self._try_refresh_codex_client_credentials(force=True): - self._vprint(f"{self.log_prefix}🔐 Codex auth refreshed after 401. Retrying request...") - continue - if ( - self.api_mode == "chat_completions" - and self.provider == "nous" - and status_code == 401 - and not nous_auth_retry_attempted - ): - nous_auth_retry_attempted = True - if self._try_refresh_nous_client_credentials(force=True): - print(f"{self.log_prefix}🔐 Nous agent key refreshed after 401. Retrying request...") - continue - # Credential refresh didn't help — show diagnostic info. - # Most common causes: Portal OAuth expired/revoked, - # account out of credits, or agent key blocked. - from hermes_constants import display_hermes_home as _dhh_fn - _dhh = _dhh_fn() - _body_text = "" - try: - _body = getattr(api_error, "body", None) or getattr(api_error, "response", None) - if _body is not None: - _body_text = str(_body)[:200] - except Exception: - pass - print(f"{self.log_prefix}🔐 Nous 401 — Portal authentication failed.") - if _body_text: - print(f"{self.log_prefix} Response: {_body_text}") - print(f"{self.log_prefix} Most likely: Portal OAuth expired, account out of credits, or agent key revoked.") - print(f"{self.log_prefix} Troubleshooting:") - print(f"{self.log_prefix} • Re-authenticate: hermes login --provider nous") - print(f"{self.log_prefix} • Check credits / billing: https://portal.nousresearch.com") - print(f"{self.log_prefix} • Verify stored credentials: {_dhh}/auth.json") - print(f"{self.log_prefix} • Switch providers temporarily: /model --provider openrouter") - if ( - self.provider == "copilot" - and status_code == 401 - and not copilot_auth_retry_attempted - ): - copilot_auth_retry_attempted = True - if self._try_refresh_copilot_client_credentials(): - self._vprint(f"{self.log_prefix}🔐 Copilot credentials refreshed after 401. Retrying request...") - continue - if ( - self.api_mode == "anthropic_messages" - and status_code == 401 - and hasattr(self, '_anthropic_api_key') - and not anthropic_auth_retry_attempted - ): - anthropic_auth_retry_attempted = True - from agent.anthropic_adapter import _is_oauth_token - if self._try_refresh_anthropic_client_credentials(): - print(f"{self.log_prefix}🔐 Anthropic credentials refreshed after 401. Retrying request...") - continue - # Credential refresh didn't help — show diagnostic info - key = self._anthropic_api_key - auth_method = "Bearer (OAuth/setup-token)" if _is_oauth_token(key) else "x-api-key (API key)" - print(f"{self.log_prefix}🔐 Anthropic 401 — authentication failed.") - print(f"{self.log_prefix} Auth method: {auth_method}") - print(f"{self.log_prefix} Token prefix: {key[:12]}..." if key and len(key) > 12 else f"{self.log_prefix} Token: (empty or short)") - print(f"{self.log_prefix} Troubleshooting:") - from hermes_constants import display_hermes_home as _dhh_fn - _dhh = _dhh_fn() - print(f"{self.log_prefix} • Check ANTHROPIC_TOKEN in {_dhh}/.env for Hermes-managed OAuth/setup tokens") - print(f"{self.log_prefix} • Check ANTHROPIC_API_KEY in {_dhh}/.env for API keys or legacy token values") - print(f"{self.log_prefix} • For API keys: verify at https://platform.claude.com/settings/keys") - print(f"{self.log_prefix} • For Claude Code: run 'claude /login' to refresh, then retry") - print(f"{self.log_prefix} • Legacy cleanup: hermes config set ANTHROPIC_TOKEN \"\"") - print(f"{self.log_prefix} • Clear stale keys: hermes config set ANTHROPIC_API_KEY \"\"") - - # ── Thinking block signature recovery ───────────────── - # Anthropic signs thinking blocks against the full turn - # content. Any upstream mutation (context compression, - # session truncation, message merging) invalidates the - # signature → HTTP 400. Recovery: strip reasoning_details - # from all messages so the next retry sends no thinking - # blocks at all. One-shot — don't retry infinitely. - if ( - classified.reason == FailoverReason.thinking_signature - and not thinking_sig_retry_attempted - ): - thinking_sig_retry_attempted = True - for _m in messages: - if isinstance(_m, dict): - _m.pop("reasoning_details", None) - self._vprint( - f"{self.log_prefix}⚠️ Thinking block signature invalid — " - f"stripped all thinking blocks, retrying...", - force=True, - ) - logging.warning( - "%sThinking block signature recovery: stripped " - "reasoning_details from %d messages", - self.log_prefix, len(messages), - ) - continue - - # ── llama.cpp grammar-parse recovery ────────────────── - # llama.cpp's ``json-schema-to-grammar`` converter rejects - # regex escape classes (``\d``, ``\w``, ``\s``) and most - # ``format`` values in tool schemas. MCP servers emit - # these routinely for date/phone/email params. Recovery: - # strip ``pattern``/``format`` from ``self.tools`` and - # retry once. We keep the keywords by default so cloud - # providers get the full prompting hints; this branch - # fires only for users on llama.cpp's OAI server. - if ( - classified.reason == FailoverReason.llama_cpp_grammar_pattern - and not llama_cpp_grammar_retry_attempted - ): - llama_cpp_grammar_retry_attempted = True - try: - from tools.schema_sanitizer import strip_pattern_and_format - _, _stripped = strip_pattern_and_format(self.tools) - except Exception as _strip_exc: # pragma: no cover — defensive - logging.warning( - "%sllama.cpp grammar recovery: strip helper failed: %s", - self.log_prefix, _strip_exc, - ) - _stripped = 0 - if _stripped: - self._vprint( - f"{self.log_prefix}⚠️ llama.cpp rejected tool schema grammar — " - f"stripped {_stripped} pattern/format keyword(s), retrying...", - force=True, - ) - logging.warning( - "%sllama.cpp grammar recovery: stripped %d " - "pattern/format keyword(s) from tool schemas", - self.log_prefix, _stripped, - ) - continue - # No keywords found to strip — fall through to normal - # retry path rather than loop forever on the same error. - logging.warning( - "%sllama.cpp grammar error but no pattern/format " - "keywords to strip — falling through to normal retry", - self.log_prefix, - ) - - retry_count += 1 - elapsed_time = time.time() - api_start_time - self._touch_activity( - f"API error recovery (attempt {retry_count}/{max_retries})" - ) - - error_type = type(api_error).__name__ - error_msg = str(api_error).lower() - _error_summary = self._summarize_api_error(api_error) - logger.warning( - "API call failed (attempt %s/%s) error_type=%s %s summary=%s", - retry_count, - max_retries, - error_type, - self._client_log_context(), - _error_summary, - ) - - _provider = getattr(self, "provider", "unknown") - _base = getattr(self, "base_url", "unknown") - _model = getattr(self, "model", "unknown") - _status_code_str = f" [HTTP {status_code}]" if status_code else "" - self._vprint(f"{self.log_prefix}⚠️ API call failed (attempt {retry_count}/{max_retries}): {error_type}{_status_code_str}", force=True) - self._vprint(f"{self.log_prefix} 🔌 Provider: {_provider} Model: {_model}", force=True) - self._vprint(f"{self.log_prefix} 🌐 Endpoint: {_base}", force=True) - self._vprint(f"{self.log_prefix} 📝 Error: {_error_summary}", force=True) - if status_code and status_code < 500: - _err_body = getattr(api_error, "body", None) - _err_body_str = str(_err_body)[:300] if _err_body else None - if _err_body_str: - self._vprint(f"{self.log_prefix} 📋 Details: {_err_body_str}", force=True) - self._vprint(f"{self.log_prefix} ⏱️ Elapsed: {elapsed_time:.2f}s Context: {len(api_messages)} msgs, ~{approx_tokens:,} tokens") - - # Actionable hint for OpenRouter "no tool endpoints" error. - # This fires regardless of whether fallback succeeds — the - # user needs to know WHY their model failed so they can fix - # their provider routing, not just silently fall back. - if ( - self._is_openrouter_url() - and "support tool use" in error_msg - ): - self._vprint( - f"{self.log_prefix} 💡 No OpenRouter providers for {_model} support tool calling with your current settings.", - force=True, - ) - if self.providers_allowed: - self._vprint( - f"{self.log_prefix} Your provider_routing.only restriction is filtering out tool-capable providers.", - force=True, - ) - self._vprint( - f"{self.log_prefix} Try removing the restriction or adding providers that support tools for this model.", - force=True, - ) - self._vprint( - f"{self.log_prefix} Check which providers support tools: https://openrouter.ai/models/{_model}", - force=True, - ) - - # Check for interrupt before deciding to retry - if self._interrupt_requested: - self._vprint(f"{self.log_prefix}⚡ Interrupt detected during error handling, aborting retries.", force=True) - self._persist_session(messages, conversation_history) - self.clear_interrupt() - return { - "final_response": f"Operation interrupted: handling API error ({error_type}: {self._clean_error_message(str(api_error))}).", - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "interrupted": True, - } - - # Check for 413 payload-too-large BEFORE generic 4xx handler. - # A 413 is a payload-size error — the correct response is to - # compress history and retry, not abort immediately. - status_code = getattr(api_error, "status_code", None) - - # ── Anthropic Sonnet long-context tier gate ─────────── - # Anthropic returns HTTP 429 "Extra usage is required for - # long context requests" when a Claude Max (or similar) - # subscription doesn't include the 1M-context tier. This - # is NOT a transient rate limit — retrying or switching - # credentials won't help. Reduce context to 200k (the - # standard tier) and compress. - if classified.reason == FailoverReason.long_context_tier: - _reduced_ctx = 200000 - compressor = self.context_compressor - old_ctx = compressor.context_length - if old_ctx > _reduced_ctx: - compressor.update_model( - model=self.model, - context_length=_reduced_ctx, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - ) - # Context probing flags — only set on built-in - # compressor (plugin engines manage their own). - if hasattr(compressor, "_context_probed"): - compressor._context_probed = True - # Don't persist — this is a subscription-tier - # limitation, not a model capability. If the - # user later enables extra usage the 1M limit - # should come back automatically. - compressor._context_probe_persistable = False - self._vprint( - f"{self.log_prefix}⚠️ Anthropic long-context tier " - f"requires extra usage — reducing context: " - f"{old_ctx:,} → {_reduced_ctx:,} tokens", - force=True, - ) - - compression_attempts += 1 - if compression_attempts <= max_compression_attempts: - original_len = len(messages) - messages, active_system_prompt = self._compress_context( - messages, system_message, - approx_tokens=approx_tokens, - task_id=effective_task_id, - ) - # Compression created a new session — clear history - # so _flush_messages_to_session_db writes compressed - # messages to the new session, not skipping them. - conversation_history = None - if len(messages) < original_len or old_ctx > _reduced_ctx: - self._emit_status( - f"🗜️ Context reduced to {_reduced_ctx:,} tokens " - f"(was {old_ctx:,}), retrying..." - ) - time.sleep(2) - restart_with_compressed_messages = True - break - # Fall through to normal error handling if compression - # is exhausted or didn't help. - - # Eager fallback for rate-limit errors (429 or quota exhaustion). - # When a fallback model is configured, switch immediately instead - # of burning through retries with exponential backoff -- the - # primary provider won't recover within the retry window. - is_rate_limited = classified.reason in { - FailoverReason.rate_limit, - FailoverReason.billing, - } - if is_rate_limited and self._fallback_index < len(self._fallback_chain): - # Don't eagerly fallback if credential pool rotation may - # still recover. See _pool_may_recover_from_rate_limit - # for the single-credential-pool and CloudCode-quota - # exceptions. Fixes #11314 and #13636. - pool_may_recover = _pool_may_recover_from_rate_limit( - self._credential_pool, - provider=self.provider, - base_url=getattr(self, "base_url", None), - ) - if not pool_may_recover: - self._emit_status("⚠️ Rate limited — switching to fallback provider...") - if self._try_activate_fallback(reason=classified.reason): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - - # ── Nous Portal: record rate limit & skip retries ───── - # When Nous returns a 429 that is a genuine account- - # level rate limit, record the reset time to a shared - # file so ALL sessions (cron, gateway, auxiliary) know - # not to pile on, then skip further retries -- each - # one burns another RPH request and deepens the hole. - # The retry loop's top-of-iteration guard will catch - # this on the next pass and try fallback or bail. - # - # IMPORTANT: Nous Portal multiplexes multiple upstream - # providers (DeepSeek, Kimi, MiMo, Hermes). A 429 can - # also mean an UPSTREAM provider is out of capacity - # for one specific model -- transient, clears in - # seconds, nothing to do with the caller's quota. - # Tripping the cross-session breaker on that would - # block every Nous model for minutes. We use - # ``is_genuine_nous_rate_limit`` to tell the two - # apart via the 429's own x-ratelimit-* headers and - # the last-known-good state captured on the previous - # successful response. - if ( - is_rate_limited - and self.provider == "nous" - and classified.reason == FailoverReason.rate_limit - and not recovered_with_pool - ): - _genuine_nous_rate_limit = False - try: - from agent.nous_rate_guard import ( - is_genuine_nous_rate_limit, - record_nous_rate_limit, - ) - _err_resp = getattr(api_error, "response", None) - _err_hdrs = ( - getattr(_err_resp, "headers", None) - if _err_resp else None - ) - _genuine_nous_rate_limit = is_genuine_nous_rate_limit( - headers=_err_hdrs, - last_known_state=self._rate_limit_state, - ) - if _genuine_nous_rate_limit: - record_nous_rate_limit( - headers=_err_hdrs, - error_context=error_context, - ) - else: - logging.info( - "Nous 429 looks like upstream capacity " - "(no exhausted bucket in headers or " - "last-known state) -- not tripping " - "cross-session breaker." - ) - except Exception: - pass - if _genuine_nous_rate_limit: - # Skip straight to max_retries -- the - # top-of-loop guard will handle fallback or - # bail cleanly. - retry_count = max_retries - continue - # Upstream capacity 429: fall through to normal - # retry logic. A different model (or the same - # model a moment later) will typically succeed. - - is_payload_too_large = ( - classified.reason == FailoverReason.payload_too_large - ) - - if is_payload_too_large: - compression_attempts += 1 - if compression_attempts > max_compression_attempts: - self._vprint(f"{self.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached for payload-too-large error.", force=True) - self._vprint(f"{self.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logging.error(f"{self.log_prefix}413 compression failed after {max_compression_attempts} attempts.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": f"Request payload too large: max compression attempts ({max_compression_attempts}) reached.", - "partial": True, - "failed": True, - "compression_exhausted": True, - } - self._emit_status(f"⚠️ Request payload too large (413) — compression attempt {compression_attempts}/{max_compression_attempts}...") - - original_len = len(messages) - messages, active_system_prompt = self._compress_context( - messages, system_message, approx_tokens=approx_tokens, - task_id=effective_task_id, - ) - # Compression created a new session — clear history - # so _flush_messages_to_session_db writes compressed - # messages to the new session, not skipping them. - conversation_history = None - - if len(messages) < original_len: - self._emit_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") - time.sleep(2) # Brief pause between compression retries - restart_with_compressed_messages = True - break - else: - self._vprint(f"{self.log_prefix}❌ Payload too large and cannot compress further.", force=True) - self._vprint(f"{self.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logging.error(f"{self.log_prefix}413 payload too large. Cannot compress further.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": "Request payload too large (413). Cannot compress further.", - "partial": True, - "failed": True, - "compression_exhausted": True, - } - - # Check for context-length errors BEFORE generic 4xx handler. - # The classifier detects context overflow from: explicit error - # messages, generic 400 + large session heuristic (#1630), and - # server disconnect + large session pattern (#2153). - is_context_length_error = ( - classified.reason == FailoverReason.context_overflow - ) - - if is_context_length_error: - compressor = self.context_compressor - old_ctx = compressor.context_length - - # ── Distinguish two very different errors ─────────── - # 1. "Prompt too long": the INPUT exceeds the context window. - # Fix: reduce context_length + compress history. - # 2. "max_tokens too large": input is fine, but - # input_tokens + requested max_tokens > context_window. - # Fix: reduce max_tokens (the OUTPUT cap) for this call. - # Do NOT shrink context_length — the window is unchanged. - # - # Note: max_tokens = output token cap (one response). - # context_length = total window (input + output combined). - available_out = parse_available_output_tokens_from_error(error_msg) - if available_out is not None: - # Error is purely about the output cap being too large. - # Cap output to the available space and retry without - # touching context_length or triggering compression. - safe_out = max(1, available_out - 64) # small safety margin - self._ephemeral_max_output_tokens = safe_out - self._vprint( - f"{self.log_prefix}⚠️ Output cap too large for current prompt — " - f"retrying with max_tokens={safe_out:,} " - f"(available_tokens={available_out:,}; context_length unchanged at {old_ctx:,})", - force=True, - ) - # Still count against compression_attempts so we don't - # loop forever if the error keeps recurring. - compression_attempts += 1 - if compression_attempts > max_compression_attempts: - self._vprint(f"{self.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) - self._vprint(f"{self.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logging.error(f"{self.log_prefix}Context compression failed after {max_compression_attempts} attempts.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached.", - "partial": True, - "failed": True, - "compression_exhausted": True, - } - restart_with_compressed_messages = True - break - - # Error is about the INPUT being too large — reduce context_length. - # Try to parse the actual limit from the error message - parsed_limit = parse_context_limit_from_error(error_msg) - _provider_lower = (getattr(self, "provider", "") or "").lower() - _base_lower = (getattr(self, "base_url", "") or "").rstrip("/").lower() - is_minimax_provider = ( - _provider_lower in {"minimax", "minimax-cn"} - or _base_lower.startswith(( - "https://api.minimax.io/anthropic", - "https://api.minimaxi.com/anthropic", - )) - ) - minimax_delta_only_overflow = ( - is_minimax_provider - and parsed_limit is None - and "context window exceeds limit (" in error_msg - ) - if parsed_limit and parsed_limit < old_ctx: - new_ctx = parsed_limit - self._vprint(f"{self.log_prefix}Context limit detected from API: {new_ctx:,} tokens (was {old_ctx:,})", force=True) - elif minimax_delta_only_overflow: - new_ctx = old_ctx - self._vprint( - f"{self.log_prefix}Provider reported overflow amount only; " - f"keeping context_length at {old_ctx:,} tokens and compressing.", - force=True, - ) - else: - # Step down to the next probe tier - new_ctx = get_next_probe_tier(old_ctx) - - if new_ctx and new_ctx < old_ctx: - compressor.update_model( - model=self.model, - context_length=new_ctx, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - ) - # Context probing flags — only set on built-in - # compressor (plugin engines manage their own). - if hasattr(compressor, "_context_probed"): - compressor._context_probed = True - # Only persist limits parsed from the provider's - # error message (a real number). Guessed fallback - # tiers from get_next_probe_tier() should stay - # in-memory only — persisting them pollutes the - # cache with wrong values. - compressor._context_probe_persistable = bool( - parsed_limit and parsed_limit == new_ctx - ) - self._vprint(f"{self.log_prefix}⚠️ Context length exceeded — stepping down: {old_ctx:,} → {new_ctx:,} tokens", force=True) - else: - self._vprint(f"{self.log_prefix}⚠️ Context length exceeded at minimum tier — attempting compression...", force=True) - - compression_attempts += 1 - if compression_attempts > max_compression_attempts: - self._vprint(f"{self.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) - self._vprint(f"{self.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logging.error(f"{self.log_prefix}Context compression failed after {max_compression_attempts} attempts.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached.", - "partial": True, - "failed": True, - "compression_exhausted": True, - } - self._emit_status(f"🗜️ Context too large (~{approx_tokens:,} tokens) — compressing ({compression_attempts}/{max_compression_attempts})...") - - original_len = len(messages) - messages, active_system_prompt = self._compress_context( - messages, system_message, approx_tokens=approx_tokens, - task_id=effective_task_id, - ) - # Compression created a new session — clear history - # so _flush_messages_to_session_db writes compressed - # messages to the new session, not skipping them. - conversation_history = None - - if len(messages) < original_len or new_ctx and new_ctx < old_ctx: - if len(messages) < original_len: - self._emit_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") - time.sleep(2) # Brief pause between compression retries - restart_with_compressed_messages = True - break - else: - # Can't compress further and already at minimum tier - self._vprint(f"{self.log_prefix}❌ Context length exceeded and cannot compress further.", force=True) - self._vprint(f"{self.log_prefix} 💡 The conversation has accumulated too much content. Try /new to start fresh, or /compress to manually trigger compression.", force=True) - logging.error(f"{self.log_prefix}Context length exceeded: {approx_tokens:,} tokens. Cannot compress further.") - self._persist_session(messages, conversation_history) - return { - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": f"Context length exceeded ({approx_tokens:,} tokens). Cannot compress further.", - "partial": True, - "failed": True, - "compression_exhausted": True, - } - - # Check for non-retryable client errors. The classifier - # already accounts for 413, 429, 529 (transient), context - # overflow, and generic-400 heuristics. Local validation - # errors (ValueError, TypeError) are programming bugs. - # Exclude UnicodeEncodeError — it's a ValueError subclass - # but is handled separately by the surrogate sanitization - # path above. Exclude json.JSONDecodeError — also a - # ValueError subclass, but it indicates a transient - # provider/network failure (malformed response body, - # truncated stream, routing layer corruption), not a - # local programming bug, and should be retried (#14782). - is_local_validation_error = ( - isinstance(api_error, (ValueError, TypeError)) - and not isinstance( - api_error, (UnicodeEncodeError, json.JSONDecodeError) - ) - # ssl.SSLError (and its subclass SSLCertVerificationError) - # inherits from OSError *and* ValueError via Python MRO, - # so the isinstance(ValueError) check above would - # misclassify a TLS transport failure as a local - # programming bug and abort without retrying. Exclude - # ssl.SSLError explicitly so the error classifier's - # retryable=True mapping takes effect instead. - and not isinstance(api_error, ssl.SSLError) - ) - is_client_error = ( - is_local_validation_error - or ( - not classified.retryable - and not classified.should_compress - and classified.reason not in { - FailoverReason.rate_limit, - FailoverReason.billing, - FailoverReason.overloaded, - FailoverReason.context_overflow, - FailoverReason.payload_too_large, - FailoverReason.long_context_tier, - FailoverReason.thinking_signature, - } - ) - ) and not is_context_length_error - - if is_client_error: - # Try fallback before aborting — a different provider - # may not have the same issue (rate limit, auth, etc.) - self._emit_status(f"⚠️ Non-retryable error (HTTP {status_code}) — trying fallback...") - if self._try_activate_fallback(): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - if api_kwargs is not None: - self._dump_api_request_debug( - api_kwargs, reason="non_retryable_client_error", error=api_error, - ) - self._emit_status( - f"❌ Non-retryable error (HTTP {status_code}): " - f"{self._summarize_api_error(api_error)}" - ) - self._vprint(f"{self.log_prefix}❌ Non-retryable client error (HTTP {status_code}). Aborting.", force=True) - self._vprint(f"{self.log_prefix} 🔌 Provider: {_provider} Model: {_model}", force=True) - self._vprint(f"{self.log_prefix} 🌐 Endpoint: {_base}", force=True) - # Actionable guidance for common auth errors - if classified.is_auth or classified.reason == FailoverReason.billing: - if _provider == "openai-codex" and status_code == 401: - self._vprint(f"{self.log_prefix} 💡 Codex OAuth token was rejected (HTTP 401). Your token may have been", force=True) - self._vprint(f"{self.log_prefix} refreshed by another client (Codex CLI, VS Code). To fix:", force=True) - self._vprint(f"{self.log_prefix} 1. Run `codex` in your terminal to generate fresh tokens.", force=True) - self._vprint(f"{self.log_prefix} 2. Then run `hermes auth` to re-authenticate.", force=True) - else: - self._vprint(f"{self.log_prefix} 💡 Your API key was rejected by the provider. Check:", force=True) - self._vprint(f"{self.log_prefix} • Is the key valid? Run: hermes setup", force=True) - self._vprint(f"{self.log_prefix} • Does your account have access to {_model}?", force=True) - if base_url_host_matches(str(_base), "openrouter.ai"): - self._vprint(f"{self.log_prefix} • Check credits: https://openrouter.ai/settings/credits", force=True) - else: - self._vprint(f"{self.log_prefix} 💡 This type of error won't be fixed by retrying.", force=True) - logging.error(f"{self.log_prefix}Non-retryable client error: {api_error}") - # Skip session persistence when the error is likely - # context-overflow related (status 400 + large session). - # Persisting the failed user message would make the - # session even larger, causing the same failure on the - # next attempt. (#1630) - if status_code == 400 and (approx_tokens > 50000 or len(api_messages) > 80): - self._vprint( - f"{self.log_prefix}⚠️ Skipping session persistence " - f"for large failed session to prevent growth loop.", - force=True, - ) - else: - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "failed": True, - "error": str(api_error), - } - - if retry_count >= max_retries: - # Before falling back, try rebuilding the primary - # client once for transient transport errors (stale - # connection pool, TCP reset). Only attempted once - # per API call block. - if not primary_recovery_attempted and self._try_recover_primary_transport( - api_error, retry_count=retry_count, max_retries=max_retries, - ): - primary_recovery_attempted = True - retry_count = 0 - continue - # Try fallback before giving up entirely - self._emit_status(f"⚠️ Max retries ({max_retries}) exhausted — trying fallback...") - if self._try_activate_fallback(): - retry_count = 0 - compression_attempts = 0 - primary_recovery_attempted = False - continue - _final_summary = self._summarize_api_error(api_error) - if is_rate_limited: - self._emit_status(f"❌ Rate limited after {max_retries} retries — {_final_summary}") - else: - self._emit_status(f"❌ API failed after {max_retries} retries — {_final_summary}") - self._vprint(f"{self.log_prefix} 💀 Final error: {_final_summary}", force=True) - - # Detect SSE stream-drop pattern (e.g. "Network - # connection lost") and surface actionable guidance. - # This typically happens when the model generates a - # very large tool call (write_file with huge content) - # and the proxy/CDN drops the stream mid-response. - _is_stream_drop = ( - not getattr(api_error, "status_code", None) - and any(p in error_msg for p in ( - "connection lost", "connection reset", - "connection closed", "network connection", - "network error", "terminated", - )) - ) - if _is_stream_drop: - self._vprint( - f"{self.log_prefix} 💡 The provider's stream " - f"connection keeps dropping. This often happens " - f"when the model tries to write a very large " - f"file in a single tool call.", - force=True, - ) - self._vprint( - f"{self.log_prefix} Try asking the model " - f"to use execute_code with Python's open() for " - f"large files, or to write the file in smaller " - f"sections.", - force=True, - ) - - logging.error( - "%sAPI call failed after %s retries. %s | provider=%s model=%s msgs=%s tokens=~%s", - self.log_prefix, max_retries, _final_summary, - _provider, _model, len(api_messages), f"{approx_tokens:,}", - ) - if api_kwargs is not None: - self._dump_api_request_debug( - api_kwargs, reason="max_retries_exhausted", error=api_error, - ) - self._persist_session(messages, conversation_history) - _final_response = f"API call failed after {max_retries} retries: {_final_summary}" - if _is_stream_drop: - _final_response += ( - "\n\nThe provider's stream connection keeps " - "dropping — this often happens when generating " - "very large tool call responses (e.g. write_file " - "with long content). Try asking me to use " - "execute_code with Python's open() for large " - "files, or to write in smaller sections." - ) - return { - "final_response": _final_response, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "failed": True, - "error": _final_summary, - } - - # For rate limits, respect the Retry-After header if present - _retry_after = None - if is_rate_limited: - _resp_headers = getattr(getattr(api_error, "response", None), "headers", None) - if _resp_headers and hasattr(_resp_headers, "get"): - _ra_raw = _resp_headers.get("retry-after") or _resp_headers.get("Retry-After") - if _ra_raw: - try: - _retry_after = min(float(_ra_raw), 120) # Cap at 2 minutes - except (TypeError, ValueError): - pass - wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0) - if is_rate_limited: - self._emit_status(f"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries})...") - else: - self._emit_status(f"⏳ Retrying in {wait_time:.1f}s (attempt {retry_count}/{max_retries})...") - logger.warning( - "Retrying API call in %ss (attempt %s/%s) %s error=%s", - wait_time, - retry_count, - max_retries, - self._client_log_context(), - api_error, - ) - # Sleep in small increments so we can respond to interrupts quickly - # instead of blocking the entire wait_time in one sleep() call - sleep_end = time.time() + wait_time - _backoff_touch_counter = 0 - while time.time() < sleep_end: - if self._interrupt_requested: - self._vprint(f"{self.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) - self._persist_session(messages, conversation_history) - self.clear_interrupt() - return { - "final_response": f"Operation interrupted: retrying API call after error (retry {retry_count}/{max_retries}).", - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "interrupted": True, - } - time.sleep(0.2) # Check interrupt every 200ms - # Touch activity every ~30s so the gateway's inactivity - # monitor knows we're alive during backoff waits. - _backoff_touch_counter += 1 - if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s - self._touch_activity( - f"error retry backoff ({retry_count}/{max_retries}), " - f"{int(sleep_end - time.time())}s remaining" - ) - - # If the API call was interrupted, skip response processing - if interrupted: - _turn_exit_reason = "interrupted_during_api_call" - break - - if restart_with_compressed_messages: - api_call_count -= 1 - self.iteration_budget.refund() - # Count compression restarts toward the retry limit to prevent - # infinite loops when compression reduces messages but not enough - # to fit the context window. - retry_count += 1 - restart_with_compressed_messages = False - continue - - if restart_with_length_continuation: - # Progressively boost the output token budget on each retry. - # Retry 1 → 2× base, retry 2 → 3× base, capped at 32 768. - # Applies to all providers via _ephemeral_max_output_tokens. - _boost_base = self.max_tokens if self.max_tokens else 4096 - _boost = _boost_base * (length_continue_retries + 1) - self._ephemeral_max_output_tokens = min(_boost, 32768) - continue - - # Guard: if all retries exhausted without a successful response - # (e.g. repeated context-length errors that exhausted retry_count), - # the `response` variable is still None. Break out cleanly. - if response is None: - _turn_exit_reason = "all_retries_exhausted_no_response" - print(f"{self.log_prefix}❌ All API retries exhausted with no successful response.") - self._persist_session(messages, conversation_history) - break - - try: - _transport = self._get_transport() - _normalize_kwargs = {} - if self.api_mode == "anthropic_messages": - _normalize_kwargs["strip_tool_prefix"] = self._is_anthropic_oauth - normalized = _transport.normalize_response(response, **_normalize_kwargs) - assistant_message = normalized - finish_reason = normalized.finish_reason - - # Normalize content to string — some OpenAI-compatible servers - # (llama-server, etc.) return content as a dict or list instead - # of a plain string, which crashes downstream .strip() calls. - if assistant_message.content is not None and not isinstance(assistant_message.content, str): - raw = assistant_message.content - if isinstance(raw, dict): - assistant_message.content = raw.get("text", "") or raw.get("content", "") or json.dumps(raw) - elif isinstance(raw, list): - # Multimodal content list — extract text parts - parts = [] - for part in raw: - if isinstance(part, str): - parts.append(part) - elif isinstance(part, dict) and part.get("type") == "text": - parts.append(part.get("text", "")) - elif isinstance(part, dict) and "text" in part: - parts.append(str(part["text"])) - assistant_message.content = "\n".join(parts) - else: - assistant_message.content = str(raw) - - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _assistant_tool_calls = getattr(assistant_message, "tool_calls", None) or [] - _assistant_text = assistant_message.content or "" - _invoke_hook( - "post_api_request", - task_id=effective_task_id, - session_id=self.session_id or "", - platform=self.platform or "", - model=self.model, - provider=self.provider, - base_url=self.base_url, - api_mode=self.api_mode, - api_call_count=api_call_count, - api_duration=api_duration, - finish_reason=finish_reason, - message_count=len(api_messages), - response_model=getattr(response, "model", None), - usage=self._usage_summary_for_api_request_hook(response), - assistant_content_chars=len(_assistant_text), - assistant_tool_call_count=len(_assistant_tool_calls), - ) - except Exception: - pass - - # Handle assistant response - if assistant_message.content and not self.quiet_mode: - if self.verbose_logging: - self._vprint(f"{self.log_prefix}🤖 Assistant: {assistant_message.content}") - else: - self._vprint(f"{self.log_prefix}🤖 Assistant: {assistant_message.content[:100]}{'...' if len(assistant_message.content) > 100 else ''}") - - # Notify progress callback of model's thinking (used by subagent - # delegation to relay the child's reasoning to the parent display). - if (assistant_message.content and self.tool_progress_callback): - _think_text = assistant_message.content.strip() - # Strip reasoning XML tags that shouldn't leak to parent display - _think_text = re.sub( - r'', '', _think_text - ).strip() - # For subagents: relay first line to parent display (existing behaviour). - # For all agents with a structured callback: emit reasoning.available event. - first_line = _think_text.split('\n')[0][:80] if _think_text else "" - if first_line and getattr(self, '_delegate_depth', 0) > 0: - try: - self.tool_progress_callback("_thinking", first_line) - except Exception: - pass - elif _think_text: - try: - self.tool_progress_callback("reasoning.available", "_thinking", _think_text[:500], None) - except Exception: - pass - - # Check for incomplete (opened but never closed) - # This means the model ran out of output tokens mid-reasoning — retry up to 2 times - if has_incomplete_scratchpad(assistant_message.content or ""): - self._incomplete_scratchpad_retries += 1 - - self._vprint(f"{self.log_prefix}⚠️ Incomplete detected (opened but never closed)") - - if self._incomplete_scratchpad_retries <= 2: - self._vprint(f"{self.log_prefix}🔄 Retrying API call ({self._incomplete_scratchpad_retries}/2)...") - # Don't add the broken message, just retry - continue - else: - # Max retries - discard this turn and save as partial - self._vprint(f"{self.log_prefix}❌ Max retries (2) for incomplete scratchpad. Saving as partial.", force=True) - self._incomplete_scratchpad_retries = 0 - - rolled_back_messages = self._get_messages_up_to_last_assistant(messages) - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - - return { - "final_response": None, - "messages": rolled_back_messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Incomplete REASONING_SCRATCHPAD after 2 retries" - } - - # Reset incomplete scratchpad counter on clean response - self._incomplete_scratchpad_retries = 0 - - if self.api_mode == "codex_responses" and finish_reason == "incomplete": - self._codex_incomplete_retries += 1 - - interim_msg = self._build_assistant_message(assistant_message, finish_reason) - interim_has_content = bool((interim_msg.get("content") or "").strip()) - interim_has_reasoning = bool(interim_msg.get("reasoning", "").strip()) if isinstance(interim_msg.get("reasoning"), str) else False - interim_has_codex_reasoning = bool(interim_msg.get("codex_reasoning_items")) - interim_has_codex_message_items = bool(interim_msg.get("codex_message_items")) - - if ( - interim_has_content - or interim_has_reasoning - or interim_has_codex_reasoning - or interim_has_codex_message_items - ): - last_msg = messages[-1] if messages else None - # Duplicate detection: two consecutive incomplete assistant - # messages with identical content AND reasoning are collapsed. - # For provider-state-only changes (encrypted reasoning - # items or replayable message ids/phases/statuses differ - # while visible content/reasoning are unchanged), compare - # those opaque payloads too so we don't silently drop the - # newer continuation state. - last_codex_items = last_msg.get("codex_reasoning_items") if isinstance(last_msg, dict) else None - interim_codex_items = interim_msg.get("codex_reasoning_items") - last_codex_message_items = last_msg.get("codex_message_items") if isinstance(last_msg, dict) else None - interim_codex_message_items = interim_msg.get("codex_message_items") - duplicate_interim = ( - isinstance(last_msg, dict) - and last_msg.get("role") == "assistant" - and last_msg.get("finish_reason") == "incomplete" - and (last_msg.get("content") or "") == (interim_msg.get("content") or "") - and (last_msg.get("reasoning") or "") == (interim_msg.get("reasoning") or "") - and last_codex_items == interim_codex_items - and last_codex_message_items == interim_codex_message_items - ) - if not duplicate_interim: - messages.append(interim_msg) - self._emit_interim_assistant_message(interim_msg) - - if self._codex_incomplete_retries < 3: - if not self.quiet_mode: - self._vprint(f"{self.log_prefix}↻ Codex response incomplete; continuing turn ({self._codex_incomplete_retries}/3)") - self._session_messages = messages - self._save_session_log(messages) - continue - - self._codex_incomplete_retries = 0 - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Codex response remained incomplete after 3 continuation attempts", - } - elif hasattr(self, "_codex_incomplete_retries"): - self._codex_incomplete_retries = 0 - - # Check for tool calls - if assistant_message.tool_calls: - if not self.quiet_mode: - self._vprint(f"{self.log_prefix}🔧 Processing {len(assistant_message.tool_calls)} tool call(s)...") - - if self.verbose_logging: - for tc in assistant_message.tool_calls: - logging.debug(f"Tool call: {tc.function.name} with args: {tc.function.arguments[:200]}...") - - # Validate tool call names - detect model hallucinations - # Repair mismatched tool names before validating - for tc in assistant_message.tool_calls: - if tc.function.name not in self.valid_tool_names: - repaired = self._repair_tool_call(tc.function.name) - if repaired: - print(f"{self.log_prefix}🔧 Auto-repaired tool name: '{tc.function.name}' -> '{repaired}'") - tc.function.name = repaired - invalid_tool_calls = [ - tc.function.name for tc in assistant_message.tool_calls - if tc.function.name not in self.valid_tool_names - ] - if invalid_tool_calls: - # Track retries for invalid tool calls - self._invalid_tool_retries += 1 - - # Return helpful error to model — model can self-correct next turn - available = ", ".join(sorted(self.valid_tool_names)) - invalid_name = invalid_tool_calls[0] - invalid_preview = invalid_name[:80] + "..." if len(invalid_name) > 80 else invalid_name - self._vprint(f"{self.log_prefix}⚠️ Unknown tool '{invalid_preview}' — sending error to model for self-correction ({self._invalid_tool_retries}/3)") - - if self._invalid_tool_retries >= 3: - self._vprint(f"{self.log_prefix}❌ Max retries (3) for invalid tool calls exceeded. Stopping as partial.", force=True) - self._invalid_tool_retries = 0 - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": f"Model generated invalid tool call: {invalid_preview}" - } - - assistant_msg = self._build_assistant_message(assistant_message, finish_reason) - messages.append(assistant_msg) - for tc in assistant_message.tool_calls: - if tc.function.name not in self.valid_tool_names: - content = f"Tool '{tc.function.name}' does not exist. Available tools: {available}" - else: - content = "Skipped: another tool call in this turn used an invalid name. Please retry this tool call." - messages.append({ - "role": "tool", - "name": tc.function.name, - "tool_call_id": tc.id, - "content": content, - }) - continue - # Reset retry counter on successful tool call validation - self._invalid_tool_retries = 0 - - # Validate tool call arguments are valid JSON - # Handle empty strings as empty objects (common model quirk) - invalid_json_args = [] - for tc in assistant_message.tool_calls: - args = tc.function.arguments - if isinstance(args, (dict, list)): - tc.function.arguments = json.dumps(args) - continue - if args is not None and not isinstance(args, str): - tc.function.arguments = str(args) - args = tc.function.arguments - # Treat empty/whitespace strings as empty object - if not args or not args.strip(): - tc.function.arguments = "{}" - continue - try: - json.loads(args) - except json.JSONDecodeError as e: - invalid_json_args.append((tc.function.name, str(e))) - - if invalid_json_args: - # Check if the invalid JSON is due to truncation rather - # than a model formatting mistake. Routers sometimes - # rewrite finish_reason from "length" to "tool_calls", - # hiding the truncation from the length handler above. - # Detect truncation: args that don't end with } or ] - # (after stripping whitespace) are cut off mid-stream. - _truncated = any( - not (tc.function.arguments or "").rstrip().endswith(("}", "]")) - for tc in assistant_message.tool_calls - if tc.function.name in {n for n, _ in invalid_json_args} - ) - if _truncated: - self._vprint( - f"{self.log_prefix}⚠️ Truncated tool call arguments detected " - f"(finish_reason={finish_reason!r}) — refusing to execute.", - force=True, - ) - self._invalid_json_retries = 0 - self._cleanup_task_resources(effective_task_id) - self._persist_session(messages, conversation_history) - return { - "final_response": None, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "partial": True, - "error": "Response truncated due to output length limit", - } - - # Track retries for invalid JSON arguments - self._invalid_json_retries += 1 - - tool_name, error_msg = invalid_json_args[0] - self._vprint(f"{self.log_prefix}⚠️ Invalid JSON in tool call arguments for '{tool_name}': {error_msg}") - - if self._invalid_json_retries < 3: - self._vprint(f"{self.log_prefix}🔄 Retrying API call ({self._invalid_json_retries}/3)...") - # Don't add anything to messages, just retry the API call - continue - else: - # Instead of returning partial, inject tool error results so the model can recover. - # Using tool results (not user messages) preserves role alternation. - self._vprint(f"{self.log_prefix}⚠️ Injecting recovery tool results for invalid JSON...") - self._invalid_json_retries = 0 # Reset for next attempt - - # Append the assistant message with its (broken) tool_calls - recovery_assistant = self._build_assistant_message(assistant_message, finish_reason) - messages.append(recovery_assistant) - - # Respond with tool error results for each tool call - invalid_names = {name for name, _ in invalid_json_args} - for tc in assistant_message.tool_calls: - if tc.function.name in invalid_names: - err = next(e for n, e in invalid_json_args if n == tc.function.name) - tool_result = ( - f"Error: Invalid JSON arguments. {err}. " - f"For tools with no required parameters, use an empty object: {{}}. " - f"Please retry with valid JSON." - ) - else: - tool_result = "Skipped: other tool call in this response had invalid JSON." - messages.append({ - "role": "tool", - "name": tc.function.name, - "tool_call_id": tc.id, - "content": tool_result, - }) - continue - - # Reset retry counter on successful JSON validation - self._invalid_json_retries = 0 - - # ── Post-call guardrails ────────────────────────── - assistant_message.tool_calls = self._cap_delegate_task_calls( - assistant_message.tool_calls - ) - assistant_message.tool_calls = self._deduplicate_tool_calls( - assistant_message.tool_calls - ) - - assistant_msg = self._build_assistant_message(assistant_message, finish_reason) - - # If this turn has both content AND tool_calls, capture the content - # as a fallback final response. Common pattern: model delivers its - # answer and calls memory/skill tools as a side-effect in the same - # turn. If the follow-up turn after tools is empty, we use this. - turn_content = assistant_message.content or "" - if turn_content and self._has_content_after_think_block(turn_content): - self._last_content_with_tools = turn_content - # Only mute subsequent output when EVERY tool call in - # this turn is post-response housekeeping (memory, todo, - # skill_manage, etc.). If any substantive tool is present - # (search_files, read_file, write_file, terminal, ...), - # keep output visible so the user sees progress. - _HOUSEKEEPING_TOOLS = frozenset({ - "memory", "todo", "skill_manage", "session_search", - }) - _all_housekeeping = all( - tc.function.name in _HOUSEKEEPING_TOOLS - for tc in assistant_message.tool_calls - ) - self._last_content_tools_all_housekeeping = _all_housekeeping - if _all_housekeeping and self._has_stream_consumers(): - self._mute_post_response = True - elif self._should_emit_quiet_tool_messages(): - clean = self._strip_think_blocks(turn_content).strip() - if clean: - self._vprint(f" ┊ 💬 {clean}") - - # Pop thinking-only prefill message(s) before appending - # (tool-call path — same rationale as the final-response path). - _had_prefill = False - while ( - messages - and isinstance(messages[-1], dict) - and messages[-1].get("_thinking_prefill") - ): - messages.pop() - _had_prefill = True - - # Reset prefill counter when tool calls follow a prefill - # recovery. Without this, the counter accumulates across - # the whole conversation — a model that intermittently - # empties (empty → prefill → tools → empty → prefill → - # tools) burns both prefill attempts and the third empty - # gets zero recovery. Resetting here treats each tool- - # call success as a fresh start. - if _had_prefill: - self._thinking_prefill_retries = 0 - self._empty_content_retries = 0 - # Successful tool execution — reset the post-tool nudge - # flag so it can fire again if the model goes empty on - # a LATER tool round. - self._post_tool_empty_retried = False - - messages.append(assistant_msg) - self._emit_interim_assistant_message(assistant_msg) - - # Close any open streaming display (response box, reasoning - # box) before tool execution begins. Intermediate turns may - # have streamed early content that opened the response box; - # flushing here prevents it from wrapping tool feed lines. - # Only signal the display callback — TTS (_stream_callback) - # should NOT receive None (it uses None as end-of-stream). - if self.stream_delta_callback: - try: - self.stream_delta_callback(None) - except Exception: - pass - - self._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count) - - if self._tool_guardrail_halt_decision is not None: - decision = self._tool_guardrail_halt_decision - _turn_exit_reason = "guardrail_halt" - final_response = self._toolguard_controlled_halt_response(decision) - self._emit_status( - f"⚠️ Tool guardrail halted {decision.tool_name}: {decision.code}" - ) - messages.append({"role": "assistant", "content": final_response}) - break - - # Reset per-turn retry counters after successful tool - # execution so a single truncation doesn't poison the - # entire conversation. - truncated_tool_call_retries = 0 - - # Signal that a paragraph break is needed before the next - # streamed text. We don't emit it immediately because - # multiple consecutive tool iterations would stack up - # redundant blank lines. Instead, _fire_stream_delta() - # will prepend a single "\n\n" the next time real text - # arrives. - self._stream_needs_break = True - - # Refund the iteration if the ONLY tool(s) called were - # execute_code (programmatic tool calling). These are - # cheap RPC-style calls that shouldn't eat the budget. - _tc_names = {tc.function.name for tc in assistant_message.tool_calls} - if _tc_names == {"execute_code"}: - self.iteration_budget.refund() - - # Use real token counts from the API response to decide - # compression. prompt_tokens + completion_tokens is the - # actual context size the provider reported plus the - # assistant turn — a tight lower bound for the next prompt. - # Tool results appended above aren't counted yet, but the - # threshold (default 50%) leaves ample headroom; if tool - # results push past it, the next API call will report the - # real total and trigger compression then. - # - # If last_prompt_tokens is 0 (stale after API disconnect - # or provider returned no usage data), fall back to rough - # estimate to avoid missing compression. Without this, - # a session can grow unbounded after disconnects because - # should_compress(0) never fires. (#2153) - _compressor = self.context_compressor - if _compressor.last_prompt_tokens > 0: - # Only use prompt_tokens — completion/reasoning - # tokens don't consume context window space. - # Thinking models (GLM-5.1, QwQ, DeepSeek R1) - # inflate completion_tokens with reasoning, - # causing premature compression. (#12026) - _real_tokens = _compressor.last_prompt_tokens - else: - # Include tool schemas — with 50+ tools enabled - # these add 20-30K tokens the messages-only - # estimate misses, which can skip compression - # past the configured threshold (#14695). - _real_tokens = estimate_request_tokens_rough( - messages, tools=self.tools or None - ) - - if self.compression_enabled and _compressor.should_compress(_real_tokens): - self._safe_print(" ⟳ compacting context…") - messages, active_system_prompt = self._compress_context( - messages, system_message, - approx_tokens=self.context_compressor.last_prompt_tokens, - task_id=effective_task_id, - ) - # Compression created a new session — clear history so - # _flush_messages_to_session_db writes compressed messages - # to the new session (see preflight compression comment). - conversation_history = None - - # Save session log incrementally (so progress is visible even if interrupted) - self._session_messages = messages - self._save_session_log(messages) - - # Continue loop for next response - continue - - else: - # No tool calls - this is the final response - final_response = assistant_message.content or "" - - # Fix: unmute output when entering the no-tool-call branch - # so the user can see empty-response warnings and recovery - # status messages. _mute_post_response was set during a - # prior housekeeping tool turn and should not silence the - # final response path. - self._mute_post_response = False - - # Check if response only has think block with no actual content after it - if not self._has_content_after_think_block(final_response): - # ── Partial stream recovery ───────────────────── - # If content was already streamed to the user before - # the connection died, use it as the final response - # instead of falling through to prior-turn fallback - # or wasting API calls on retries. - _partial_streamed = ( - getattr(self, "_current_streamed_assistant_text", "") or "" - ) - if self._has_content_after_think_block(_partial_streamed): - _turn_exit_reason = "partial_stream_recovery" - _recovered = self._strip_think_blocks(_partial_streamed).strip() - logger.info( - "Partial stream content delivered (%d chars) " - "— using as final response", - len(_recovered), - ) - self._emit_status( - "↻ Stream interrupted — using delivered content " - "as final response" - ) - final_response = _recovered - self._response_was_previewed = True - break - - # If the previous turn already delivered real content alongside - # HOUSEKEEPING tool calls (e.g. "You're welcome!" + memory save), - # the model has nothing more to say. Use the earlier content - # immediately instead of wasting API calls on retries. - # NOTE: Only use this shortcut when ALL tools in that turn were - # housekeeping (memory, todo, etc.). When substantive tools - # were called (terminal, search_files, etc.), the content was - # likely mid-task narration ("I'll scan the directory...") and - # the empty follow-up means the model choked — let the - # post-tool nudge below handle that instead of exiting early. - fallback = getattr(self, '_last_content_with_tools', None) - if fallback and getattr(self, '_last_content_tools_all_housekeeping', False): - _turn_exit_reason = "fallback_prior_turn_content" - logger.info("Empty follow-up after tool calls — using prior turn content as final response") - self._emit_status("↻ Empty response after tool calls — using earlier content as final answer") - self._last_content_with_tools = None - self._last_content_tools_all_housekeeping = False - self._empty_content_retries = 0 - # Do NOT modify the assistant message content — the - # old code injected "Calling the X tools..." which - # poisoned the conversation history. Just use the - # fallback text as the final response and break. - final_response = self._strip_think_blocks(fallback).strip() - self._response_was_previewed = True - break - - # ── Post-tool-call empty response nudge ─────────── - # The model returned empty after executing tool calls. - # This covers two cases: - # (a) No prior-turn content at all — model went silent - # (b) Prior turn had content + SUBSTANTIVE tools (the - # fallback above was skipped because the content - # was mid-task narration, not a final answer) - # Instead of giving up, nudge the model to continue by - # appending a user-level hint. This is the #9400 case: - # weaker models (mimo-v2-pro, GLM-5, etc.) sometimes - # return empty after tool results instead of continuing - # to the next step. One retry with a nudge usually - # fixes it. - _prior_was_tool = any( - m.get("role") == "tool" - for m in messages[-5:] # check recent messages - ) - # Detect Qwen3/Ollama-style in-content thinking blocks. - # Ollama puts in the content field (not in - # reasoning_content), so _has_structured below would - # miss it. We check here so thinking-only responses - # after tool calls route to prefill instead of nudge. - _has_inline_thinking = bool( - re.search( - r'||', - final_response or "", - re.IGNORECASE, - ) - ) - if ( - _prior_was_tool - and not getattr(self, "_post_tool_empty_retried", False) - and not _has_inline_thinking # thinking model still working — let prefill handle - ): - self._post_tool_empty_retried = True - # Clear stale narration so it doesn't resurface - # on a later empty response after the nudge. - self._last_content_with_tools = None - self._last_content_tools_all_housekeeping = False - logger.info( - "Empty response after tool calls — nudging model " - "to continue processing" - ) - self._emit_status( - "⚠️ Model returned empty after tool calls — " - "nudging to continue" - ) - # Append the empty assistant message first so the - # message sequence stays valid: - # tool(result) → assistant("(empty)") → user(nudge) - # Without this, we'd have tool → user which most - # APIs reject as an invalid sequence. - _nudge_msg = self._build_assistant_message(assistant_message, finish_reason) - _nudge_msg["content"] = "(empty)" - _nudge_msg["_empty_recovery_synthetic"] = True - messages.append(_nudge_msg) - messages.append({ - "role": "user", - "content": ( - "You just executed tool calls but returned an " - "empty response. Please process the tool " - "results above and continue with the task." - ), - "_empty_recovery_synthetic": True, - }) - continue - - # ── Thinking-only prefill continuation ────────── - # The model produced structured reasoning (via API - # fields) but no visible text content. Rather than - # giving up, append the assistant message as-is and - # continue — the model will see its own reasoning - # on the next turn and produce the text portion. - # Inspired by clawdbot's "incomplete-text" recovery. - # Also covers Qwen3/Ollama in-content blocks - # (detected above as _has_inline_thinking). - _has_structured = bool( - getattr(assistant_message, "reasoning", None) - or getattr(assistant_message, "reasoning_content", None) - or getattr(assistant_message, "reasoning_details", None) - or _has_inline_thinking - ) - if _has_structured and self._thinking_prefill_retries < 2: - self._thinking_prefill_retries += 1 - logger.info( - "Thinking-only response (no visible content) — " - "prefilling to continue (%d/2)", - self._thinking_prefill_retries, - ) - self._emit_status( - f"↻ Thinking-only response — prefilling to continue " - f"({self._thinking_prefill_retries}/2)" - ) - interim_msg = self._build_assistant_message( - assistant_message, "incomplete" - ) - interim_msg["_thinking_prefill"] = True - messages.append(interim_msg) - self._session_messages = messages - self._save_session_log(messages) - continue - - # ── Empty response retry ────────────────────── - # Model returned nothing usable. Retry up to 3 - # times before attempting fallback. This covers - # both truly empty responses (no content, no - # reasoning) AND reasoning-only responses after - # prefill exhaustion — models like mimo-v2-pro - # always populate reasoning fields via OpenRouter, - # so the old `not _has_structured` guard blocked - # retries for every reasoning model after prefill. - _truly_empty = not self._strip_think_blocks( - final_response - ).strip() - _prefill_exhausted = ( - _has_structured - and self._thinking_prefill_retries >= 2 - ) - if _truly_empty and (not _has_structured or _prefill_exhausted) and self._empty_content_retries < 3: - self._empty_content_retries += 1 - logger.warning( - "Empty response (no content or reasoning) — " - "retry %d/3 (model=%s)", - self._empty_content_retries, self.model, - ) - self._emit_status( - f"⚠️ Empty response from model — retrying " - f"({self._empty_content_retries}/3)" - ) - continue - - # ── Exhausted retries — try fallback provider ── - # Before giving up with "(empty)", attempt to - # switch to the next provider in the fallback - # chain. This covers the case where a model - # (e.g. GLM-4.5-Air) consistently returns empty - # due to context degradation or provider issues. - if _truly_empty and self._fallback_chain: - logger.warning( - "Empty response after %d retries — " - "attempting fallback (model=%s, provider=%s)", - self._empty_content_retries, self.model, - self.provider, - ) - self._emit_status( - "⚠️ Model returning empty responses — " - "switching to fallback provider..." - ) - if self._try_activate_fallback(): - self._empty_content_retries = 0 - self._emit_status( - f"↻ Switched to fallback: {self.model} " - f"({self.provider})" - ) - logger.info( - "Fallback activated after empty responses: " - "now using %s on %s", - self.model, self.provider, - ) - continue - - # Exhausted retries and fallback chain (or no - # fallback configured). Fall through to the - # "(empty)" terminal. - _turn_exit_reason = "empty_response_exhausted" - reasoning_text = self._extract_reasoning(assistant_message) - self._drop_trailing_empty_response_scaffolding(messages) - assistant_msg = self._build_assistant_message(assistant_message, finish_reason) - assistant_msg["content"] = "(empty)" - # This is a user-facing failure sentinel for the gateway, - # not real assistant content. Persisting it makes later - # "continue" turns replay assistant("(empty)") as if it - # were a meaningful model response, which can keep long - # tool-heavy sessions stuck in empty-response loops. - assistant_msg["_empty_terminal_sentinel"] = True - messages.append(assistant_msg) - - if reasoning_text: - reasoning_preview = reasoning_text[:500] + "..." if len(reasoning_text) > 500 else reasoning_text - logger.warning( - "Reasoning-only response (no visible content) " - "after exhausting retries and fallback. " - "Reasoning: %s", reasoning_preview, - ) - self._emit_status( - "⚠️ Model produced reasoning but no visible " - "response after all retries. Returning empty." - ) - else: - logger.warning( - "Empty response (no content or reasoning) " - "after %d retries. No fallback available. " - "model=%s provider=%s", - self._empty_content_retries, self.model, - self.provider, - ) - self._emit_status( - "❌ Model returned no content after all retries" - + (" and fallback attempts." if self._fallback_chain else - ". No fallback providers configured.") - ) - - final_response = "(empty)" - break - - # Reset retry counter/signature on successful content - self._empty_content_retries = 0 - self._thinking_prefill_retries = 0 - - if ( - self.api_mode == "codex_responses" - and self.valid_tool_names - and codex_ack_continuations < 2 - and self._looks_like_codex_intermediate_ack( - user_message=user_message, - assistant_content=final_response, - messages=messages, - ) - ): - codex_ack_continuations += 1 - interim_msg = self._build_assistant_message(assistant_message, "incomplete") - messages.append(interim_msg) - self._emit_interim_assistant_message(interim_msg) - - continue_msg = { - "role": "user", - "content": ( - "[System: Continue now. Execute the required tool calls and only " - "send your final answer after completing the task.]" - ), - } - messages.append(continue_msg) - self._session_messages = messages - self._save_session_log(messages) - continue - - codex_ack_continuations = 0 - - if truncated_response_prefix: - final_response = truncated_response_prefix + final_response - truncated_response_prefix = "" - length_continue_retries = 0 - - final_response = self._strip_think_blocks(final_response).strip() - - final_msg = self._build_assistant_message(assistant_message, finish_reason) - - # Pop thinking-only prefill and empty-response retry - # scaffolding before appending the final response. These - # internal turns are only for the next API retry and should - # not become durable transcript context. - while ( - messages - and isinstance(messages[-1], dict) - and ( - messages[-1].get("_thinking_prefill") - or messages[-1].get("_empty_recovery_synthetic") - or messages[-1].get("_empty_terminal_sentinel") - ) - ): - messages.pop() - - messages.append(final_msg) - - _turn_exit_reason = f"text_response(finish_reason={finish_reason})" - if not self.quiet_mode: - self._safe_print(f"🎉 Conversation completed after {api_call_count} OpenAI-compatible API call(s)") - break - - except Exception as e: - error_msg = f"Error during OpenAI-compatible API call #{api_call_count}: {str(e)}" - try: - print(f"❌ {error_msg}") - except (OSError, ValueError): - logger.error(error_msg) - - logger.debug("Outer loop error in API call #%d", api_call_count, exc_info=True) - - # If an assistant message with tool_calls was already appended, - # the API expects a role="tool" result for every tool_call_id. - # Fill in error results for any that weren't answered yet. - for idx in range(len(messages) - 1, -1, -1): - msg = messages[idx] - if not isinstance(msg, dict): - break - if msg.get("role") == "tool": - continue - if msg.get("role") == "assistant" and msg.get("tool_calls"): - answered_ids = { - m["tool_call_id"] - for m in messages[idx + 1:] - if isinstance(m, dict) and m.get("role") == "tool" - } - for tc in msg["tool_calls"]: - if not tc or not isinstance(tc, dict): continue - if tc["id"] not in answered_ids: - err_msg = { - "role": "tool", - "name": AIAgent._get_tool_call_name_static(tc), - "tool_call_id": tc["id"], - "content": f"Error executing tool: {error_msg}", - } - messages.append(err_msg) - break - - # Non-tool errors don't need a synthetic message injected. - # The error is already printed to the user (line above), and - # the retry loop continues. Injecting a fake user/assistant - # message pollutes history, burns tokens, and risks violating - # role-alternation invariants. - - # If we're near the limit, break to avoid infinite loops - if api_call_count >= self.max_iterations - 1: - _turn_exit_reason = f"error_near_max_iterations({error_msg[:80]})" - final_response = f"I apologize, but I encountered repeated errors: {error_msg}" - # Append as assistant so the history stays valid for - # session resume (avoids consecutive user messages). - messages.append({"role": "assistant", "content": final_response}) - break - - if final_response is None and ( - api_call_count >= self.max_iterations - or self.iteration_budget.remaining <= 0 - ): - # Budget exhausted — ask the model for a summary via one extra - # API call with tools stripped. _handle_max_iterations injects a - # user message and makes a single toolless request. - _turn_exit_reason = f"max_iterations_reached({api_call_count}/{self.max_iterations})" - self._emit_status( - f"⚠️ Iteration budget exhausted ({api_call_count}/{self.max_iterations}) " - "— asking model to summarise" - ) - if not self.quiet_mode: - self._safe_print( - f"\n⚠️ Iteration budget exhausted ({api_call_count}/{self.max_iterations}) " - "— requesting summary..." - ) - final_response = self._handle_max_iterations(messages, api_call_count) - - # If running as a kanban worker, block the task so the dispatcher - # knows the worker could not complete (rather than treating it as a - # protocol violation). The agent loop strips tools before calling - # _handle_max_iterations, so the model cannot call kanban_block - # itself — we must do it on its behalf. - _kanban_task = os.environ.get("HERMES_KANBAN_TASK") - if _kanban_task: - try: - handle_function_call( - "kanban_block", - { - "task_id": _kanban_task, - "reason": ( - f"Iteration budget exhausted " - f"({api_call_count}/{self.max_iterations}) — " - "task could not complete within the allowed " - "iterations" - ), - }, - task_id=effective_task_id, - ) - logger.info( - "kanban_block called for task %s after iteration " - "exhaustion (%d/%d)", - _kanban_task, api_call_count, self.max_iterations, - ) - except Exception: - logger.warning( - "Failed to call kanban_block after iteration " - "exhaustion for task %s", - _kanban_task, - exc_info=True, - ) - - # Determine if conversation completed successfully - completed = final_response is not None and api_call_count < self.max_iterations - - # Save trajectory if enabled. ``user_message`` may be a multimodal - # list of parts; the trajectory format wants a plain string. - self._save_trajectory(messages, _summarize_user_message_for_log(user_message), completed) - - # Clean up VM and browser for this task after conversation completes - self._cleanup_task_resources(effective_task_id) - - # Persist session to both JSON log and SQLite only after private retry - # scaffolding has been removed. Otherwise a later user "continue" turn - # can replay assistant("(empty)") / recovery nudges and fall into the - # same empty-response loop again. - self._drop_trailing_empty_response_scaffolding(messages) - self._persist_session(messages, conversation_history) - - # ── Turn-exit diagnostic log ───────────────────────────────────── - # Always logged at INFO so agent.log captures WHY every turn ended. - # When the last message is a tool result (agent was mid-work), log - # at WARNING — this is the "just stops" scenario users report. - _last_msg_role = messages[-1].get("role") if messages else None - _last_tool_name = None - if _last_msg_role == "tool": - # Walk back to find the assistant message with the tool call - for _m in reversed(messages): - if _m.get("role") == "assistant" and _m.get("tool_calls"): - _tcs = _m["tool_calls"] - if _tcs and isinstance(_tcs[0], dict): - _last_tool_name = _tcs[-1].get("function", {}).get("name") - break - - _turn_tool_count = sum( - 1 for m in messages - if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls") - ) - _resp_len = len(final_response) if final_response else 0 - _budget_used = self.iteration_budget.used if self.iteration_budget else 0 - _budget_max = self.iteration_budget.max_total if self.iteration_budget else 0 - - _diag_msg = ( - "Turn ended: reason=%s model=%s api_calls=%d/%d budget=%d/%d " - "tool_turns=%d last_msg_role=%s response_len=%d session=%s" - ) - _diag_args = ( - _turn_exit_reason, self.model, api_call_count, self.max_iterations, - _budget_used, _budget_max, - _turn_tool_count, _last_msg_role, _resp_len, - self.session_id or "none", - ) - - if _last_msg_role == "tool" and not interrupted: - # Agent was mid-work — this is the "just stops" case. - logger.warning( - "Turn ended with pending tool result (agent may appear stuck). " - + _diag_msg + " last_tool=%s", - *_diag_args, _last_tool_name, - ) - else: - logger.info(_diag_msg, *_diag_args) - - # File-mutation verifier footer. - # If one or more ``write_file`` / ``patch`` calls failed during this - # turn and were never superseded by a successful write to the same - # path, append an advisory footer to the assistant response. This - # catches the specific case — reported by Ben Eng (#15524-adjacent) - # — where a model issues a batch of parallel patches, half of them - # fail with "Could not find old_string", and the model summarises - # the turn claiming every file was edited. The user then has to - # manually run ``git status`` to catch the lie. With this footer - # the truth is surfaced on every turn, so over-claiming is - # structurally impossible past the model. - # - # Gate: only applied when a real text response exists for this - # turn and the user didn't interrupt. Empty/interrupted turns - # already have other surface text that shouldn't be augmented. - if final_response and not interrupted: - try: - _failed = getattr(self, "_turn_failed_file_mutations", None) or {} - if _failed and self._file_mutation_verifier_enabled(): - footer = self._format_file_mutation_failure_footer(_failed) - if footer: - final_response = final_response.rstrip() + "\n\n" + footer - except Exception as _ver_err: - logger.debug("file-mutation verifier footer failed: %s", _ver_err) - - # Plugin hook: transform_llm_output - # Fired once per turn after the tool-calling loop completes. - # Plugins can transform the LLM's output text before it's returned. - # First hook to return a string wins; None/empty return leaves text unchanged. - if final_response and not interrupted: - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _transform_results = _invoke_hook( - "transform_llm_output", - response_text=final_response, - session_id=self.session_id or "", - model=self.model, - platform=getattr(self, "platform", None) or "", - ) - for _hook_result in _transform_results: - if isinstance(_hook_result, str) and _hook_result: - final_response = _hook_result - break # First non-empty string wins - except Exception as exc: - logger.warning("transform_llm_output hook failed: %s", exc) - - # Plugin hook: post_llm_call - # Fired once per turn after the tool-calling loop completes. - # Plugins can use this to persist conversation data (e.g. sync - # to an external memory system). - if final_response and not interrupted: - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "post_llm_call", - session_id=self.session_id, - user_message=original_user_message, - assistant_response=final_response, - conversation_history=list(messages), - model=self.model, - platform=getattr(self, "platform", None) or "", - ) - except Exception as exc: - logger.warning("post_llm_call hook failed: %s", exc) - - # Extract reasoning from the CURRENT turn only. Walk backwards - # but stop at the user message that started this turn — anything - # earlier is from a prior turn and must not leak into the reasoning - # box (confusing stale display; #17055). Within the current turn - # we still want the *most recent* non-empty reasoning: many - # providers (Claude thinking, DeepSeek v4, Codex Responses) emit - # reasoning on the tool-call step and leave the final-answer step - # with reasoning=None, so picking only the last assistant would - # silently drop legitimate same-turn reasoning. - last_reasoning = None - for msg in reversed(messages): - if msg.get("role") == "user": - break # turn boundary — don't cross into prior turns - if msg.get("role") == "assistant" and msg.get("reasoning"): - last_reasoning = msg["reasoning"] - break - - # Build result with interrupt info if applicable - result = { - "final_response": final_response, - "last_reasoning": last_reasoning, - "messages": messages, - "api_calls": api_call_count, - "completed": completed, - "turn_exit_reason": _turn_exit_reason, - "partial": False, # True only when stopped due to invalid tool calls - "interrupted": interrupted, - "response_previewed": getattr(self, "_response_was_previewed", False), - "model": self.model, - "provider": self.provider, - "base_url": self.base_url, - "input_tokens": self.session_input_tokens, - "output_tokens": self.session_output_tokens, - "cache_read_tokens": self.session_cache_read_tokens, - "cache_write_tokens": self.session_cache_write_tokens, - "reasoning_tokens": self.session_reasoning_tokens, - "prompt_tokens": self.session_prompt_tokens, - "completion_tokens": self.session_completion_tokens, - "total_tokens": self.session_total_tokens, - "last_prompt_tokens": getattr(self.context_compressor, "last_prompt_tokens", 0) or 0, - "estimated_cost_usd": self.session_estimated_cost_usd, - "cost_status": self.session_cost_status, - "cost_source": self.session_cost_source, - } - if self._tool_guardrail_halt_decision is not None: - result["guardrail"] = self._tool_guardrail_halt_decision.to_metadata() - # If a /steer landed after the final assistant turn (no more tool - # batches to drain into), hand it back to the caller so it can be - # delivered as the next user turn instead of being silently lost. - _leftover_steer = self._drain_pending_steer() - if _leftover_steer: - result["pending_steer"] = _leftover_steer - self._response_was_previewed = False - - # Include interrupt message if one triggered the interrupt - if interrupted and self._interrupt_message: - result["interrupt_message"] = self._interrupt_message - - # Clear interrupt state after handling - self.clear_interrupt() - - # Clear stream callback so it doesn't leak into future calls - self._stream_callback = None - - # Check skill trigger NOW — based on how many tool iterations THIS turn used. - _should_review_skills = False - if (self._skill_nudge_interval > 0 - and self._iters_since_skill >= self._skill_nudge_interval - and "skill_manage" in self.valid_tool_names): - _should_review_skills = True - self._iters_since_skill = 0 - - # External memory provider: sync the completed turn + queue next prefetch. - self._sync_external_memory_for_turn( - original_user_message=original_user_message, - final_response=final_response, - interrupted=interrupted, - ) - - # Background memory/skill review — runs AFTER the response is delivered - # so it never competes with the user's task for model attention. - if final_response and not interrupted and (_should_review_memory or _should_review_skills): - try: - self._spawn_background_review( - messages_snapshot=list(messages), - review_memory=_should_review_memory, - review_skills=_should_review_skills, - ) - except Exception: - pass # Background review is best-effort - - # Note: Memory provider on_session_end() + shutdown_all() are NOT - # called here — run_conversation() is called once per user message in - # multi-turn sessions. Shutting down after every turn would kill the - # provider before the second message. Actual session-end cleanup is - # handled by the CLI (atexit / /reset) and gateway (session expiry / - # _reset_session). - - # Plugin hook: on_session_end - # Fired at the very end of every run_conversation call. - # Plugins can use this for cleanup, flushing buffers, etc. - try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_end", - session_id=self.session_id, - completed=completed, - interrupted=interrupted, - model=self.model, - platform=getattr(self, "platform", None) or "", - ) - except Exception as exc: - logger.warning("on_session_end hook failed: %s", exc) - - return result + """Forwarder — see ``agent.conversation_loop.run_conversation``.""" + from agent.conversation_loop import run_conversation + return run_conversation(self, user_message, system_message, conversation_history, task_id, stream_callback, persist_user_message) def chat(self, message: str, stream_callback: Optional[callable] = None) -> str: """ diff --git a/tests/run_agent/test_jsondecodeerror_retryable.py b/tests/run_agent/test_jsondecodeerror_retryable.py index 201521ddb22..e810092613e 100644 --- a/tests/run_agent/test_jsondecodeerror_retryable.py +++ b/tests/run_agent/test_jsondecodeerror_retryable.py @@ -75,7 +75,9 @@ class TestAgentLoopSourceStillHasCarveOut: def test_run_agent_excludes_jsondecodeerror_from_local_validation(self): import run_agent import inspect - src = inspect.getsource(run_agent) + from agent import conversation_loop + # The body moved into agent/conversation_loop.py; scan both for safety. + src = inspect.getsource(run_agent) + inspect.getsource(conversation_loop) # The predicate we care about must reference json.JSONDecodeError # in its exclusion tuple. We check for the specific co-occurrence # rather than the literal string so harmless reformatting doesn't diff --git a/tests/run_agent/test_memory_nudge_counter_hydration.py b/tests/run_agent/test_memory_nudge_counter_hydration.py index abf97d265a6..f3923f83442 100644 --- a/tests/run_agent/test_memory_nudge_counter_hydration.py +++ b/tests/run_agent/test_memory_nudge_counter_hydration.py @@ -120,10 +120,20 @@ def test_production_code_contains_hydration_block(): """Smoke test: confirm the hydration code is actually wired into run_conversation(). If someone deletes it, tests above still pass against the inline replica — this fails them awake. + + The body now lives in agent/conversation_loop.py after the + run_agent.py refactor; check both files for safety. """ from pathlib import Path - src = Path(__file__).resolve().parents[2] / "run_agent.py" - content = src.read_text(encoding="utf-8") + repo = Path(__file__).resolve().parents[2] + src_ra = (repo / "run_agent.py").read_text(encoding="utf-8") + src_cl = (repo / "agent" / "conversation_loop.py").read_text(encoding="utf-8") + content = src_ra + src_cl # Anchor on the unique comment + the modulo line. assert "Hydrate per-session nudge counters from persisted history" in content - assert "self._turns_since_memory = prior_user_turns % self._memory_nudge_interval" in content + # The line uses ``self.`` in run_agent.py form and ``agent.`` in the + # extracted module, accept either. + assert ( + "self._turns_since_memory = prior_user_turns % self._memory_nudge_interval" in content + or "agent._turns_since_memory = prior_user_turns % agent._memory_nudge_interval" in content + ) diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index eb5efcafca7..76254d4eda5 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -5205,14 +5205,19 @@ class TestMemoryNudgeCounterPersistence: def test_counters_not_reset_in_preamble(self): """The run_conversation preamble must not zero the nudge counters.""" import inspect - src = inspect.getsource(AIAgent.run_conversation) + from agent.conversation_loop import run_conversation as _rc + src = inspect.getsource(_rc) # The preamble resets many fields (retry counts, budget, etc.) # before the main loop. Find that reset block and verify our # counters aren't in it. The reset block ends at iteration_budget. - preamble_end = src.index("self.iteration_budget = IterationBudget") + # After the run_agent.py refactor the body uses ``agent.X`` instead + # of ``self.X``, so accept either form. + preamble_end = src.index("iteration_budget = IterationBudget") preamble = src[:preamble_end] assert "self._turns_since_memory = 0" not in preamble assert "self._iters_since_skill = 0" not in preamble + assert "agent._turns_since_memory = 0" not in preamble + assert "agent._iters_since_skill = 0" not in preamble class TestDeadRetryCode: @@ -5220,7 +5225,8 @@ class TestDeadRetryCode: def test_no_unreachable_max_retries_after_backoff(self): import inspect - source = inspect.getsource(AIAgent.run_conversation) + from agent.conversation_loop import run_conversation as _rc + source = inspect.getsource(_rc) occurrences = source.count("if retry_count >= max_retries:") assert occurrences == 2, ( f"Expected 2 occurrences of 'if retry_count >= max_retries:' " @@ -5258,7 +5264,8 @@ class TestMemoryContextSanitization: a literal tag we don't silently delete their text. The streaming scrubber + plugin-side scrub cover real leak paths.""" import inspect - src = inspect.getsource(AIAgent.run_conversation) + from agent.conversation_loop import run_conversation as _rc + src = inspect.getsource(_rc) assert "sanitize_context(user_message)" not in src assert "sanitize_context(persist_user_message)" not in src @@ -5294,7 +5301,8 @@ class TestMemoryProviderTurnStart: def test_on_turn_start_called_before_prefetch(self): """Source-level check: on_turn_start appears before prefetch_all in run_conversation.""" import inspect - src = inspect.getsource(AIAgent.run_conversation) + from agent.conversation_loop import run_conversation as _rc + src = inspect.getsource(_rc) # Find the actual method calls, not comments idx_turn_start = src.index(".on_turn_start(") idx_prefetch = src.index(".prefetch_all(") @@ -5304,7 +5312,13 @@ class TestMemoryProviderTurnStart: ) def test_on_turn_start_uses_user_turn_count(self): - """Source-level check: on_turn_start receives self._user_turn_count.""" + """Source-level check: on_turn_start receives the user_turn_count.""" import inspect - src = inspect.getsource(AIAgent.run_conversation) - assert "on_turn_start(self._user_turn_count" in src + from agent.conversation_loop import run_conversation as _rc + src = inspect.getsource(_rc) + # After the run_agent.py refactor the body uses ``agent.X`` instead + # of ``self.X``. Accept either spelling. + assert ( + "on_turn_start(self._user_turn_count" in src + or "on_turn_start(agent._user_turn_count" in src + ) From 9f408989c40c2f1ca5830bd571eb5e1701cad0ce Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 19:43:38 -0700 Subject: [PATCH 014/142] refactor(run_agent): extract __init__ (1,381 LOC) to agent/agent_init.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The largest method left on AIAgent (60+ parameters, the entire startup sequence — credential resolution, provider auto-detection, context engine bootstrap, memory store hydration, plugin lifecycle hooks) moves into agent/agent_init.py. AIAgent.__init__ is now a thin wrapper that calls agent.agent_init.init_agent(self, ...) with the original full parameter list preserved. Module-level run_agent names referenced in the body (_openrouter_prewarm_done, _qwen_portal_headers, _routermint_headers, _hermes_home, OpenAI, get_tool_definitions, check_toolset_requirements) are resolved through _ra() so test patches on those names keep working. agent_init's logger warnings are routed via _ra().logger so tests patching run_agent.logger capture them (TestStringKSuffixContextLengthWarns, TestCustomProvidersInvalidContextLengthWarns). Live E2E reconfirmed on three model paths (openai/gpt-5.4, anthropic/claude-sonnet-4.6, moonshotai/kimi-k2-thinking). tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure). run_agent.py: 5944 -> 4564 lines (-1380). Total reduction since baseline: 16083 -> 4564 (-11519, 72%). --- agent/agent_init.py | 1457 +++++++++++++++++++++++++++++++++++++++++++ run_agent.py | 1316 +------------------------------------- 2 files changed, 1460 insertions(+), 1313 deletions(-) create mode 100644 agent/agent_init.py diff --git a/agent/agent_init.py b/agent/agent_init.py new file mode 100644 index 00000000000..acae61487c4 --- /dev/null +++ b/agent/agent_init.py @@ -0,0 +1,1457 @@ +"""Implementation of :meth:`AIAgent.__init__` — extracted as a module function. + +``AIAgent.__init__`` is one of the longest methods in the codebase (60+ +parameters, ~1,400 lines of attribute initialization, provider +auto-detection, credential resolution, context-engine bootstrap, etc.). +Keeping it in ``run_agent.py`` bloats that file with code that's mostly +"setup state, then forget". + +After this extraction the body lives here as ``init_agent(agent, ...)`` +and :meth:`AIAgent.__init__` is a thin wrapper that calls +``init_agent(self, ...)``. All imports the body needs at module-load +time are listed below; the body also performs many lazy imports inside +its own scope that come along unchanged. + +Symbols that tests patch on ``run_agent.*`` (``OpenAI``, ``cleanup_vm``, +etc.) are resolved through :func:`_ra` so the patch contract is +preserved. +""" + +from __future__ import annotations + +import logging +import os +import re +import sys +import threading +import time +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse, parse_qs, urlunparse + +from agent.context_compressor import ContextCompressor +from agent.iteration_budget import IterationBudget +from agent.memory_manager import StreamingContextScrubber +from agent.model_metadata import ( + MINIMUM_CONTEXT_LENGTH, + fetch_model_metadata, + get_model_context_length, + is_local_endpoint, + query_ollama_num_ctx, +) +from agent.process_bootstrap import _install_safe_stdio +from agent.subdirectory_hints import SubdirectoryHintTracker +from agent.think_scrubber import StreamingThinkScrubber +from agent.tool_guardrails import ( + ToolCallGuardrailConfig, + ToolCallGuardrailController, + ToolGuardrailDecision, +) +from hermes_cli.config import cfg_get +from hermes_cli.timeouts import get_provider_request_timeout +from hermes_constants import get_hermes_home +from model_tools import check_toolset_requirements, get_tool_definitions +from utils import base_url_host_matches + +# Use the same logger name as run_agent so tests patching ``run_agent.logger`` +# capture our warnings. (run_agent.py also does +# ``logger = logging.getLogger(__name__)``, which resolves to "run_agent" +# from inside that module.) +logger = logging.getLogger("run_agent") + + +def _ra(): + """Lazy reference to ``run_agent`` so callers can patch + ``run_agent.OpenAI`` / ``run_agent.cleanup_vm`` / ... and have those + patches reach this code path. + """ + import run_agent + return run_agent + + +def init_agent( + agent, + base_url: str = None, + api_key: str = None, + provider: str = None, + api_mode: str = None, + acp_command: str = None, + acp_args: list[str] | None = None, + command: str = None, + args: list[str] | None = None, + model: str = "", + max_iterations: int = 90, # Default tool-calling iterations (shared with subagents) + tool_delay: float = 1.0, + enabled_toolsets: List[str] = None, + disabled_toolsets: List[str] = None, + save_trajectories: bool = False, + verbose_logging: bool = False, + quiet_mode: bool = False, + ephemeral_system_prompt: str = None, + log_prefix_chars: int = 100, + log_prefix: str = "", + providers_allowed: List[str] = None, + providers_ignored: List[str] = None, + providers_order: List[str] = None, + provider_sort: str = None, + provider_require_parameters: bool = False, + provider_data_collection: str = None, + openrouter_min_coding_score: Optional[float] = None, + session_id: str = None, + tool_progress_callback: callable = None, + tool_start_callback: callable = None, + tool_complete_callback: callable = None, + thinking_callback: callable = None, + reasoning_callback: callable = None, + clarify_callback: callable = None, + step_callback: callable = None, + stream_delta_callback: callable = None, + interim_assistant_callback: callable = None, + tool_gen_callback: callable = None, + status_callback: callable = None, + max_tokens: int = None, + reasoning_config: Dict[str, Any] = None, + service_tier: str = None, + request_overrides: Dict[str, Any] = None, + prefill_messages: List[Dict[str, Any]] = None, + platform: str = None, + user_id: str = None, + user_name: str = None, + chat_id: str = None, + chat_name: str = None, + chat_type: str = None, + thread_id: str = None, + gateway_session_key: str = None, + skip_context_files: bool = False, + load_soul_identity: bool = False, + skip_memory: bool = False, + session_db=None, + parent_session_id: str = None, + iteration_budget: "IterationBudget" = None, + fallback_model: Dict[str, Any] = None, + credential_pool=None, + checkpoints_enabled: bool = False, + checkpoint_max_snapshots: int = 20, + checkpoint_max_total_size_mb: int = 500, + checkpoint_max_file_size_mb: int = 10, + pass_session_id: bool = False, +): + """ + Initialize the AI Agent. + + Args: + base_url (str): Base URL for the model API (optional) + api_key (str): API key for authentication (optional, uses env var if not provided) + provider (str): Provider identifier (optional; used for telemetry/routing hints) + api_mode (str): API mode override: "chat_completions" or "codex_responses" + model (str): Model name to use (default: "anthropic/claude-opus-4.6") + max_iterations (int): Maximum number of tool calling iterations (default: 90) + tool_delay (float): Delay between tool calls in seconds (default: 1.0) + enabled_toolsets (List[str]): Only enable tools from these toolsets (optional) + disabled_toolsets (List[str]): Disable tools from these toolsets (optional) + save_trajectories (bool): Whether to save conversation trajectories to JSONL files (default: False) + verbose_logging (bool): Enable verbose logging for debugging (default: False) + quiet_mode (bool): Suppress progress output for clean CLI experience (default: False) + ephemeral_system_prompt (str): System prompt used during agent execution but NOT saved to trajectories (optional) + log_prefix_chars (int): Number of characters to show in log previews for tool calls/responses (default: 100) + log_prefix (str): Prefix to add to all log messages for identification in parallel processing (default: "") + providers_allowed (List[str]): OpenRouter providers to allow (optional) + providers_ignored (List[str]): OpenRouter providers to ignore (optional) + providers_order (List[str]): OpenRouter providers to try in order (optional) + provider_sort (str): Sort providers by price/throughput/latency (optional) + openrouter_min_coding_score (float): Coding-score floor (0.0-1.0) for the + openrouter/pareto-code router. Only applied when model == "openrouter/pareto-code". + None or empty = let OpenRouter pick the strongest available coder. + session_id (str): Pre-generated session ID for logging (optional, auto-generated if not provided) + tool_progress_callback (callable): Callback function(tool_name, args_preview) for progress notifications + clarify_callback (callable): Callback function(question, choices) -> str for interactive user questions. + Provided by the platform layer (CLI or gateway). If None, the clarify tool returns an error. + max_tokens (int): Maximum tokens for model responses (optional, uses model default if not set) + reasoning_config (Dict): OpenRouter reasoning configuration override (e.g. {"effort": "none"} to disable thinking). + If None, defaults to {"enabled": True, "effort": "medium"} for OpenRouter. Set to disable/customize reasoning. + prefill_messages (List[Dict]): Messages to prepend to conversation history as prefilled context. + Useful for injecting a few-shot example or priming the model's response style. + Example: [{"role": "user", "content": "Hi!"}, {"role": "assistant", "content": "Hello!"}] + NOTE: Anthropic Sonnet 4.6+ and Opus 4.6+ reject a conversation that ends on an + assistant-role message (400 error). For those models use structured outputs or + output_config.format instead of a trailing-assistant prefill. + platform (str): The interface platform the user is on (e.g. "cli", "telegram", "discord", "whatsapp"). + Used to inject platform-specific formatting hints into the system prompt. + skip_context_files (bool): If True, skip auto-injection of SOUL.md, AGENTS.md, and .cursorrules + into the system prompt. Use this for batch processing and data generation to avoid + polluting trajectories with user-specific persona or project instructions. + load_soul_identity (bool): If True, still use ~/.hermes/SOUL.md as the primary + identity even when skip_context_files=True. Project context files from the cwd + remain skipped. + """ + _install_safe_stdio() + + agent.model = model + agent.max_iterations = max_iterations + # Shared iteration budget — parent creates, children inherit. + # Consumed by every LLM turn across parent + all subagents. + agent.iteration_budget = iteration_budget or IterationBudget(max_iterations) + agent.tool_delay = tool_delay + agent.save_trajectories = save_trajectories + agent.verbose_logging = verbose_logging + agent.quiet_mode = quiet_mode + agent.ephemeral_system_prompt = ephemeral_system_prompt + agent.platform = platform # "cli", "telegram", "discord", "whatsapp", etc. + agent._user_id = user_id # Platform user identifier (gateway sessions) + agent._user_name = user_name + agent._chat_id = chat_id + agent._chat_name = chat_name + agent._chat_type = chat_type + agent._thread_id = thread_id + agent._gateway_session_key = gateway_session_key # Stable per-chat key (e.g. agent:main:telegram:dm:123) + # Pluggable print function — CLI replaces this with _cprint so that + # raw ANSI status lines are routed through prompt_toolkit's renderer + # instead of going directly to stdout where patch_stdout's StdoutProxy + # would mangle the escape sequences. None = use builtins.print. + agent._print_fn = None + agent.background_review_callback = None # Optional sync callback for gateway delivery + agent.skip_context_files = skip_context_files + agent.load_soul_identity = load_soul_identity + agent.pass_session_id = pass_session_id + agent._credential_pool = credential_pool + agent.log_prefix_chars = log_prefix_chars + agent.log_prefix = f"{log_prefix} " if log_prefix else "" + # Store effective base URL for feature detection (prompt caching, reasoning, etc.) + agent.base_url = base_url or "" + provider_name = provider.strip().lower() if isinstance(provider, str) and provider.strip() else None + agent.provider = provider_name or "" + agent.acp_command = acp_command or command + agent.acp_args = list(acp_args or args or []) + if api_mode in {"chat_completions", "codex_responses", "anthropic_messages", "bedrock_converse", "codex_app_server"}: + agent.api_mode = api_mode + elif agent.provider == "openai-codex": + agent.api_mode = "codex_responses" + elif agent.provider == "xai": + agent.api_mode = "codex_responses" + elif (provider_name is None) and ( + agent._base_url_hostname == "chatgpt.com" + and "/backend-api/codex" in agent._base_url_lower + ): + agent.api_mode = "codex_responses" + agent.provider = "openai-codex" + elif (provider_name is None) and agent._base_url_hostname == "api.x.ai": + agent.api_mode = "codex_responses" + agent.provider = "xai" + elif agent.provider == "anthropic" or (provider_name is None and agent._base_url_hostname == "api.anthropic.com"): + agent.api_mode = "anthropic_messages" + agent.provider = "anthropic" + elif agent._base_url_lower.rstrip("/").endswith("/anthropic"): + # Third-party Anthropic-compatible endpoints (e.g. MiniMax, DashScope) + # use a URL convention ending in /anthropic. Auto-detect these so the + # Anthropic Messages API adapter is used instead of chat completions. + agent.api_mode = "anthropic_messages" + elif agent.provider == "bedrock" or ( + agent._base_url_hostname.startswith("bedrock-runtime.") + and base_url_host_matches(agent._base_url_lower, "amazonaws.com") + ): + # AWS Bedrock — auto-detect from provider name or base URL + # (bedrock-runtime..amazonaws.com). + agent.api_mode = "bedrock_converse" + else: + agent.api_mode = "chat_completions" + + # Eagerly warm the transport cache so import errors surface at init, + # not mid-conversation. Also validates the api_mode is registered. + try: + agent._get_transport() + except Exception: + pass # Non-fatal — transport may not exist for all modes yet + + try: + from hermes_cli.model_normalize import ( + _AGGREGATOR_PROVIDERS, + normalize_model_for_provider, + ) + + if agent.provider not in _AGGREGATOR_PROVIDERS: + agent.model = normalize_model_for_provider(agent.model, agent.provider) + except Exception: + pass + + # GPT-5.x models usually require the Responses API path, but some + # providers have exceptions (for example Copilot's gpt-5-mini still + # uses chat completions). Also auto-upgrade for direct OpenAI URLs + # (api.openai.com) since all newer tool-calling models prefer + # Responses there. ACP runtimes are excluded: CopilotACPClient + # handles its own routing and does not implement the Responses API + # surface. + # When api_mode was explicitly provided, respect it — the user + # knows what their endpoint supports (#10473). + # Exception: Azure OpenAI serves gpt-5.x on /chat/completions and + # does NOT support the Responses API — skip the upgrade for Azure + # (openai.azure.com), even though it looks OpenAI-compatible. + if ( + api_mode is None + and agent.api_mode == "chat_completions" + and agent.provider != "copilot-acp" + and not str(agent.base_url or "").lower().startswith("acp://copilot") + and not str(agent.base_url or "").lower().startswith("acp+tcp://") + and not agent._is_azure_openai_url() + and ( + agent._is_direct_openai_url() + or agent._provider_model_requires_responses_api( + agent.model, + provider=agent.provider, + ) + ) + ): + agent.api_mode = "codex_responses" + # Invalidate the eager-warmed transport cache — api_mode changed + # from chat_completions to codex_responses after the warm at __init__. + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + + # Pre-warm OpenRouter model metadata cache in a background thread. + # fetch_model_metadata() is cached for 1 hour; this avoids a blocking + # HTTP request on the first API response when pricing is estimated. + # Use a process-level Event so this thread is only spawned once — a new + # AIAgent is created for every gateway request, so without the guard + # each message leaks one OS thread and the process eventually exhausts + # the system thread limit (RuntimeError: can't start new thread). + if (agent.provider == "openrouter" or agent._is_openrouter_url()) and \ + not _ra()._openrouter_prewarm_done.is_set(): + _ra()._openrouter_prewarm_done.set() + threading.Thread( + target=fetch_model_metadata, + daemon=True, + name="openrouter-prewarm", + ).start() + + agent.tool_progress_callback = tool_progress_callback + agent.tool_start_callback = tool_start_callback + agent.tool_complete_callback = tool_complete_callback + agent.suppress_status_output = False + agent.thinking_callback = thinking_callback + agent.reasoning_callback = reasoning_callback + agent.clarify_callback = clarify_callback + agent.step_callback = step_callback + agent.stream_delta_callback = stream_delta_callback + agent.interim_assistant_callback = interim_assistant_callback + agent.status_callback = status_callback + agent.tool_gen_callback = tool_gen_callback + + + # Tool execution state — allows _vprint during tool execution + # even when stream consumers are registered (no tokens streaming then) + agent._executing_tools = False + agent._tool_guardrails = ToolCallGuardrailController() + agent._tool_guardrail_halt_decision: ToolGuardrailDecision | None = None + + # Interrupt mechanism for breaking out of tool loops + agent._interrupt_requested = False + agent._interrupt_message = None # Optional message that triggered interrupt + agent._execution_thread_id: int | None = None # Set at run_conversation() start + agent._interrupt_thread_signal_pending = False + agent._client_lock = threading.RLock() + + # /steer mechanism — inject a user note into the next tool result + # without interrupting the agent. Unlike interrupt(), steer() does + # NOT set _interrupt_requested; it waits for the current tool batch + # to finish naturally, then the drain hook appends the text to the + # last tool result's content so the model sees it on its next + # iteration. Message-role alternation is preserved (we modify an + # existing tool message rather than inserting a new user turn). + agent._pending_steer: Optional[str] = None + agent._pending_steer_lock = threading.Lock() + + # Concurrent-tool worker thread tracking. `_execute_tool_calls_concurrent` + # runs each tool on its own ThreadPoolExecutor worker — those worker + # threads have tids distinct from `_execution_thread_id`, so + # `_set_interrupt(True, _execution_thread_id)` alone does NOT cause + # `is_interrupted()` inside the worker to return True. Track the + # workers here so `interrupt()` / `clear_interrupt()` can fan out to + # their tids explicitly. + agent._tool_worker_threads: set[int] = set() + agent._tool_worker_threads_lock = threading.Lock() + + # Subagent delegation state + agent._delegate_depth = 0 # 0 = top-level agent, incremented for children + agent._active_children = [] # Running child AIAgents (for interrupt propagation) + agent._active_children_lock = threading.Lock() + + # Store OpenRouter provider preferences + agent.providers_allowed = providers_allowed + agent.providers_ignored = providers_ignored + agent.providers_order = providers_order + agent.provider_sort = provider_sort + agent.provider_require_parameters = provider_require_parameters + agent.provider_data_collection = provider_data_collection + agent.openrouter_min_coding_score = openrouter_min_coding_score + + # Store toolset filtering options + agent.enabled_toolsets = enabled_toolsets + agent.disabled_toolsets = disabled_toolsets + + # Model response configuration + agent.max_tokens = max_tokens # None = use model default + agent.reasoning_config = reasoning_config # None = use default (medium for OpenRouter) + agent.service_tier = service_tier + agent.request_overrides = dict(request_overrides or {}) + agent.prefill_messages = prefill_messages or [] # Prefilled conversation turns + agent._force_ascii_payload = False + + # Anthropic prompt caching: auto-enabled for Claude models on native + # Anthropic, OpenRouter, and third-party gateways that speak the + # Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces + # input costs by ~75% on multi-turn conversations. Uses system_and_3 + # strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy`` + # for the layout-vs-transport decision. + agent._use_prompt_caching, agent._use_native_cache_layout = ( + agent._anthropic_prompt_cache_policy() + ) + # Anthropic supports "5m" (default) and "1h" cache TTL tiers. Read from + # config.yaml under prompt_caching.cache_ttl; unknown values keep "5m". + # 1h tier costs 2x on write vs 1.25x for 5m, but amortizes across long + # sessions with >5-minute pauses between turns (#14971). + agent._cache_ttl = "5m" + try: + from hermes_cli.config import load_config as _load_pc_cfg + + _pc_cfg = _load_pc_cfg().get("prompt_caching", {}) or {} + _ttl = _pc_cfg.get("cache_ttl", "5m") + if _ttl in {"5m", "1h"}: + agent._cache_ttl = _ttl + except Exception: + pass + + # Iteration budget: the LLM is only notified when it actually exhausts + # the iteration budget (api_call_count >= max_iterations). At that + # point we inject ONE message, allow one final API call, and if the + # model doesn't produce a text response, force a user-message asking + # it to summarise. No intermediate pressure warnings — they caused + # models to "give up" prematurely on complex tasks (#7915). + agent._budget_exhausted_injected = False + agent._budget_grace_call = False + + # Activity tracking — updated on each API call, tool execution, and + # stream chunk. Used by the gateway timeout handler to report what the + # agent was doing when it was killed, and by the "still working" + # notifications to show progress. + agent._last_activity_ts: float = time.time() + agent._last_activity_desc: str = "initializing" + agent._current_tool: str | None = None + agent._api_call_count: int = 0 + + # Rate limit tracking — updated from x-ratelimit-* response headers + # after each API call. Accessed by /usage slash command. + agent._rate_limit_state: Optional["RateLimitState"] = None + + # OpenRouter response cache hit counter — incremented when + # X-OpenRouter-Cache-Status: HIT is seen in streaming response headers. + agent._or_cache_hits: int = 0 + + # Centralized logging — agent.log (INFO+) and errors.log (WARNING+) + # both live under ~/.hermes/logs/. Idempotent, so gateway mode + # (which creates a new AIAgent per message) won't duplicate handlers. + from hermes_logging import setup_logging, setup_verbose_logging + setup_logging(hermes_home=_ra()._hermes_home) + + if agent.verbose_logging: + setup_verbose_logging() + _ra().logger.info("Verbose logging enabled (third-party library logs suppressed)") + elif agent.quiet_mode: + # In quiet mode (CLI default), keep console output clean — + # but DO NOT raise per-logger levels. Doing so prevents the + # root logger's file handlers (agent.log, errors.log) from + # ever seeing the records, because Python checks + # logger.isEnabledFor() before handler propagation. We rely + # on the fact that hermes_logging.setup_logging() does not + # install a console StreamHandler in quiet mode — so INFO + # records flow to the file handlers but never reach a + # console. Any future noise reduction belongs at the + # handler level inside hermes_logging.py, not here. + pass + + # Internal stream callback (set during streaming TTS). + # Initialized here so _vprint can reference it before run_conversation. + agent._stream_callback = None + # Deferred paragraph break flag — set after tool iterations so a + # single "\n\n" is prepended to the next real text delta. + agent._stream_needs_break = False + # Stateful scrubber for spans split across stream + # deltas (#5719). sanitize_context() alone can't survive chunk + # boundaries because the block regex needs both tags in one string. + agent._stream_context_scrubber = StreamingContextScrubber() + # Stateful scrubber for reasoning/thinking tags in streamed deltas + # (#17924). Replaces the per-delta _strip_think_blocks regex that + # destroyed downstream state (e.g. MiniMax-M2.7 streaming + # '' as delta1 and 'Let me check' as delta2 — the regex + # erased delta1, so downstream state machines never learned a + # block was open and leaked delta2 as content). + agent._stream_think_scrubber = StreamingThinkScrubber() + # Visible assistant text already delivered through live token callbacks + # during the current model response. Used to avoid re-sending the same + # commentary when the provider later returns it as a completed interim + # assistant message. + agent._current_streamed_assistant_text = "" + + # Optional current-turn user-message override used when the API-facing + # user message intentionally differs from the persisted transcript + # (e.g. CLI voice mode adds a temporary prefix for the live call only). + agent._persist_user_message_idx = None + agent._persist_user_message_override = None + + # Cache anthropic image-to-text fallbacks per image payload/URL so a + # single tool loop does not repeatedly re-run auxiliary vision on the + # same image history. + agent._anthropic_image_fallback_cache: Dict[str, str] = {} + + # Initialize LLM client via centralized provider router. + # The router handles auth resolution, base URL, headers, and + # Codex/Anthropic wrapping for all known providers. + # raw_codex=True because the main agent needs direct responses.stream() + # access for Codex Responses API streaming. + agent._anthropic_client = None + agent._is_anthropic_oauth = False + + # Resolve per-provider / per-model request timeout once up front so + # every client construction path below (Anthropic native, OpenAI-wire, + # router-based implicit auth) can apply it consistently. Bedrock + # Claude uses its own timeout path and is not covered here. + _provider_timeout = get_provider_request_timeout(agent.provider, agent.model) + + if agent.api_mode == "anthropic_messages": + from agent.anthropic_adapter import build_anthropic_client, resolve_anthropic_token + # Bedrock + Claude → use AnthropicBedrock SDK for full feature parity + # (prompt caching, thinking budgets, adaptive thinking). + _is_bedrock_anthropic = agent.provider == "bedrock" + if _is_bedrock_anthropic: + from agent.anthropic_adapter import build_anthropic_bedrock_client + _region_match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") + _br_region = _region_match.group(1) if _region_match else "us-east-1" + agent._bedrock_region = _br_region + agent._anthropic_client = build_anthropic_bedrock_client(_br_region) + agent._anthropic_api_key = "aws-sdk" + agent._anthropic_base_url = base_url + agent._is_anthropic_oauth = False + agent.api_key = "aws-sdk" + agent.client = None + agent._client_kwargs = {} + if not agent.quiet_mode: + print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {_br_region})") + else: + # Only fall back to ANTHROPIC_TOKEN when the provider is actually Anthropic. + # Other anthropic_messages providers (MiniMax, Alibaba, etc.) must use their own API key. + # Falling back would send Anthropic credentials to third-party endpoints (Fixes #1739, #minimax-401). + _is_native_anthropic = agent.provider == "anthropic" + effective_key = (api_key or resolve_anthropic_token() or "") if _is_native_anthropic else (api_key or "") + agent.api_key = effective_key + agent._anthropic_api_key = effective_key + agent._anthropic_base_url = base_url + # Only mark the session as OAuth-authenticated when the token + # genuinely belongs to native Anthropic. Third-party providers + # (MiniMax, Kimi, GLM, LiteLLM proxies) that accept the + # Anthropic protocol must never trip OAuth code paths — doing + # so injects Claude-Code identity headers and system prompts + # that cause 401/403 on their endpoints. Guards #1739 and + # the third-party identity-injection bug. + from agent.anthropic_adapter import _is_oauth_token as _is_oat + agent._is_anthropic_oauth = _is_oat(effective_key) if _is_native_anthropic else False + agent._anthropic_client = build_anthropic_client(effective_key, base_url, timeout=_provider_timeout) + # No OpenAI client needed for Anthropic mode + agent.client = None + agent._client_kwargs = {} + if not agent.quiet_mode: + print(f"🤖 AI Agent initialized with model: {agent.model} (Anthropic native)") + if effective_key and len(effective_key) > 12: + print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}") + elif agent.api_mode == "bedrock_converse": + # AWS Bedrock — uses boto3 directly, no OpenAI client needed. + # Region is extracted from the base_url or defaults to us-east-1. + _region_match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") + agent._bedrock_region = _region_match.group(1) if _region_match else "us-east-1" + # Guardrail config — read from config.yaml at init time. + agent._bedrock_guardrail_config = None + try: + from hermes_cli.config import load_config as _load_br_cfg + _gr = _load_br_cfg().get("bedrock", {}).get("guardrail", {}) + if _gr.get("guardrail_identifier") and _gr.get("guardrail_version"): + agent._bedrock_guardrail_config = { + "guardrailIdentifier": _gr["guardrail_identifier"], + "guardrailVersion": _gr["guardrail_version"], + } + if _gr.get("stream_processing_mode"): + agent._bedrock_guardrail_config["streamProcessingMode"] = _gr["stream_processing_mode"] + if _gr.get("trace"): + agent._bedrock_guardrail_config["trace"] = _gr["trace"] + except Exception: + pass + agent.client = None + agent._client_kwargs = {} + if not agent.quiet_mode: + _gr_label = " + Guardrails" if agent._bedrock_guardrail_config else "" + print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock, {agent._bedrock_region}{_gr_label})") + else: + if api_key and base_url: + # Explicit credentials from CLI/gateway — construct directly. + # The runtime provider resolver already handled auth for us. + # Extract query params (e.g. Azure api-version) from base_url + # and pass via default_query to prevent loss during SDK URL + # joining (httpx drops query string when joining paths). + _parsed_url = urlparse(base_url) + if _parsed_url.query: + _clean_url = urlunparse(_parsed_url._replace(query="")) + _query_params = { + k: v[0] for k, v in parse_qs(_parsed_url.query).items() + } + client_kwargs = { + "api_key": api_key, + "base_url": _clean_url, + "default_query": _query_params, + } + else: + client_kwargs = {"api_key": api_key, "base_url": base_url} + if _provider_timeout is not None: + client_kwargs["timeout"] = _provider_timeout + if agent.provider == "copilot-acp": + client_kwargs["command"] = agent.acp_command + client_kwargs["args"] = agent.acp_args + effective_base = base_url + if base_url_host_matches(effective_base, "openrouter.ai"): + from agent.auxiliary_client import build_or_headers + client_kwargs["default_headers"] = build_or_headers() + elif base_url_host_matches(effective_base, "api.routermint.com"): + client_kwargs["default_headers"] = _ra()._routermint_headers() + elif base_url_host_matches(effective_base, "api.githubcopilot.com"): + from hermes_cli.models import copilot_default_headers + + client_kwargs["default_headers"] = copilot_default_headers() + elif base_url_host_matches(effective_base, "api.kimi.com"): + client_kwargs["default_headers"] = { + "User-Agent": "claude-code/0.1.0", + } + elif base_url_host_matches(effective_base, "portal.qwen.ai"): + client_kwargs["default_headers"] = _ra()._qwen_portal_headers() + elif base_url_host_matches(effective_base, "chatgpt.com"): + from agent.auxiliary_client import _codex_cloudflare_headers + client_kwargs["default_headers"] = _codex_cloudflare_headers(api_key) + elif "default_headers" not in client_kwargs: + # Fall back to profile.default_headers for providers that + # declare custom headers (e.g. Vercel AI Gateway attribution, + # Kimi User-Agent on non-kimi.com endpoints). + try: + from providers import get_provider_profile as _gpf + _ph = _gpf(agent.provider) + if _ph and _ph.default_headers: + client_kwargs["default_headers"] = dict(_ph.default_headers) + except Exception: + pass + else: + # No explicit creds — use the centralized provider router + from agent.auxiliary_client import resolve_provider_client + _routed_client, _ = resolve_provider_client( + agent.provider or "auto", model=agent.model, raw_codex=True) + if _routed_client is not None: + client_kwargs = { + "api_key": _routed_client.api_key, + "base_url": str(_routed_client.base_url), + } + if _provider_timeout is not None: + client_kwargs["timeout"] = _provider_timeout + # Preserve any default_headers the router set + if hasattr(_routed_client, '_default_headers') and _routed_client._default_headers: + client_kwargs["default_headers"] = dict(_routed_client._default_headers) + else: + # When the user explicitly chose a non-OpenRouter provider + # but no credentials were found, fail fast with a clear + # message instead of silently routing through OpenRouter. + _explicit = (agent.provider or "").strip().lower() + if _explicit and _explicit not in {"auto", "openrouter", "custom"}: + # Look up the actual env var name from the provider + # config — some providers use non-standard names + # (e.g. alibaba → DASHSCOPE_API_KEY, not ALIBABA_API_KEY). + _env_hint = f"{_explicit.upper()}_API_KEY" + try: + from hermes_cli.auth import PROVIDER_REGISTRY + _pcfg = PROVIDER_REGISTRY.get(_explicit) + if _pcfg and _pcfg.api_key_env_vars: + _env_hint = _pcfg.api_key_env_vars[0] + except Exception: + pass + # --- Init-time fallback (#17929) --- + _fb_entries = [] + if isinstance(fallback_model, list): + _fb_entries = [ + f for f in fallback_model + if isinstance(f, dict) and f.get("provider") and f.get("model") + ] + elif isinstance(fallback_model, dict) and fallback_model.get("provider") and fallback_model.get("model"): + _fb_entries = [fallback_model] + _fb_resolved = False + for _fb in _fb_entries: + _fb_explicit_key = (_fb.get("api_key") or "").strip() or None + if not _fb_explicit_key: + _fb_key_env = (_fb.get("key_env") or _fb.get("api_key_env") or "").strip() + if _fb_key_env: + _fb_explicit_key = os.getenv(_fb_key_env, "").strip() or None + _fb_client, _fb_model = resolve_provider_client( + _fb["provider"], model=_fb["model"], raw_codex=True, + explicit_base_url=_fb.get("base_url"), + explicit_api_key=_fb_explicit_key, + ) + if _fb_client is not None: + agent.provider = _fb["provider"] + agent.model = _fb_model or _fb["model"] + agent._fallback_activated = True + client_kwargs = { + "api_key": _fb_client.api_key, + "base_url": str(_fb_client.base_url), + } + if _provider_timeout is not None: + client_kwargs["timeout"] = _provider_timeout + if hasattr(_fb_client, "_default_headers") and _fb_client._default_headers: + client_kwargs["default_headers"] = dict(_fb_client._default_headers) + _fb_resolved = True + break + if not _fb_resolved: + raise RuntimeError( + f"Provider '{_explicit}' is set in config.yaml but no API key " + f"was found. Set the {_env_hint} environment " + f"variable, or switch to a different provider with `hermes model`." + ) + if not getattr(agent, "_fallback_activated", False): + # No provider configured — reject with a clear message. + raise RuntimeError( + "No LLM provider configured. Run `hermes model` to " + "select a provider, or run `hermes setup` for first-time " + "configuration." + ) + + agent._client_kwargs = client_kwargs # stored for rebuilding after interrupt + + # Enable fine-grained tool streaming for Claude on OpenRouter. + # Without this, Anthropic buffers the entire tool call and goes + # silent for minutes while thinking — OpenRouter's upstream proxy + # times out during the silence. The beta header makes Anthropic + # stream tool call arguments token-by-token, keeping the + # connection alive. + _effective_base = str(client_kwargs.get("base_url", "")).lower() + if base_url_host_matches(_effective_base, "openrouter.ai") and "claude" in (agent.model or "").lower(): + headers = client_kwargs.get("default_headers") or {} + existing_beta = headers.get("x-anthropic-beta", "") + _FINE_GRAINED = "fine-grained-tool-streaming-2025-05-14" + if _FINE_GRAINED not in existing_beta: + if existing_beta: + headers["x-anthropic-beta"] = f"{existing_beta},{_FINE_GRAINED}" + else: + headers["x-anthropic-beta"] = _FINE_GRAINED + client_kwargs["default_headers"] = headers + + agent.api_key = client_kwargs.get("api_key", "") + agent.base_url = client_kwargs.get("base_url", agent.base_url) + try: + agent.client = agent._create_openai_client(client_kwargs, reason="agent_init", shared=True) + if not agent.quiet_mode: + print(f"🤖 AI Agent initialized with model: {agent.model}") + if base_url: + print(f"🔗 Using custom base URL: {base_url}") + # Always show API key info (masked) for debugging auth issues + key_used = client_kwargs.get("api_key", "none") + if key_used and key_used != "dummy-key" and len(key_used) > 12: + print(f"🔑 Using API key: {key_used[:8]}...{key_used[-4:]}") + else: + print(f"⚠️ Warning: API key appears invalid or missing (got: '{key_used[:20] if key_used else 'none'}...')") + except Exception as e: + raise RuntimeError(f"Failed to initialize OpenAI client: {e}") + + # Provider fallback chain — ordered list of backup providers tried + # when the primary is exhausted (rate-limit, overload, connection + # failure). Supports both legacy single-dict ``fallback_model`` and + # new list ``fallback_providers`` format. + if isinstance(fallback_model, list): + agent._fallback_chain = [ + f for f in fallback_model + if isinstance(f, dict) and f.get("provider") and f.get("model") + ] + elif isinstance(fallback_model, dict) and fallback_model.get("provider") and fallback_model.get("model"): + agent._fallback_chain = [fallback_model] + else: + agent._fallback_chain = [] + agent._fallback_index = 0 + agent._fallback_activated = getattr(agent, "_fallback_activated", False) + # Legacy attribute kept for backward compat (tests, external callers) + agent._fallback_model = agent._fallback_chain[0] if agent._fallback_chain else None + if agent._fallback_chain and not agent.quiet_mode: + if len(agent._fallback_chain) == 1: + fb = agent._fallback_chain[0] + print(f"🔄 Fallback model: {fb['model']} ({fb['provider']})") + else: + print(f"🔄 Fallback chain ({len(agent._fallback_chain)} providers): " + + " → ".join(f"{f['model']} ({f['provider']})" for f in agent._fallback_chain)) + + # Get available tools with filtering + agent.tools = _ra().get_tool_definitions( + enabled_toolsets=enabled_toolsets, + disabled_toolsets=disabled_toolsets, + quiet_mode=agent.quiet_mode, + ) + + # Show tool configuration and store valid tool names for validation + agent.valid_tool_names = set() + if agent.tools: + agent.valid_tool_names = {tool["function"]["name"] for tool in agent.tools} + tool_names = sorted(agent.valid_tool_names) + if not agent.quiet_mode: + print(f"🛠️ Loaded {len(agent.tools)} tools: {', '.join(tool_names)}") + + # Show filtering info if applied + if enabled_toolsets: + print(f" ✅ Enabled toolsets: {', '.join(enabled_toolsets)}") + if disabled_toolsets: + print(f" ❌ Disabled toolsets: {', '.join(disabled_toolsets)}") + elif not agent.quiet_mode: + print("🛠️ No tools loaded (all tools filtered out or unavailable)") + + # Check tool requirements + if agent.tools and not agent.quiet_mode: + requirements = _ra().check_toolset_requirements() + missing_reqs = [name for name, available in requirements.items() if not available] + if missing_reqs: + print(f"⚠️ Some tools may not work due to missing requirements: {missing_reqs}") + + # Show trajectory saving status + if agent.save_trajectories and not agent.quiet_mode: + print("📝 Trajectory saving enabled") + + # Show ephemeral system prompt status + if agent.ephemeral_system_prompt and not agent.quiet_mode: + prompt_preview = agent.ephemeral_system_prompt[:60] + "..." if len(agent.ephemeral_system_prompt) > 60 else agent.ephemeral_system_prompt + print(f"🔒 Ephemeral system prompt: '{prompt_preview}' (not saved to trajectories)") + + # Show prompt caching status + if agent._use_prompt_caching and not agent.quiet_mode: + if agent._use_native_cache_layout and agent.provider == "anthropic": + source = "native Anthropic" + elif agent._use_native_cache_layout: + source = "Anthropic-compatible endpoint" + else: + source = "Claude via OpenRouter" + print(f"💾 Prompt caching: ENABLED ({source}, {agent._cache_ttl} TTL)") + + # Session logging setup - auto-save conversation trajectories for debugging + agent.session_start = datetime.now() + if session_id: + # Use provided session ID (e.g., from CLI) + agent.session_id = session_id + else: + # Generate a new session ID + timestamp_str = agent.session_start.strftime("%Y%m%d_%H%M%S") + short_uuid = uuid.uuid4().hex[:6] + agent.session_id = f"{timestamp_str}_{short_uuid}" + + # Expose session ID to tools (terminal, execute_code) so agents can + # reference their own session for --resume commands, cross-session + # coordination, and logging. Uses the ContextVar system from + # session_context.py for concurrency safety (gateway runs multiple + # sessions in one process). Also writes os.environ as fallback for + # CLI mode where ContextVars aren't used. + os.environ["HERMES_SESSION_ID"] = agent.session_id + try: + from gateway.session_context import _SESSION_ID + _SESSION_ID.set(agent.session_id) + except Exception: + pass # CLI/test mode — ContextVar not needed + + # Session logs go into ~/.hermes/sessions/ alongside gateway sessions + hermes_home = get_hermes_home() + agent.logs_dir = hermes_home / "sessions" + agent.logs_dir.mkdir(parents=True, exist_ok=True) + agent.session_log_file = agent.logs_dir / f"session_{agent.session_id}.json" + + # Track conversation messages for session logging + agent._session_messages: List[Dict[str, Any]] = [] + agent._memory_write_origin = "assistant_tool" + agent._memory_write_context = "foreground" + + # Cached system prompt -- built once per session, only rebuilt on compression + agent._cached_system_prompt: Optional[str] = None + + # Filesystem checkpoint manager (transparent — not a tool) + from tools.checkpoint_manager import CheckpointManager + agent._checkpoint_mgr = CheckpointManager( + enabled=checkpoints_enabled, + max_snapshots=checkpoint_max_snapshots, + max_total_size_mb=checkpoint_max_total_size_mb, + max_file_size_mb=checkpoint_max_file_size_mb, + ) + + # SQLite session store (optional -- provided by CLI or gateway) + agent._session_db = session_db + agent._parent_session_id = parent_session_id + agent._last_flushed_db_idx = 0 # tracks DB-write cursor to prevent duplicate writes + agent._session_db_created = False # DB row deferred to run_conversation() + agent._session_init_model_config = { + "max_iterations": agent.max_iterations, + "reasoning_config": reasoning_config, + "max_tokens": max_tokens, + } + + # In-memory todo list for task planning (one per agent/session) + from tools.todo_tool import TodoStore + agent._todo_store = TodoStore() + + # Load config once for memory, skills, and compression sections + try: + from hermes_cli.config import load_config as _load_agent_config + _agent_cfg = _load_agent_config() + except Exception: + _agent_cfg = {} + try: + agent._tool_guardrails = ToolCallGuardrailController( + ToolCallGuardrailConfig.from_mapping( + _agent_cfg.get("tool_loop_guardrails", {}) + ) + ) + except Exception as _tlg_err: + _ra().logger.warning("Tool loop guardrail config ignored: %s", _tlg_err) + # Cache only the derived auxiliary compression context override that is + # needed later by the startup feasibility check. Avoid exposing a + # broad pseudo-public config object on the agent instance. + agent._aux_compression_context_length_config = None + + # Persistent memory (MEMORY.md + USER.md) -- loaded from disk + agent._memory_store = None + agent._memory_enabled = False + agent._user_profile_enabled = False + agent._memory_nudge_interval = 10 + agent._turns_since_memory = 0 + agent._iters_since_skill = 0 + if not skip_memory: + try: + mem_config = _agent_cfg.get("memory", {}) + agent._memory_enabled = mem_config.get("memory_enabled", False) + agent._user_profile_enabled = mem_config.get("user_profile_enabled", False) + agent._memory_nudge_interval = int(mem_config.get("nudge_interval", 10)) + if agent._memory_enabled or agent._user_profile_enabled: + from tools.memory_tool import MemoryStore + agent._memory_store = MemoryStore( + memory_char_limit=mem_config.get("memory_char_limit", 2200), + user_char_limit=mem_config.get("user_char_limit", 1375), + ) + agent._memory_store.load_from_disk() + except Exception: + pass # Memory is optional -- don't break agent init + + + + # Memory provider plugin (external — one at a time, alongside built-in) + # Reads memory.provider from config to select which plugin to activate. + agent._memory_manager = None + if not skip_memory: + try: + _mem_provider_name = mem_config.get("provider", "") if mem_config else "" + + if _mem_provider_name: + from agent.memory_manager import MemoryManager as _MemoryManager + from plugins.memory import load_memory_provider as _load_mem + agent._memory_manager = _MemoryManager() + _mp = _load_mem(_mem_provider_name) + if _mp and _mp.is_available(): + agent._memory_manager.add_provider(_mp) + if agent._memory_manager.providers: + _init_kwargs = { + "session_id": agent.session_id, + "platform": platform or "cli", + "hermes_home": str(get_hermes_home()), + "agent_context": "primary", + } + # Thread session title for memory provider scoping + # (e.g. honcho uses this to derive chat-scoped session keys) + if agent._session_db: + try: + _st = agent._session_db.get_session_title(agent.session_id) + if _st: + _init_kwargs["session_title"] = _st + except Exception: + pass + # Thread gateway user identity for per-user memory scoping + if agent._user_id: + _init_kwargs["user_id"] = agent._user_id + if agent._user_name: + _init_kwargs["user_name"] = agent._user_name + if agent._chat_id: + _init_kwargs["chat_id"] = agent._chat_id + if agent._chat_name: + _init_kwargs["chat_name"] = agent._chat_name + if agent._chat_type: + _init_kwargs["chat_type"] = agent._chat_type + if agent._thread_id: + _init_kwargs["thread_id"] = agent._thread_id + # Thread gateway session key for stable per-chat Honcho session isolation + if agent._gateway_session_key: + _init_kwargs["gateway_session_key"] = agent._gateway_session_key + # Profile identity for per-profile provider scoping + try: + from hermes_cli.profiles import get_active_profile_name + _profile = get_active_profile_name() + _init_kwargs["agent_identity"] = _profile + _init_kwargs["agent_workspace"] = "hermes" + except Exception: + pass + agent._memory_manager.initialize_all(**_init_kwargs) + _ra().logger.info("Memory provider '%s' activated", _mem_provider_name) + else: + _ra().logger.debug("Memory provider '%s' not found or not available", _mem_provider_name) + agent._memory_manager = None + except Exception as _mpe: + _ra().logger.warning("Memory provider plugin init failed: %s", _mpe) + agent._memory_manager = None + + # Inject memory provider tool schemas into the tool surface. + # Skip tools whose names already exist (plugins may register the + # same tools via ctx.register_tool(), which lands in agent.tools + # through _ra().get_tool_definitions()). Duplicate function names cause + # 400 errors on providers that enforce unique names (e.g. Xiaomi + # MiMo via Nous Portal). + if agent._memory_manager and agent.tools is not None: + _existing_tool_names = { + t.get("function", {}).get("name") + for t in agent.tools + if isinstance(t, dict) + } + for _schema in agent._memory_manager.get_all_tool_schemas(): + _tname = _schema.get("name", "") + if _tname and _tname in _existing_tool_names: + continue # already registered via plugin path + _wrapped = {"type": "function", "function": _schema} + agent.tools.append(_wrapped) + if _tname: + agent.valid_tool_names.add(_tname) + _existing_tool_names.add(_tname) + + # Skills config: nudge interval for skill creation reminders + agent._skill_nudge_interval = 10 + try: + skills_config = _agent_cfg.get("skills", {}) + agent._skill_nudge_interval = int(skills_config.get("creation_nudge_interval", 10)) + except Exception: + pass + + # Tool-use enforcement config: "auto" (default — matches hardcoded + # model list), true (always), false (never), or list of substrings. + _agent_section = _agent_cfg.get("agent", {}) + if not isinstance(_agent_section, dict): + _agent_section = {} + agent._tool_use_enforcement = _agent_section.get("tool_use_enforcement", "auto") + + # App-level API retry count (wraps each model API call). Default 3, + # overridable via agent.api_max_retries in config.yaml. See #11616. + try: + _raw_api_retries = _agent_section.get("api_max_retries", 3) + _api_retries = int(_raw_api_retries) + _api_retries = max(_api_retries, 1) # 1 = no retry (single attempt) + except (TypeError, ValueError): + _api_retries = 3 + agent._api_max_retries = _api_retries + + # Initialize context compressor for automatic context management + # Compresses conversation when approaching model's context limit + # Configuration via config.yaml (compression section) + _compression_cfg = _agent_cfg.get("compression", {}) + if not isinstance(_compression_cfg, dict): + _compression_cfg = {} + compression_threshold = float(_compression_cfg.get("threshold", 0.50)) + try: + from agent.auxiliary_client import _compression_threshold_for_model as _cthresh_fn + _model_cthresh = _cthresh_fn(agent.model) + if _model_cthresh is not None: + compression_threshold = _model_cthresh + except Exception: + pass + compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"} + compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20)) + compression_protect_last = int(_compression_cfg.get("protect_last_n", 20)) + # protect_first_n is the number of non-system messages to protect at + # the head, in addition to the system prompt (which is always + # implicitly protected by the compressor). Floor at 0 — a value of + # 0 means "preserve only the system prompt + summary + tail", which + # is a legitimate (and common) configuration for long-running + # rolling-compaction sessions. + compression_protect_first = max( + 0, int(_compression_cfg.get("protect_first_n", 3)) + ) + + # Read optional explicit context_length override for the auxiliary + # compression model. Custom endpoints often cannot report this via + # /models, so the startup feasibility check needs the config hint. + try: + _aux_cfg = cfg_get(_agent_cfg, "auxiliary", "compression", default={}) + except Exception: + _aux_cfg = {} + if isinstance(_aux_cfg, dict): + _aux_context_config = _aux_cfg.get("context_length") + else: + _aux_context_config = None + if _aux_context_config is not None: + try: + _aux_context_config = int(_aux_context_config) + except (TypeError, ValueError): + _aux_context_config = None + agent._aux_compression_context_length_config = _aux_context_config + + # Read explicit model output-token override from config when the + # caller did not pass one directly. + _model_cfg = _agent_cfg.get("model", {}) + if agent.max_tokens is None and isinstance(_model_cfg, dict): + _config_max_tokens = _model_cfg.get("max_tokens") + if _config_max_tokens is not None: + try: + if isinstance(_config_max_tokens, bool): + raise ValueError + _parsed_max_tokens = int(_config_max_tokens) + if _parsed_max_tokens <= 0: + raise ValueError + agent.max_tokens = _parsed_max_tokens + except (TypeError, ValueError): + _ra().logger.warning( + "Invalid model.max_tokens in config.yaml: %r — " + "must be a positive integer (e.g. 4096). " + "Falling back to provider default.", + _config_max_tokens, + ) + print( + f"\n⚠ Invalid model.max_tokens in config.yaml: {_config_max_tokens!r}\n" + f" Must be a positive integer (e.g. 4096).\n" + f" Falling back to provider default.\n", + file=sys.stderr, + ) + agent._session_init_model_config["max_tokens"] = agent.max_tokens + + # Read explicit context_length override from model config + if isinstance(_model_cfg, dict): + _config_context_length = _model_cfg.get("context_length") + else: + _config_context_length = None + if _config_context_length is not None: + try: + _config_context_length = int(_config_context_length) + except (TypeError, ValueError): + _ra().logger.warning( + "Invalid model.context_length in config.yaml: %r — " + "must be a plain integer (e.g. 256000, not '256K'). " + "Falling back to auto-detection.", + _config_context_length, + ) + print( + f"\n⚠ Invalid model.context_length in config.yaml: {_config_context_length!r}\n" + f" Must be a plain integer (e.g. 256000, not '256K').\n" + f" Falling back to auto-detected context window.\n", + file=sys.stderr, + ) + _config_context_length = None + + # Resolve custom_providers list once for reuse below (startup + # context-length override and plugin context-engine init). + try: + from hermes_cli.config import get_compatible_custom_providers + _custom_providers = get_compatible_custom_providers(_agent_cfg) + except Exception: + _custom_providers = _agent_cfg.get("custom_providers") + if not isinstance(_custom_providers, list): + _custom_providers = [] + + # Store for reuse by _check_compression_model_feasibility (auxiliary + # compression model context-length detection needs the same list). + agent._custom_providers = _custom_providers + + # Check custom_providers per-model context_length + if _config_context_length is None and _custom_providers: + try: + from hermes_cli.config import get_custom_provider_context_length + _cp_ctx_resolved = get_custom_provider_context_length( + model=agent.model, + base_url=agent.base_url, + custom_providers=_custom_providers, + ) + if _cp_ctx_resolved: + _config_context_length = int(_cp_ctx_resolved) + except Exception: + _cp_ctx_resolved = None + + # Surface a clear warning if the user set a context_length but it + # wasn't a valid positive int — the helper silently skips those. + if _config_context_length is None: + _target = agent.base_url.rstrip("/") if agent.base_url else "" + for _cp_entry in _custom_providers: + if not isinstance(_cp_entry, dict): + continue + _cp_url = (_cp_entry.get("base_url") or "").rstrip("/") + if _target and _cp_url == _target: + _cp_models = _cp_entry.get("models", {}) + if isinstance(_cp_models, dict): + _cp_model_cfg = _cp_models.get(agent.model, {}) + if isinstance(_cp_model_cfg, dict): + _cp_ctx = _cp_model_cfg.get("context_length") + if _cp_ctx is not None: + try: + _parsed = int(_cp_ctx) + if _parsed <= 0: + raise ValueError + except (TypeError, ValueError): + _ra().logger.warning( + "Invalid context_length for model %r in " + "custom_providers: %r — must be a positive " + "integer (e.g. 256000, not '256K'). " + "Falling back to auto-detection.", + agent.model, _cp_ctx, + ) + print( + f"\n⚠ Invalid context_length for model {agent.model!r} in custom_providers: {_cp_ctx!r}\n" + f" Must be a positive integer (e.g. 256000, not '256K').\n" + f" Falling back to auto-detected context window.\n", + file=sys.stderr, + ) + break + + # Persist for reuse on switch_model / fallback activation. Must come + # AFTER the custom_providers branch so per-model overrides aren't lost. + agent._config_context_length = _config_context_length + + agent._ensure_lmstudio_runtime_loaded(_config_context_length) + + + + # Select context engine: config-driven (like memory providers). + # 1. Check config.yaml context.engine setting + # 2. Check plugins/context_engine// directory (repo-shipped) + # 3. Check general plugin system (user-installed plugins) + # 4. Fall back to built-in ContextCompressor + _selected_engine = None + _engine_name = "compressor" # default + try: + _ctx_cfg = _agent_cfg.get("context", {}) if isinstance(_agent_cfg, dict) else {} + _engine_name = _ctx_cfg.get("engine", "compressor") or "compressor" + except Exception: + pass + + if _engine_name != "compressor": + # Try loading from plugins/context_engine// + try: + from plugins.context_engine import load_context_engine + _selected_engine = load_context_engine(_engine_name) + except Exception as _ce_load_err: + _ra().logger.debug("Context engine load from plugins/context_engine/: %s", _ce_load_err) + + # Try general plugin system as fallback + if _selected_engine is None: + try: + from hermes_cli.plugins import get_plugin_context_engine + _candidate = get_plugin_context_engine() + if _candidate and _candidate.name == _engine_name: + _selected_engine = _candidate + except Exception: + pass + + if _selected_engine is None: + _ra().logger.warning( + "Context engine '%s' not found — falling back to built-in compressor", + _engine_name, + ) + # else: config says "compressor" — use built-in, don't auto-activate plugins + + if _selected_engine is not None: + agent.context_compressor = _selected_engine + # Resolve context_length for plugin engines — mirrors switch_model() path + from agent.model_metadata import get_model_context_length + _plugin_ctx_len = get_model_context_length( + agent.model, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + config_context_length=_config_context_length, + provider=agent.provider, + custom_providers=_custom_providers, + ) + agent.context_compressor.update_model( + model=agent.model, + context_length=_plugin_ctx_len, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + ) + if not agent.quiet_mode: + _ra().logger.info("Using context engine: %s", _selected_engine.name) + else: + agent.context_compressor = ContextCompressor( + model=agent.model, + threshold_percent=compression_threshold, + protect_first_n=compression_protect_first, + protect_last_n=compression_protect_last, + summary_target_ratio=compression_target_ratio, + summary_model_override=None, + quiet_mode=agent.quiet_mode, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + config_context_length=_config_context_length, + provider=agent.provider, + api_mode=agent.api_mode, + ) + agent.compression_enabled = compression_enabled + + # Reject models whose context window is below the minimum required + # for reliable tool-calling workflows (64K tokens). + from agent.model_metadata import MINIMUM_CONTEXT_LENGTH + _ctx = getattr(agent.context_compressor, "context_length", 0) + if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH: + raise ValueError( + f"Model {agent.model} has a context window of {_ctx:,} tokens, " + f"which is below the minimum {MINIMUM_CONTEXT_LENGTH:,} required " + f"by Hermes Agent. Choose a model with at least " + f"{MINIMUM_CONTEXT_LENGTH // 1000}K context, or set " + f"model.context_length in config.yaml to override." + ) + + # Inject context engine tool schemas (e.g. lcm_grep, lcm_describe, lcm_expand). + # Skip names that are already present — the _ra().get_tool_definitions() + # quiet_mode cache returned a shared list pre-#17335, so a stray + # mutation here would poison subsequent agent inits in the same + # Gateway process and trip provider-side 'duplicate tool name' + # errors. Even with the cache fix, dedup is the right defense + # against plugin paths that may register the same schemas via + # ctx.register_tool(). Mirrors the memory tools dedup above. + agent._context_engine_tool_names: set = set() + if hasattr(agent, "context_compressor") and agent.context_compressor and agent.tools is not None: + _existing_tool_names = { + t.get("function", {}).get("name") + for t in agent.tools + if isinstance(t, dict) + } + for _schema in agent.context_compressor.get_tool_schemas(): + _tname = _schema.get("name", "") + if _tname and _tname in _existing_tool_names: + continue # already registered via plugin/cache path + _wrapped = {"type": "function", "function": _schema} + agent.tools.append(_wrapped) + if _tname: + agent.valid_tool_names.add(_tname) + agent._context_engine_tool_names.add(_tname) + _existing_tool_names.add(_tname) + + # Notify context engine of session start + if hasattr(agent, "context_compressor") and agent.context_compressor: + try: + agent.context_compressor.on_session_start( + agent.session_id, + hermes_home=str(get_hermes_home()), + platform=agent.platform or "cli", + model=agent.model, + context_length=getattr(agent.context_compressor, "context_length", 0), + ) + except Exception as _ce_err: + _ra().logger.debug("Context engine on_session_start: %s", _ce_err) + + agent._subdirectory_hints = SubdirectoryHintTracker( + working_dir=os.getenv("TERMINAL_CWD") or None, + ) + agent._user_turn_count = 0 + + # Cumulative token usage for the session + agent.session_prompt_tokens = 0 + agent.session_completion_tokens = 0 + agent.session_total_tokens = 0 + agent.session_api_calls = 0 + agent.session_input_tokens = 0 + agent.session_output_tokens = 0 + agent.session_cache_read_tokens = 0 + agent.session_cache_write_tokens = 0 + agent.session_reasoning_tokens = 0 + agent.session_estimated_cost_usd = 0.0 + agent.session_cost_status = "unknown" + agent.session_cost_source = "none" + + # ── Ollama num_ctx injection ── + # Ollama defaults to 2048 context regardless of the model's capabilities. + # When running against an Ollama server, detect the model's max context + # and pass num_ctx on every chat request so the full window is used. + # User override: set model.ollama_num_ctx in config.yaml to cap VRAM use. + # If model.context_length is set, it caps num_ctx so the user's VRAM + # budget is respected even when GGUF metadata advertises a larger window. + agent._ollama_num_ctx: int | None = None + _ollama_num_ctx_override = None + if isinstance(_model_cfg, dict): + _ollama_num_ctx_override = _model_cfg.get("ollama_num_ctx") + if _ollama_num_ctx_override is not None: + try: + agent._ollama_num_ctx = int(_ollama_num_ctx_override) + except (TypeError, ValueError): + _ra().logger.debug("Invalid ollama_num_ctx config value: %r", _ollama_num_ctx_override) + if agent._ollama_num_ctx is None and agent.base_url and is_local_endpoint(agent.base_url): + try: + _detected = query_ollama_num_ctx(agent.model, agent.base_url, api_key=agent.api_key or "") + if _detected and _detected > 0: + agent._ollama_num_ctx = _detected + except Exception as exc: + _ra().logger.debug("Ollama num_ctx detection failed: %s", exc) + # Cap auto-detected ollama_num_ctx to the user's explicit context_length. + # Without this, GGUF metadata can advertise 256K+ which Ollama honours + # by allocating that much VRAM — blowing up small GPUs even though the + # user explicitly set a smaller context_length in config.yaml. + if ( + agent._ollama_num_ctx + and _config_context_length + and _ollama_num_ctx_override is None # don't override explicit ollama_num_ctx + and agent._ollama_num_ctx > _config_context_length + ): + _ra().logger.info( + "Ollama num_ctx capped: %d -> %d (model.context_length override)", + agent._ollama_num_ctx, _config_context_length, + ) + agent._ollama_num_ctx = _config_context_length + if agent._ollama_num_ctx and not agent.quiet_mode: + _ra().logger.info( + "Ollama num_ctx: will request %d tokens (model max from /api/show)", + agent._ollama_num_ctx, + ) + + if not agent.quiet_mode: + if compression_enabled: + print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(compression_threshold*100)}% = {agent.context_compressor.threshold_tokens:,})") + else: + print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (auto-compression disabled)") + + # Check immediately so CLI users see the warning at startup. + # Gateway status_callback is not yet wired, so any warning is stored + # in _compression_warning and replayed in the first run_conversation(). + agent._compression_warning = None + agent._check_compression_model_feasibility() + + # Snapshot primary runtime for per-turn restoration. When fallback + # activates during a turn, the next turn restores these values so the + # preferred model gets a fresh attempt each time. Uses a single dict + # so new state fields are easy to add without N individual attributes. + _cc = agent.context_compressor + agent._primary_runtime = { + "model": agent.model, + "provider": agent.provider, + "base_url": agent.base_url, + "api_mode": agent.api_mode, + "api_key": getattr(agent, "api_key", ""), + "client_kwargs": dict(agent._client_kwargs), + "use_prompt_caching": agent._use_prompt_caching, + "use_native_cache_layout": agent._use_native_cache_layout, + # Context engine state that _try_activate_fallback() overwrites. + # Use getattr for model/base_url/api_key/provider since plugin + # engines may not have these (they're ContextCompressor-specific). + "compressor_model": getattr(_cc, "model", agent.model), + "compressor_base_url": getattr(_cc, "base_url", agent.base_url), + "compressor_api_key": getattr(_cc, "api_key", ""), + "compressor_provider": getattr(_cc, "provider", agent.provider), + "compressor_context_length": _cc.context_length, + "compressor_threshold_tokens": _cc.threshold_tokens, + } + if agent.api_mode == "anthropic_messages": + agent._primary_runtime.update({ + "anthropic_api_key": agent._anthropic_api_key, + "anthropic_base_url": agent._anthropic_base_url, + "is_anthropic_oauth": agent._is_anthropic_oauth, + }) + + + +__all__ = ["init_agent"] diff --git a/run_agent.py b/run_agent.py index b13eb851175..05d648f94e2 100644 --- a/run_agent.py +++ b/run_agent.py @@ -374,1319 +374,9 @@ class AIAgent: checkpoint_max_file_size_mb: int = 10, pass_session_id: bool = False, ): - """ - Initialize the AI Agent. - - Args: - base_url (str): Base URL for the model API (optional) - api_key (str): API key for authentication (optional, uses env var if not provided) - provider (str): Provider identifier (optional; used for telemetry/routing hints) - api_mode (str): API mode override: "chat_completions" or "codex_responses" - model (str): Model name to use (default: "anthropic/claude-opus-4.6") - max_iterations (int): Maximum number of tool calling iterations (default: 90) - tool_delay (float): Delay between tool calls in seconds (default: 1.0) - enabled_toolsets (List[str]): Only enable tools from these toolsets (optional) - disabled_toolsets (List[str]): Disable tools from these toolsets (optional) - save_trajectories (bool): Whether to save conversation trajectories to JSONL files (default: False) - verbose_logging (bool): Enable verbose logging for debugging (default: False) - quiet_mode (bool): Suppress progress output for clean CLI experience (default: False) - ephemeral_system_prompt (str): System prompt used during agent execution but NOT saved to trajectories (optional) - log_prefix_chars (int): Number of characters to show in log previews for tool calls/responses (default: 100) - log_prefix (str): Prefix to add to all log messages for identification in parallel processing (default: "") - providers_allowed (List[str]): OpenRouter providers to allow (optional) - providers_ignored (List[str]): OpenRouter providers to ignore (optional) - providers_order (List[str]): OpenRouter providers to try in order (optional) - provider_sort (str): Sort providers by price/throughput/latency (optional) - openrouter_min_coding_score (float): Coding-score floor (0.0-1.0) for the - openrouter/pareto-code router. Only applied when model == "openrouter/pareto-code". - None or empty = let OpenRouter pick the strongest available coder. - session_id (str): Pre-generated session ID for logging (optional, auto-generated if not provided) - tool_progress_callback (callable): Callback function(tool_name, args_preview) for progress notifications - clarify_callback (callable): Callback function(question, choices) -> str for interactive user questions. - Provided by the platform layer (CLI or gateway). If None, the clarify tool returns an error. - max_tokens (int): Maximum tokens for model responses (optional, uses model default if not set) - reasoning_config (Dict): OpenRouter reasoning configuration override (e.g. {"effort": "none"} to disable thinking). - If None, defaults to {"enabled": True, "effort": "medium"} for OpenRouter. Set to disable/customize reasoning. - prefill_messages (List[Dict]): Messages to prepend to conversation history as prefilled context. - Useful for injecting a few-shot example or priming the model's response style. - Example: [{"role": "user", "content": "Hi!"}, {"role": "assistant", "content": "Hello!"}] - NOTE: Anthropic Sonnet 4.6+ and Opus 4.6+ reject a conversation that ends on an - assistant-role message (400 error). For those models use structured outputs or - output_config.format instead of a trailing-assistant prefill. - platform (str): The interface platform the user is on (e.g. "cli", "telegram", "discord", "whatsapp"). - Used to inject platform-specific formatting hints into the system prompt. - skip_context_files (bool): If True, skip auto-injection of SOUL.md, AGENTS.md, and .cursorrules - into the system prompt. Use this for batch processing and data generation to avoid - polluting trajectories with user-specific persona or project instructions. - load_soul_identity (bool): If True, still use ~/.hermes/SOUL.md as the primary - identity even when skip_context_files=True. Project context files from the cwd - remain skipped. - """ - _install_safe_stdio() - - self.model = model - self.max_iterations = max_iterations - # Shared iteration budget — parent creates, children inherit. - # Consumed by every LLM turn across parent + all subagents. - self.iteration_budget = iteration_budget or IterationBudget(max_iterations) - self.tool_delay = tool_delay - self.save_trajectories = save_trajectories - self.verbose_logging = verbose_logging - self.quiet_mode = quiet_mode - self.ephemeral_system_prompt = ephemeral_system_prompt - self.platform = platform # "cli", "telegram", "discord", "whatsapp", etc. - self._user_id = user_id # Platform user identifier (gateway sessions) - self._user_name = user_name - self._chat_id = chat_id - self._chat_name = chat_name - self._chat_type = chat_type - self._thread_id = thread_id - self._gateway_session_key = gateway_session_key # Stable per-chat key (e.g. agent:main:telegram:dm:123) - # Pluggable print function — CLI replaces this with _cprint so that - # raw ANSI status lines are routed through prompt_toolkit's renderer - # instead of going directly to stdout where patch_stdout's StdoutProxy - # would mangle the escape sequences. None = use builtins.print. - self._print_fn = None - self.background_review_callback = None # Optional sync callback for gateway delivery - self.skip_context_files = skip_context_files - self.load_soul_identity = load_soul_identity - self.pass_session_id = pass_session_id - self._credential_pool = credential_pool - self.log_prefix_chars = log_prefix_chars - self.log_prefix = f"{log_prefix} " if log_prefix else "" - # Store effective base URL for feature detection (prompt caching, reasoning, etc.) - self.base_url = base_url or "" - provider_name = provider.strip().lower() if isinstance(provider, str) and provider.strip() else None - self.provider = provider_name or "" - self.acp_command = acp_command or command - self.acp_args = list(acp_args or args or []) - if api_mode in {"chat_completions", "codex_responses", "anthropic_messages", "bedrock_converse", "codex_app_server"}: - self.api_mode = api_mode - elif self.provider == "openai-codex": - self.api_mode = "codex_responses" - elif self.provider == "xai": - self.api_mode = "codex_responses" - elif (provider_name is None) and ( - self._base_url_hostname == "chatgpt.com" - and "/backend-api/codex" in self._base_url_lower - ): - self.api_mode = "codex_responses" - self.provider = "openai-codex" - elif (provider_name is None) and self._base_url_hostname == "api.x.ai": - self.api_mode = "codex_responses" - self.provider = "xai" - elif self.provider == "anthropic" or (provider_name is None and self._base_url_hostname == "api.anthropic.com"): - self.api_mode = "anthropic_messages" - self.provider = "anthropic" - elif self._base_url_lower.rstrip("/").endswith("/anthropic"): - # Third-party Anthropic-compatible endpoints (e.g. MiniMax, DashScope) - # use a URL convention ending in /anthropic. Auto-detect these so the - # Anthropic Messages API adapter is used instead of chat completions. - self.api_mode = "anthropic_messages" - elif self.provider == "bedrock" or ( - self._base_url_hostname.startswith("bedrock-runtime.") - and base_url_host_matches(self._base_url_lower, "amazonaws.com") - ): - # AWS Bedrock — auto-detect from provider name or base URL - # (bedrock-runtime..amazonaws.com). - self.api_mode = "bedrock_converse" - else: - self.api_mode = "chat_completions" - - # Eagerly warm the transport cache so import errors surface at init, - # not mid-conversation. Also validates the api_mode is registered. - try: - self._get_transport() - except Exception: - pass # Non-fatal — transport may not exist for all modes yet - - try: - from hermes_cli.model_normalize import ( - _AGGREGATOR_PROVIDERS, - normalize_model_for_provider, - ) - - if self.provider not in _AGGREGATOR_PROVIDERS: - self.model = normalize_model_for_provider(self.model, self.provider) - except Exception: - pass - - # GPT-5.x models usually require the Responses API path, but some - # providers have exceptions (for example Copilot's gpt-5-mini still - # uses chat completions). Also auto-upgrade for direct OpenAI URLs - # (api.openai.com) since all newer tool-calling models prefer - # Responses there. ACP runtimes are excluded: CopilotACPClient - # handles its own routing and does not implement the Responses API - # surface. - # When api_mode was explicitly provided, respect it — the user - # knows what their endpoint supports (#10473). - # Exception: Azure OpenAI serves gpt-5.x on /chat/completions and - # does NOT support the Responses API — skip the upgrade for Azure - # (openai.azure.com), even though it looks OpenAI-compatible. - if ( - api_mode is None - and self.api_mode == "chat_completions" - and self.provider != "copilot-acp" - and not str(self.base_url or "").lower().startswith("acp://copilot") - and not str(self.base_url or "").lower().startswith("acp+tcp://") - and not self._is_azure_openai_url() - and ( - self._is_direct_openai_url() - or self._provider_model_requires_responses_api( - self.model, - provider=self.provider, - ) - ) - ): - self.api_mode = "codex_responses" - # Invalidate the eager-warmed transport cache — api_mode changed - # from chat_completions to codex_responses after the warm at __init__. - if hasattr(self, "_transport_cache"): - self._transport_cache.clear() - - # Pre-warm OpenRouter model metadata cache in a background thread. - # fetch_model_metadata() is cached for 1 hour; this avoids a blocking - # HTTP request on the first API response when pricing is estimated. - # Use a process-level Event so this thread is only spawned once — a new - # AIAgent is created for every gateway request, so without the guard - # each message leaks one OS thread and the process eventually exhausts - # the system thread limit (RuntimeError: can't start new thread). - if (self.provider == "openrouter" or self._is_openrouter_url()) and \ - not _openrouter_prewarm_done.is_set(): - _openrouter_prewarm_done.set() - threading.Thread( - target=fetch_model_metadata, - daemon=True, - name="openrouter-prewarm", - ).start() - - self.tool_progress_callback = tool_progress_callback - self.tool_start_callback = tool_start_callback - self.tool_complete_callback = tool_complete_callback - self.suppress_status_output = False - self.thinking_callback = thinking_callback - self.reasoning_callback = reasoning_callback - self.clarify_callback = clarify_callback - self.step_callback = step_callback - self.stream_delta_callback = stream_delta_callback - self.interim_assistant_callback = interim_assistant_callback - self.status_callback = status_callback - self.tool_gen_callback = tool_gen_callback - - - # Tool execution state — allows _vprint during tool execution - # even when stream consumers are registered (no tokens streaming then) - self._executing_tools = False - self._tool_guardrails = ToolCallGuardrailController() - self._tool_guardrail_halt_decision: ToolGuardrailDecision | None = None - - # Interrupt mechanism for breaking out of tool loops - self._interrupt_requested = False - self._interrupt_message = None # Optional message that triggered interrupt - self._execution_thread_id: int | None = None # Set at run_conversation() start - self._interrupt_thread_signal_pending = False - self._client_lock = threading.RLock() - - # /steer mechanism — inject a user note into the next tool result - # without interrupting the agent. Unlike interrupt(), steer() does - # NOT set _interrupt_requested; it waits for the current tool batch - # to finish naturally, then the drain hook appends the text to the - # last tool result's content so the model sees it on its next - # iteration. Message-role alternation is preserved (we modify an - # existing tool message rather than inserting a new user turn). - self._pending_steer: Optional[str] = None - self._pending_steer_lock = threading.Lock() - - # Concurrent-tool worker thread tracking. `_execute_tool_calls_concurrent` - # runs each tool on its own ThreadPoolExecutor worker — those worker - # threads have tids distinct from `_execution_thread_id`, so - # `_set_interrupt(True, _execution_thread_id)` alone does NOT cause - # `is_interrupted()` inside the worker to return True. Track the - # workers here so `interrupt()` / `clear_interrupt()` can fan out to - # their tids explicitly. - self._tool_worker_threads: set[int] = set() - self._tool_worker_threads_lock = threading.Lock() - - # Subagent delegation state - self._delegate_depth = 0 # 0 = top-level agent, incremented for children - self._active_children = [] # Running child AIAgents (for interrupt propagation) - self._active_children_lock = threading.Lock() - - # Store OpenRouter provider preferences - self.providers_allowed = providers_allowed - self.providers_ignored = providers_ignored - self.providers_order = providers_order - self.provider_sort = provider_sort - self.provider_require_parameters = provider_require_parameters - self.provider_data_collection = provider_data_collection - self.openrouter_min_coding_score = openrouter_min_coding_score - - # Store toolset filtering options - self.enabled_toolsets = enabled_toolsets - self.disabled_toolsets = disabled_toolsets - - # Model response configuration - self.max_tokens = max_tokens # None = use model default - self.reasoning_config = reasoning_config # None = use default (medium for OpenRouter) - self.service_tier = service_tier - self.request_overrides = dict(request_overrides or {}) - self.prefill_messages = prefill_messages or [] # Prefilled conversation turns - self._force_ascii_payload = False - - # Anthropic prompt caching: auto-enabled for Claude models on native - # Anthropic, OpenRouter, and third-party gateways that speak the - # Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces - # input costs by ~75% on multi-turn conversations. Uses system_and_3 - # strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy`` - # for the layout-vs-transport decision. - self._use_prompt_caching, self._use_native_cache_layout = ( - self._anthropic_prompt_cache_policy() - ) - # Anthropic supports "5m" (default) and "1h" cache TTL tiers. Read from - # config.yaml under prompt_caching.cache_ttl; unknown values keep "5m". - # 1h tier costs 2x on write vs 1.25x for 5m, but amortizes across long - # sessions with >5-minute pauses between turns (#14971). - self._cache_ttl = "5m" - try: - from hermes_cli.config import load_config as _load_pc_cfg - - _pc_cfg = _load_pc_cfg().get("prompt_caching", {}) or {} - _ttl = _pc_cfg.get("cache_ttl", "5m") - if _ttl in {"5m", "1h"}: - self._cache_ttl = _ttl - except Exception: - pass - - # Iteration budget: the LLM is only notified when it actually exhausts - # the iteration budget (api_call_count >= max_iterations). At that - # point we inject ONE message, allow one final API call, and if the - # model doesn't produce a text response, force a user-message asking - # it to summarise. No intermediate pressure warnings — they caused - # models to "give up" prematurely on complex tasks (#7915). - self._budget_exhausted_injected = False - self._budget_grace_call = False - - # Activity tracking — updated on each API call, tool execution, and - # stream chunk. Used by the gateway timeout handler to report what the - # agent was doing when it was killed, and by the "still working" - # notifications to show progress. - self._last_activity_ts: float = time.time() - self._last_activity_desc: str = "initializing" - self._current_tool: str | None = None - self._api_call_count: int = 0 - - # Rate limit tracking — updated from x-ratelimit-* response headers - # after each API call. Accessed by /usage slash command. - self._rate_limit_state: Optional["RateLimitState"] = None - - # OpenRouter response cache hit counter — incremented when - # X-OpenRouter-Cache-Status: HIT is seen in streaming response headers. - self._or_cache_hits: int = 0 - - # Centralized logging — agent.log (INFO+) and errors.log (WARNING+) - # both live under ~/.hermes/logs/. Idempotent, so gateway mode - # (which creates a new AIAgent per message) won't duplicate handlers. - from hermes_logging import setup_logging, setup_verbose_logging - setup_logging(hermes_home=_hermes_home) - - if self.verbose_logging: - setup_verbose_logging() - logger.info("Verbose logging enabled (third-party library logs suppressed)") - elif self.quiet_mode: - # In quiet mode (CLI default), keep console output clean — - # but DO NOT raise per-logger levels. Doing so prevents the - # root logger's file handlers (agent.log, errors.log) from - # ever seeing the records, because Python checks - # logger.isEnabledFor() before handler propagation. We rely - # on the fact that hermes_logging.setup_logging() does not - # install a console StreamHandler in quiet mode — so INFO - # records flow to the file handlers but never reach a - # console. Any future noise reduction belongs at the - # handler level inside hermes_logging.py, not here. - pass - - # Internal stream callback (set during streaming TTS). - # Initialized here so _vprint can reference it before run_conversation. - self._stream_callback = None - # Deferred paragraph break flag — set after tool iterations so a - # single "\n\n" is prepended to the next real text delta. - self._stream_needs_break = False - # Stateful scrubber for spans split across stream - # deltas (#5719). sanitize_context() alone can't survive chunk - # boundaries because the block regex needs both tags in one string. - self._stream_context_scrubber = StreamingContextScrubber() - # Stateful scrubber for reasoning/thinking tags in streamed deltas - # (#17924). Replaces the per-delta _strip_think_blocks regex that - # destroyed downstream state (e.g. MiniMax-M2.7 streaming - # '' as delta1 and 'Let me check' as delta2 — the regex - # erased delta1, so downstream state machines never learned a - # block was open and leaked delta2 as content). - self._stream_think_scrubber = StreamingThinkScrubber() - # Visible assistant text already delivered through live token callbacks - # during the current model response. Used to avoid re-sending the same - # commentary when the provider later returns it as a completed interim - # assistant message. - self._current_streamed_assistant_text = "" - - # Optional current-turn user-message override used when the API-facing - # user message intentionally differs from the persisted transcript - # (e.g. CLI voice mode adds a temporary prefix for the live call only). - self._persist_user_message_idx = None - self._persist_user_message_override = None - - # Cache anthropic image-to-text fallbacks per image payload/URL so a - # single tool loop does not repeatedly re-run auxiliary vision on the - # same image history. - self._anthropic_image_fallback_cache: Dict[str, str] = {} - - # Initialize LLM client via centralized provider router. - # The router handles auth resolution, base URL, headers, and - # Codex/Anthropic wrapping for all known providers. - # raw_codex=True because the main agent needs direct responses.stream() - # access for Codex Responses API streaming. - self._anthropic_client = None - self._is_anthropic_oauth = False - - # Resolve per-provider / per-model request timeout once up front so - # every client construction path below (Anthropic native, OpenAI-wire, - # router-based implicit auth) can apply it consistently. Bedrock - # Claude uses its own timeout path and is not covered here. - _provider_timeout = get_provider_request_timeout(self.provider, self.model) - - if self.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client, resolve_anthropic_token - # Bedrock + Claude → use AnthropicBedrock SDK for full feature parity - # (prompt caching, thinking budgets, adaptive thinking). - _is_bedrock_anthropic = self.provider == "bedrock" - if _is_bedrock_anthropic: - from agent.anthropic_adapter import build_anthropic_bedrock_client - _region_match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") - _br_region = _region_match.group(1) if _region_match else "us-east-1" - self._bedrock_region = _br_region - self._anthropic_client = build_anthropic_bedrock_client(_br_region) - self._anthropic_api_key = "aws-sdk" - self._anthropic_base_url = base_url - self._is_anthropic_oauth = False - self.api_key = "aws-sdk" - self.client = None - self._client_kwargs = {} - if not self.quiet_mode: - print(f"🤖 AI Agent initialized with model: {self.model} (AWS Bedrock + AnthropicBedrock SDK, {_br_region})") - else: - # Only fall back to ANTHROPIC_TOKEN when the provider is actually Anthropic. - # Other anthropic_messages providers (MiniMax, Alibaba, etc.) must use their own API key. - # Falling back would send Anthropic credentials to third-party endpoints (Fixes #1739, #minimax-401). - _is_native_anthropic = self.provider == "anthropic" - effective_key = (api_key or resolve_anthropic_token() or "") if _is_native_anthropic else (api_key or "") - self.api_key = effective_key - self._anthropic_api_key = effective_key - self._anthropic_base_url = base_url - # Only mark the session as OAuth-authenticated when the token - # genuinely belongs to native Anthropic. Third-party providers - # (MiniMax, Kimi, GLM, LiteLLM proxies) that accept the - # Anthropic protocol must never trip OAuth code paths — doing - # so injects Claude-Code identity headers and system prompts - # that cause 401/403 on their endpoints. Guards #1739 and - # the third-party identity-injection bug. - from agent.anthropic_adapter import _is_oauth_token as _is_oat - self._is_anthropic_oauth = _is_oat(effective_key) if _is_native_anthropic else False - self._anthropic_client = build_anthropic_client(effective_key, base_url, timeout=_provider_timeout) - # No OpenAI client needed for Anthropic mode - self.client = None - self._client_kwargs = {} - if not self.quiet_mode: - print(f"🤖 AI Agent initialized with model: {self.model} (Anthropic native)") - if effective_key and len(effective_key) > 12: - print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}") - elif self.api_mode == "bedrock_converse": - # AWS Bedrock — uses boto3 directly, no OpenAI client needed. - # Region is extracted from the base_url or defaults to us-east-1. - _region_match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") - self._bedrock_region = _region_match.group(1) if _region_match else "us-east-1" - # Guardrail config — read from config.yaml at init time. - self._bedrock_guardrail_config = None - try: - from hermes_cli.config import load_config as _load_br_cfg - _gr = _load_br_cfg().get("bedrock", {}).get("guardrail", {}) - if _gr.get("guardrail_identifier") and _gr.get("guardrail_version"): - self._bedrock_guardrail_config = { - "guardrailIdentifier": _gr["guardrail_identifier"], - "guardrailVersion": _gr["guardrail_version"], - } - if _gr.get("stream_processing_mode"): - self._bedrock_guardrail_config["streamProcessingMode"] = _gr["stream_processing_mode"] - if _gr.get("trace"): - self._bedrock_guardrail_config["trace"] = _gr["trace"] - except Exception: - pass - self.client = None - self._client_kwargs = {} - if not self.quiet_mode: - _gr_label = " + Guardrails" if self._bedrock_guardrail_config else "" - print(f"🤖 AI Agent initialized with model: {self.model} (AWS Bedrock, {self._bedrock_region}{_gr_label})") - else: - if api_key and base_url: - # Explicit credentials from CLI/gateway — construct directly. - # The runtime provider resolver already handled auth for us. - # Extract query params (e.g. Azure api-version) from base_url - # and pass via default_query to prevent loss during SDK URL - # joining (httpx drops query string when joining paths). - _parsed_url = urlparse(base_url) - if _parsed_url.query: - _clean_url = urlunparse(_parsed_url._replace(query="")) - _query_params = { - k: v[0] for k, v in parse_qs(_parsed_url.query).items() - } - client_kwargs = { - "api_key": api_key, - "base_url": _clean_url, - "default_query": _query_params, - } - else: - client_kwargs = {"api_key": api_key, "base_url": base_url} - if _provider_timeout is not None: - client_kwargs["timeout"] = _provider_timeout - if self.provider == "copilot-acp": - client_kwargs["command"] = self.acp_command - client_kwargs["args"] = self.acp_args - effective_base = base_url - if base_url_host_matches(effective_base, "openrouter.ai"): - from agent.auxiliary_client import build_or_headers - client_kwargs["default_headers"] = build_or_headers() - elif base_url_host_matches(effective_base, "api.routermint.com"): - client_kwargs["default_headers"] = _routermint_headers() - elif base_url_host_matches(effective_base, "api.githubcopilot.com"): - from hermes_cli.models import copilot_default_headers - - client_kwargs["default_headers"] = copilot_default_headers() - elif base_url_host_matches(effective_base, "api.kimi.com"): - client_kwargs["default_headers"] = { - "User-Agent": "claude-code/0.1.0", - } - elif base_url_host_matches(effective_base, "portal.qwen.ai"): - client_kwargs["default_headers"] = _qwen_portal_headers() - elif base_url_host_matches(effective_base, "chatgpt.com"): - from agent.auxiliary_client import _codex_cloudflare_headers - client_kwargs["default_headers"] = _codex_cloudflare_headers(api_key) - elif "default_headers" not in client_kwargs: - # Fall back to profile.default_headers for providers that - # declare custom headers (e.g. Vercel AI Gateway attribution, - # Kimi User-Agent on non-kimi.com endpoints). - try: - from providers import get_provider_profile as _gpf - _ph = _gpf(self.provider) - if _ph and _ph.default_headers: - client_kwargs["default_headers"] = dict(_ph.default_headers) - except Exception: - pass - else: - # No explicit creds — use the centralized provider router - from agent.auxiliary_client import resolve_provider_client - _routed_client, _ = resolve_provider_client( - self.provider or "auto", model=self.model, raw_codex=True) - if _routed_client is not None: - client_kwargs = { - "api_key": _routed_client.api_key, - "base_url": str(_routed_client.base_url), - } - if _provider_timeout is not None: - client_kwargs["timeout"] = _provider_timeout - # Preserve any default_headers the router set - if hasattr(_routed_client, '_default_headers') and _routed_client._default_headers: - client_kwargs["default_headers"] = dict(_routed_client._default_headers) - else: - # When the user explicitly chose a non-OpenRouter provider - # but no credentials were found, fail fast with a clear - # message instead of silently routing through OpenRouter. - _explicit = (self.provider or "").strip().lower() - if _explicit and _explicit not in {"auto", "openrouter", "custom"}: - # Look up the actual env var name from the provider - # config — some providers use non-standard names - # (e.g. alibaba → DASHSCOPE_API_KEY, not ALIBABA_API_KEY). - _env_hint = f"{_explicit.upper()}_API_KEY" - try: - from hermes_cli.auth import PROVIDER_REGISTRY - _pcfg = PROVIDER_REGISTRY.get(_explicit) - if _pcfg and _pcfg.api_key_env_vars: - _env_hint = _pcfg.api_key_env_vars[0] - except Exception: - pass - # --- Init-time fallback (#17929) --- - _fb_entries = [] - if isinstance(fallback_model, list): - _fb_entries = [ - f for f in fallback_model - if isinstance(f, dict) and f.get("provider") and f.get("model") - ] - elif isinstance(fallback_model, dict) and fallback_model.get("provider") and fallback_model.get("model"): - _fb_entries = [fallback_model] - _fb_resolved = False - for _fb in _fb_entries: - _fb_explicit_key = (_fb.get("api_key") or "").strip() or None - if not _fb_explicit_key: - _fb_key_env = (_fb.get("key_env") or _fb.get("api_key_env") or "").strip() - if _fb_key_env: - _fb_explicit_key = os.getenv(_fb_key_env, "").strip() or None - _fb_client, _fb_model = resolve_provider_client( - _fb["provider"], model=_fb["model"], raw_codex=True, - explicit_base_url=_fb.get("base_url"), - explicit_api_key=_fb_explicit_key, - ) - if _fb_client is not None: - self.provider = _fb["provider"] - self.model = _fb_model or _fb["model"] - self._fallback_activated = True - client_kwargs = { - "api_key": _fb_client.api_key, - "base_url": str(_fb_client.base_url), - } - if _provider_timeout is not None: - client_kwargs["timeout"] = _provider_timeout - if hasattr(_fb_client, "_default_headers") and _fb_client._default_headers: - client_kwargs["default_headers"] = dict(_fb_client._default_headers) - _fb_resolved = True - break - if not _fb_resolved: - raise RuntimeError( - f"Provider '{_explicit}' is set in config.yaml but no API key " - f"was found. Set the {_env_hint} environment " - f"variable, or switch to a different provider with `hermes model`." - ) - if not getattr(self, "_fallback_activated", False): - # No provider configured — reject with a clear message. - raise RuntimeError( - "No LLM provider configured. Run `hermes model` to " - "select a provider, or run `hermes setup` for first-time " - "configuration." - ) - - self._client_kwargs = client_kwargs # stored for rebuilding after interrupt - - # Enable fine-grained tool streaming for Claude on OpenRouter. - # Without this, Anthropic buffers the entire tool call and goes - # silent for minutes while thinking — OpenRouter's upstream proxy - # times out during the silence. The beta header makes Anthropic - # stream tool call arguments token-by-token, keeping the - # connection alive. - _effective_base = str(client_kwargs.get("base_url", "")).lower() - if base_url_host_matches(_effective_base, "openrouter.ai") and "claude" in (self.model or "").lower(): - headers = client_kwargs.get("default_headers") or {} - existing_beta = headers.get("x-anthropic-beta", "") - _FINE_GRAINED = "fine-grained-tool-streaming-2025-05-14" - if _FINE_GRAINED not in existing_beta: - if existing_beta: - headers["x-anthropic-beta"] = f"{existing_beta},{_FINE_GRAINED}" - else: - headers["x-anthropic-beta"] = _FINE_GRAINED - client_kwargs["default_headers"] = headers - - self.api_key = client_kwargs.get("api_key", "") - self.base_url = client_kwargs.get("base_url", self.base_url) - try: - self.client = self._create_openai_client(client_kwargs, reason="agent_init", shared=True) - if not self.quiet_mode: - print(f"🤖 AI Agent initialized with model: {self.model}") - if base_url: - print(f"🔗 Using custom base URL: {base_url}") - # Always show API key info (masked) for debugging auth issues - key_used = client_kwargs.get("api_key", "none") - if key_used and key_used != "dummy-key" and len(key_used) > 12: - print(f"🔑 Using API key: {key_used[:8]}...{key_used[-4:]}") - else: - print(f"⚠️ Warning: API key appears invalid or missing (got: '{key_used[:20] if key_used else 'none'}...')") - except Exception as e: - raise RuntimeError(f"Failed to initialize OpenAI client: {e}") - - # Provider fallback chain — ordered list of backup providers tried - # when the primary is exhausted (rate-limit, overload, connection - # failure). Supports both legacy single-dict ``fallback_model`` and - # new list ``fallback_providers`` format. - if isinstance(fallback_model, list): - self._fallback_chain = [ - f for f in fallback_model - if isinstance(f, dict) and f.get("provider") and f.get("model") - ] - elif isinstance(fallback_model, dict) and fallback_model.get("provider") and fallback_model.get("model"): - self._fallback_chain = [fallback_model] - else: - self._fallback_chain = [] - self._fallback_index = 0 - self._fallback_activated = getattr(self, "_fallback_activated", False) - # Legacy attribute kept for backward compat (tests, external callers) - self._fallback_model = self._fallback_chain[0] if self._fallback_chain else None - if self._fallback_chain and not self.quiet_mode: - if len(self._fallback_chain) == 1: - fb = self._fallback_chain[0] - print(f"🔄 Fallback model: {fb['model']} ({fb['provider']})") - else: - print(f"🔄 Fallback chain ({len(self._fallback_chain)} providers): " + - " → ".join(f"{f['model']} ({f['provider']})" for f in self._fallback_chain)) - - # Get available tools with filtering - self.tools = get_tool_definitions( - enabled_toolsets=enabled_toolsets, - disabled_toolsets=disabled_toolsets, - quiet_mode=self.quiet_mode, - ) - - # Show tool configuration and store valid tool names for validation - self.valid_tool_names = set() - if self.tools: - self.valid_tool_names = {tool["function"]["name"] for tool in self.tools} - tool_names = sorted(self.valid_tool_names) - if not self.quiet_mode: - print(f"🛠️ Loaded {len(self.tools)} tools: {', '.join(tool_names)}") - - # Show filtering info if applied - if enabled_toolsets: - print(f" ✅ Enabled toolsets: {', '.join(enabled_toolsets)}") - if disabled_toolsets: - print(f" ❌ Disabled toolsets: {', '.join(disabled_toolsets)}") - elif not self.quiet_mode: - print("🛠️ No tools loaded (all tools filtered out or unavailable)") - - # Check tool requirements - if self.tools and not self.quiet_mode: - requirements = check_toolset_requirements() - missing_reqs = [name for name, available in requirements.items() if not available] - if missing_reqs: - print(f"⚠️ Some tools may not work due to missing requirements: {missing_reqs}") - - # Show trajectory saving status - if self.save_trajectories and not self.quiet_mode: - print("📝 Trajectory saving enabled") - - # Show ephemeral system prompt status - if self.ephemeral_system_prompt and not self.quiet_mode: - prompt_preview = self.ephemeral_system_prompt[:60] + "..." if len(self.ephemeral_system_prompt) > 60 else self.ephemeral_system_prompt - print(f"🔒 Ephemeral system prompt: '{prompt_preview}' (not saved to trajectories)") - - # Show prompt caching status - if self._use_prompt_caching and not self.quiet_mode: - if self._use_native_cache_layout and self.provider == "anthropic": - source = "native Anthropic" - elif self._use_native_cache_layout: - source = "Anthropic-compatible endpoint" - else: - source = "Claude via OpenRouter" - print(f"💾 Prompt caching: ENABLED ({source}, {self._cache_ttl} TTL)") - - # Session logging setup - auto-save conversation trajectories for debugging - self.session_start = datetime.now() - if session_id: - # Use provided session ID (e.g., from CLI) - self.session_id = session_id - else: - # Generate a new session ID - timestamp_str = self.session_start.strftime("%Y%m%d_%H%M%S") - short_uuid = uuid.uuid4().hex[:6] - self.session_id = f"{timestamp_str}_{short_uuid}" - - # Expose session ID to tools (terminal, execute_code) so agents can - # reference their own session for --resume commands, cross-session - # coordination, and logging. Uses the ContextVar system from - # session_context.py for concurrency safety (gateway runs multiple - # sessions in one process). Also writes os.environ as fallback for - # CLI mode where ContextVars aren't used. - os.environ["HERMES_SESSION_ID"] = self.session_id - try: - from gateway.session_context import _SESSION_ID - _SESSION_ID.set(self.session_id) - except Exception: - pass # CLI/test mode — ContextVar not needed - - # Session logs go into ~/.hermes/sessions/ alongside gateway sessions - hermes_home = get_hermes_home() - self.logs_dir = hermes_home / "sessions" - self.logs_dir.mkdir(parents=True, exist_ok=True) - self.session_log_file = self.logs_dir / f"session_{self.session_id}.json" - - # Track conversation messages for session logging - self._session_messages: List[Dict[str, Any]] = [] - self._memory_write_origin = "assistant_tool" - self._memory_write_context = "foreground" - - # Cached system prompt -- built once per session, only rebuilt on compression - self._cached_system_prompt: Optional[str] = None - - # Filesystem checkpoint manager (transparent — not a tool) - from tools.checkpoint_manager import CheckpointManager - self._checkpoint_mgr = CheckpointManager( - enabled=checkpoints_enabled, - max_snapshots=checkpoint_max_snapshots, - max_total_size_mb=checkpoint_max_total_size_mb, - max_file_size_mb=checkpoint_max_file_size_mb, - ) - - # SQLite session store (optional -- provided by CLI or gateway) - self._session_db = session_db - self._parent_session_id = parent_session_id - self._last_flushed_db_idx = 0 # tracks DB-write cursor to prevent duplicate writes - self._session_db_created = False # DB row deferred to run_conversation() - self._session_init_model_config = { - "max_iterations": self.max_iterations, - "reasoning_config": reasoning_config, - "max_tokens": max_tokens, - } - - # In-memory todo list for task planning (one per agent/session) - from tools.todo_tool import TodoStore - self._todo_store = TodoStore() - - # Load config once for memory, skills, and compression sections - try: - from hermes_cli.config import load_config as _load_agent_config - _agent_cfg = _load_agent_config() - except Exception: - _agent_cfg = {} - try: - self._tool_guardrails = ToolCallGuardrailController( - ToolCallGuardrailConfig.from_mapping( - _agent_cfg.get("tool_loop_guardrails", {}) - ) - ) - except Exception as _tlg_err: - logger.warning("Tool loop guardrail config ignored: %s", _tlg_err) - # Cache only the derived auxiliary compression context override that is - # needed later by the startup feasibility check. Avoid exposing a - # broad pseudo-public config object on the agent instance. - self._aux_compression_context_length_config = None - - # Persistent memory (MEMORY.md + USER.md) -- loaded from disk - self._memory_store = None - self._memory_enabled = False - self._user_profile_enabled = False - self._memory_nudge_interval = 10 - self._turns_since_memory = 0 - self._iters_since_skill = 0 - if not skip_memory: - try: - mem_config = _agent_cfg.get("memory", {}) - self._memory_enabled = mem_config.get("memory_enabled", False) - self._user_profile_enabled = mem_config.get("user_profile_enabled", False) - self._memory_nudge_interval = int(mem_config.get("nudge_interval", 10)) - if self._memory_enabled or self._user_profile_enabled: - from tools.memory_tool import MemoryStore - self._memory_store = MemoryStore( - memory_char_limit=mem_config.get("memory_char_limit", 2200), - user_char_limit=mem_config.get("user_char_limit", 1375), - ) - self._memory_store.load_from_disk() - except Exception: - pass # Memory is optional -- don't break agent init - - - - # Memory provider plugin (external — one at a time, alongside built-in) - # Reads memory.provider from config to select which plugin to activate. - self._memory_manager = None - if not skip_memory: - try: - _mem_provider_name = mem_config.get("provider", "") if mem_config else "" - - if _mem_provider_name: - from agent.memory_manager import MemoryManager as _MemoryManager - from plugins.memory import load_memory_provider as _load_mem - self._memory_manager = _MemoryManager() - _mp = _load_mem(_mem_provider_name) - if _mp and _mp.is_available(): - self._memory_manager.add_provider(_mp) - if self._memory_manager.providers: - _init_kwargs = { - "session_id": self.session_id, - "platform": platform or "cli", - "hermes_home": str(get_hermes_home()), - "agent_context": "primary", - } - # Thread session title for memory provider scoping - # (e.g. honcho uses this to derive chat-scoped session keys) - if self._session_db: - try: - _st = self._session_db.get_session_title(self.session_id) - if _st: - _init_kwargs["session_title"] = _st - except Exception: - pass - # Thread gateway user identity for per-user memory scoping - if self._user_id: - _init_kwargs["user_id"] = self._user_id - if self._user_name: - _init_kwargs["user_name"] = self._user_name - if self._chat_id: - _init_kwargs["chat_id"] = self._chat_id - if self._chat_name: - _init_kwargs["chat_name"] = self._chat_name - if self._chat_type: - _init_kwargs["chat_type"] = self._chat_type - if self._thread_id: - _init_kwargs["thread_id"] = self._thread_id - # Thread gateway session key for stable per-chat Honcho session isolation - if self._gateway_session_key: - _init_kwargs["gateway_session_key"] = self._gateway_session_key - # Profile identity for per-profile provider scoping - try: - from hermes_cli.profiles import get_active_profile_name - _profile = get_active_profile_name() - _init_kwargs["agent_identity"] = _profile - _init_kwargs["agent_workspace"] = "hermes" - except Exception: - pass - self._memory_manager.initialize_all(**_init_kwargs) - logger.info("Memory provider '%s' activated", _mem_provider_name) - else: - logger.debug("Memory provider '%s' not found or not available", _mem_provider_name) - self._memory_manager = None - except Exception as _mpe: - logger.warning("Memory provider plugin init failed: %s", _mpe) - self._memory_manager = None - - # Inject memory provider tool schemas into the tool surface. - # Skip tools whose names already exist (plugins may register the - # same tools via ctx.register_tool(), which lands in self.tools - # through get_tool_definitions()). Duplicate function names cause - # 400 errors on providers that enforce unique names (e.g. Xiaomi - # MiMo via Nous Portal). - if self._memory_manager and self.tools is not None: - _existing_tool_names = { - t.get("function", {}).get("name") - for t in self.tools - if isinstance(t, dict) - } - for _schema in self._memory_manager.get_all_tool_schemas(): - _tname = _schema.get("name", "") - if _tname and _tname in _existing_tool_names: - continue # already registered via plugin path - _wrapped = {"type": "function", "function": _schema} - self.tools.append(_wrapped) - if _tname: - self.valid_tool_names.add(_tname) - _existing_tool_names.add(_tname) - - # Skills config: nudge interval for skill creation reminders - self._skill_nudge_interval = 10 - try: - skills_config = _agent_cfg.get("skills", {}) - self._skill_nudge_interval = int(skills_config.get("creation_nudge_interval", 10)) - except Exception: - pass - - # Tool-use enforcement config: "auto" (default — matches hardcoded - # model list), true (always), false (never), or list of substrings. - _agent_section = _agent_cfg.get("agent", {}) - if not isinstance(_agent_section, dict): - _agent_section = {} - self._tool_use_enforcement = _agent_section.get("tool_use_enforcement", "auto") - - # App-level API retry count (wraps each model API call). Default 3, - # overridable via agent.api_max_retries in config.yaml. See #11616. - try: - _raw_api_retries = _agent_section.get("api_max_retries", 3) - _api_retries = int(_raw_api_retries) - _api_retries = max(_api_retries, 1) # 1 = no retry (single attempt) - except (TypeError, ValueError): - _api_retries = 3 - self._api_max_retries = _api_retries - - # Initialize context compressor for automatic context management - # Compresses conversation when approaching model's context limit - # Configuration via config.yaml (compression section) - _compression_cfg = _agent_cfg.get("compression", {}) - if not isinstance(_compression_cfg, dict): - _compression_cfg = {} - compression_threshold = float(_compression_cfg.get("threshold", 0.50)) - try: - from agent.auxiliary_client import _compression_threshold_for_model as _cthresh_fn - _model_cthresh = _cthresh_fn(self.model) - if _model_cthresh is not None: - compression_threshold = _model_cthresh - except Exception: - pass - compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"} - compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20)) - compression_protect_last = int(_compression_cfg.get("protect_last_n", 20)) - # protect_first_n is the number of non-system messages to protect at - # the head, in addition to the system prompt (which is always - # implicitly protected by the compressor). Floor at 0 — a value of - # 0 means "preserve only the system prompt + summary + tail", which - # is a legitimate (and common) configuration for long-running - # rolling-compaction sessions. - compression_protect_first = max( - 0, int(_compression_cfg.get("protect_first_n", 3)) - ) - - # Read optional explicit context_length override for the auxiliary - # compression model. Custom endpoints often cannot report this via - # /models, so the startup feasibility check needs the config hint. - try: - _aux_cfg = cfg_get(_agent_cfg, "auxiliary", "compression", default={}) - except Exception: - _aux_cfg = {} - if isinstance(_aux_cfg, dict): - _aux_context_config = _aux_cfg.get("context_length") - else: - _aux_context_config = None - if _aux_context_config is not None: - try: - _aux_context_config = int(_aux_context_config) - except (TypeError, ValueError): - _aux_context_config = None - self._aux_compression_context_length_config = _aux_context_config - - # Read explicit model output-token override from config when the - # caller did not pass one directly. - _model_cfg = _agent_cfg.get("model", {}) - if self.max_tokens is None and isinstance(_model_cfg, dict): - _config_max_tokens = _model_cfg.get("max_tokens") - if _config_max_tokens is not None: - try: - if isinstance(_config_max_tokens, bool): - raise ValueError - _parsed_max_tokens = int(_config_max_tokens) - if _parsed_max_tokens <= 0: - raise ValueError - self.max_tokens = _parsed_max_tokens - except (TypeError, ValueError): - logger.warning( - "Invalid model.max_tokens in config.yaml: %r — " - "must be a positive integer (e.g. 4096). " - "Falling back to provider default.", - _config_max_tokens, - ) - print( - f"\n⚠ Invalid model.max_tokens in config.yaml: {_config_max_tokens!r}\n" - f" Must be a positive integer (e.g. 4096).\n" - f" Falling back to provider default.\n", - file=sys.stderr, - ) - self._session_init_model_config["max_tokens"] = self.max_tokens - - # Read explicit context_length override from model config - if isinstance(_model_cfg, dict): - _config_context_length = _model_cfg.get("context_length") - else: - _config_context_length = None - if _config_context_length is not None: - try: - _config_context_length = int(_config_context_length) - except (TypeError, ValueError): - logger.warning( - "Invalid model.context_length in config.yaml: %r — " - "must be a plain integer (e.g. 256000, not '256K'). " - "Falling back to auto-detection.", - _config_context_length, - ) - print( - f"\n⚠ Invalid model.context_length in config.yaml: {_config_context_length!r}\n" - f" Must be a plain integer (e.g. 256000, not '256K').\n" - f" Falling back to auto-detected context window.\n", - file=sys.stderr, - ) - _config_context_length = None - - # Resolve custom_providers list once for reuse below (startup - # context-length override and plugin context-engine init). - try: - from hermes_cli.config import get_compatible_custom_providers - _custom_providers = get_compatible_custom_providers(_agent_cfg) - except Exception: - _custom_providers = _agent_cfg.get("custom_providers") - if not isinstance(_custom_providers, list): - _custom_providers = [] - - # Store for reuse by _check_compression_model_feasibility (auxiliary - # compression model context-length detection needs the same list). - self._custom_providers = _custom_providers - - # Check custom_providers per-model context_length - if _config_context_length is None and _custom_providers: - try: - from hermes_cli.config import get_custom_provider_context_length - _cp_ctx_resolved = get_custom_provider_context_length( - model=self.model, - base_url=self.base_url, - custom_providers=_custom_providers, - ) - if _cp_ctx_resolved: - _config_context_length = int(_cp_ctx_resolved) - except Exception: - _cp_ctx_resolved = None - - # Surface a clear warning if the user set a context_length but it - # wasn't a valid positive int — the helper silently skips those. - if _config_context_length is None: - _target = self.base_url.rstrip("/") if self.base_url else "" - for _cp_entry in _custom_providers: - if not isinstance(_cp_entry, dict): - continue - _cp_url = (_cp_entry.get("base_url") or "").rstrip("/") - if _target and _cp_url == _target: - _cp_models = _cp_entry.get("models", {}) - if isinstance(_cp_models, dict): - _cp_model_cfg = _cp_models.get(self.model, {}) - if isinstance(_cp_model_cfg, dict): - _cp_ctx = _cp_model_cfg.get("context_length") - if _cp_ctx is not None: - try: - _parsed = int(_cp_ctx) - if _parsed <= 0: - raise ValueError - except (TypeError, ValueError): - logger.warning( - "Invalid context_length for model %r in " - "custom_providers: %r — must be a positive " - "integer (e.g. 256000, not '256K'). " - "Falling back to auto-detection.", - self.model, _cp_ctx, - ) - print( - f"\n⚠ Invalid context_length for model {self.model!r} in custom_providers: {_cp_ctx!r}\n" - f" Must be a positive integer (e.g. 256000, not '256K').\n" - f" Falling back to auto-detected context window.\n", - file=sys.stderr, - ) - break - - # Persist for reuse on switch_model / fallback activation. Must come - # AFTER the custom_providers branch so per-model overrides aren't lost. - self._config_context_length = _config_context_length - - self._ensure_lmstudio_runtime_loaded(_config_context_length) - - - - # Select context engine: config-driven (like memory providers). - # 1. Check config.yaml context.engine setting - # 2. Check plugins/context_engine// directory (repo-shipped) - # 3. Check general plugin system (user-installed plugins) - # 4. Fall back to built-in ContextCompressor - _selected_engine = None - _engine_name = "compressor" # default - try: - _ctx_cfg = _agent_cfg.get("context", {}) if isinstance(_agent_cfg, dict) else {} - _engine_name = _ctx_cfg.get("engine", "compressor") or "compressor" - except Exception: - pass - - if _engine_name != "compressor": - # Try loading from plugins/context_engine// - try: - from plugins.context_engine import load_context_engine - _selected_engine = load_context_engine(_engine_name) - except Exception as _ce_load_err: - logger.debug("Context engine load from plugins/context_engine/: %s", _ce_load_err) - - # Try general plugin system as fallback - if _selected_engine is None: - try: - from hermes_cli.plugins import get_plugin_context_engine - _candidate = get_plugin_context_engine() - if _candidate and _candidate.name == _engine_name: - _selected_engine = _candidate - except Exception: - pass - - if _selected_engine is None: - logger.warning( - "Context engine '%s' not found — falling back to built-in compressor", - _engine_name, - ) - # else: config says "compressor" — use built-in, don't auto-activate plugins - - if _selected_engine is not None: - self.context_compressor = _selected_engine - # Resolve context_length for plugin engines — mirrors switch_model() path - from agent.model_metadata import get_model_context_length - _plugin_ctx_len = get_model_context_length( - self.model, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - config_context_length=_config_context_length, - provider=self.provider, - custom_providers=_custom_providers, - ) - self.context_compressor.update_model( - model=self.model, - context_length=_plugin_ctx_len, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - ) - if not self.quiet_mode: - logger.info("Using context engine: %s", _selected_engine.name) - else: - self.context_compressor = ContextCompressor( - model=self.model, - threshold_percent=compression_threshold, - protect_first_n=compression_protect_first, - protect_last_n=compression_protect_last, - summary_target_ratio=compression_target_ratio, - summary_model_override=None, - quiet_mode=self.quiet_mode, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - config_context_length=_config_context_length, - provider=self.provider, - api_mode=self.api_mode, - ) - self.compression_enabled = compression_enabled - - # Reject models whose context window is below the minimum required - # for reliable tool-calling workflows (64K tokens). - from agent.model_metadata import MINIMUM_CONTEXT_LENGTH - _ctx = getattr(self.context_compressor, "context_length", 0) - if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH: - raise ValueError( - f"Model {self.model} has a context window of {_ctx:,} tokens, " - f"which is below the minimum {MINIMUM_CONTEXT_LENGTH:,} required " - f"by Hermes Agent. Choose a model with at least " - f"{MINIMUM_CONTEXT_LENGTH // 1000}K context, or set " - f"model.context_length in config.yaml to override." - ) - - # Inject context engine tool schemas (e.g. lcm_grep, lcm_describe, lcm_expand). - # Skip names that are already present — the get_tool_definitions() - # quiet_mode cache returned a shared list pre-#17335, so a stray - # mutation here would poison subsequent agent inits in the same - # Gateway process and trip provider-side 'duplicate tool name' - # errors. Even with the cache fix, dedup is the right defense - # against plugin paths that may register the same schemas via - # ctx.register_tool(). Mirrors the memory tools dedup above. - self._context_engine_tool_names: set = set() - if hasattr(self, "context_compressor") and self.context_compressor and self.tools is not None: - _existing_tool_names = { - t.get("function", {}).get("name") - for t in self.tools - if isinstance(t, dict) - } - for _schema in self.context_compressor.get_tool_schemas(): - _tname = _schema.get("name", "") - if _tname and _tname in _existing_tool_names: - continue # already registered via plugin/cache path - _wrapped = {"type": "function", "function": _schema} - self.tools.append(_wrapped) - if _tname: - self.valid_tool_names.add(_tname) - self._context_engine_tool_names.add(_tname) - _existing_tool_names.add(_tname) - - # Notify context engine of session start - if hasattr(self, "context_compressor") and self.context_compressor: - try: - self.context_compressor.on_session_start( - self.session_id, - hermes_home=str(get_hermes_home()), - platform=self.platform or "cli", - model=self.model, - context_length=getattr(self.context_compressor, "context_length", 0), - ) - except Exception as _ce_err: - logger.debug("Context engine on_session_start: %s", _ce_err) - - self._subdirectory_hints = SubdirectoryHintTracker( - working_dir=os.getenv("TERMINAL_CWD") or None, - ) - self._user_turn_count = 0 - - # Cumulative token usage for the session - self.session_prompt_tokens = 0 - self.session_completion_tokens = 0 - self.session_total_tokens = 0 - self.session_api_calls = 0 - self.session_input_tokens = 0 - self.session_output_tokens = 0 - self.session_cache_read_tokens = 0 - self.session_cache_write_tokens = 0 - self.session_reasoning_tokens = 0 - self.session_estimated_cost_usd = 0.0 - self.session_cost_status = "unknown" - self.session_cost_source = "none" - - # ── Ollama num_ctx injection ── - # Ollama defaults to 2048 context regardless of the model's capabilities. - # When running against an Ollama server, detect the model's max context - # and pass num_ctx on every chat request so the full window is used. - # User override: set model.ollama_num_ctx in config.yaml to cap VRAM use. - # If model.context_length is set, it caps num_ctx so the user's VRAM - # budget is respected even when GGUF metadata advertises a larger window. - self._ollama_num_ctx: int | None = None - _ollama_num_ctx_override = None - if isinstance(_model_cfg, dict): - _ollama_num_ctx_override = _model_cfg.get("ollama_num_ctx") - if _ollama_num_ctx_override is not None: - try: - self._ollama_num_ctx = int(_ollama_num_ctx_override) - except (TypeError, ValueError): - logger.debug("Invalid ollama_num_ctx config value: %r", _ollama_num_ctx_override) - if self._ollama_num_ctx is None and self.base_url and is_local_endpoint(self.base_url): - try: - _detected = query_ollama_num_ctx(self.model, self.base_url, api_key=self.api_key or "") - if _detected and _detected > 0: - self._ollama_num_ctx = _detected - except Exception as exc: - logger.debug("Ollama num_ctx detection failed: %s", exc) - # Cap auto-detected ollama_num_ctx to the user's explicit context_length. - # Without this, GGUF metadata can advertise 256K+ which Ollama honours - # by allocating that much VRAM — blowing up small GPUs even though the - # user explicitly set a smaller context_length in config.yaml. - if ( - self._ollama_num_ctx - and _config_context_length - and _ollama_num_ctx_override is None # don't override explicit ollama_num_ctx - and self._ollama_num_ctx > _config_context_length - ): - logger.info( - "Ollama num_ctx capped: %d -> %d (model.context_length override)", - self._ollama_num_ctx, _config_context_length, - ) - self._ollama_num_ctx = _config_context_length - if self._ollama_num_ctx and not self.quiet_mode: - logger.info( - "Ollama num_ctx: will request %d tokens (model max from /api/show)", - self._ollama_num_ctx, - ) - - if not self.quiet_mode: - if compression_enabled: - print(f"📊 Context limit: {self.context_compressor.context_length:,} tokens (compress at {int(compression_threshold*100)}% = {self.context_compressor.threshold_tokens:,})") - else: - print(f"📊 Context limit: {self.context_compressor.context_length:,} tokens (auto-compression disabled)") - - # Check immediately so CLI users see the warning at startup. - # Gateway status_callback is not yet wired, so any warning is stored - # in _compression_warning and replayed in the first run_conversation(). - self._compression_warning = None - self._check_compression_model_feasibility() - - # Snapshot primary runtime for per-turn restoration. When fallback - # activates during a turn, the next turn restores these values so the - # preferred model gets a fresh attempt each time. Uses a single dict - # so new state fields are easy to add without N individual attributes. - _cc = self.context_compressor - self._primary_runtime = { - "model": self.model, - "provider": self.provider, - "base_url": self.base_url, - "api_mode": self.api_mode, - "api_key": getattr(self, "api_key", ""), - "client_kwargs": dict(self._client_kwargs), - "use_prompt_caching": self._use_prompt_caching, - "use_native_cache_layout": self._use_native_cache_layout, - # Context engine state that _try_activate_fallback() overwrites. - # Use getattr for model/base_url/api_key/provider since plugin - # engines may not have these (they're ContextCompressor-specific). - "compressor_model": getattr(_cc, "model", self.model), - "compressor_base_url": getattr(_cc, "base_url", self.base_url), - "compressor_api_key": getattr(_cc, "api_key", ""), - "compressor_provider": getattr(_cc, "provider", self.provider), - "compressor_context_length": _cc.context_length, - "compressor_threshold_tokens": _cc.threshold_tokens, - } - if self.api_mode == "anthropic_messages": - self._primary_runtime.update({ - "anthropic_api_key": self._anthropic_api_key, - "anthropic_base_url": self._anthropic_base_url, - "is_anthropic_oauth": self._is_anthropic_oauth, - }) + """Forwarder — see ``agent.agent_init.init_agent``.""" + from agent.agent_init import init_agent + init_agent(self, base_url, api_key, provider, api_mode, acp_command, acp_args, command, args, model, max_iterations, tool_delay, enabled_toolsets, disabled_toolsets, save_trajectories, verbose_logging, quiet_mode, ephemeral_system_prompt, log_prefix_chars, log_prefix, providers_allowed, providers_ignored, providers_order, provider_sort, provider_require_parameters, provider_data_collection, openrouter_min_coding_score, session_id, tool_progress_callback, tool_start_callback, tool_complete_callback, thinking_callback, reasoning_callback, clarify_callback, step_callback, stream_delta_callback, interim_assistant_callback, tool_gen_callback, status_callback, max_tokens, reasoning_config, service_tier, request_overrides, prefill_messages, platform, user_id, user_name, chat_id, chat_name, chat_type, thread_id, gateway_session_key, skip_context_files, load_soul_identity, skip_memory, session_db, parent_session_id, iteration_budget, fallback_model, credential_pool, checkpoints_enabled, checkpoint_max_snapshots, checkpoint_max_total_size_mb, checkpoint_max_file_size_mb, pass_session_id) def _get_session_db_for_recall(self): """Return a SessionDB for recall, lazily creating it if an entrypoint forgot. From 94c3e0ab8ef253eab64c3632b0e9cc8a502f16b9 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 20:35:19 -0700 Subject: [PATCH 015/142] refactor(run_agent): extract 10 more helpers to agent/agent_runtime_helpers.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final extraction pass — the methods left over after run_conversation and __init__ moved out. Together these 10 cover ~813 LOC of medium- sized helpers: * switch_model (194 LOC) — model switching mid-session * _invoke_tool (87) — central tool dispatch with overrides * _repair_tool_call (72) — argument JSON repair entrypoint * _sanitize_api_messages (71) — role-filter for API send * _looks_like_codex_intermediate_ack (72) — codex transcript heuristic * _copy_reasoning_content_for_api (70) — reasoning preservation * _cleanup_dead_connections (70) — periodic dead-socket sweep * _extract_api_error_context (65) — error-dump context builder * _apply_pending_steer_to_tool_results (63) — /steer injection * _force_close_tcp_sockets (59) — aggressive socket cleanup AIAgent keeps thin forwarder methods for all 10 (staticmethods preserved where present). Names tests patch on run_agent (handle_function_call, AIAgent class attrs, logger) routed through _ra() so the patch surface is preserved. tests/run_agent/ + tests/agent/: 4313 passed (same pre-existing test_auxiliary_client failure as on main). run_agent.py: 4634 -> 3821 lines (-813). Final total: 16083 -> 3821 (-12262, 76% reduction). --- agent/agent_runtime_helpers.py | 871 ++++++++++++++++++++++++++++++++- run_agent.py | 823 ++----------------------------- 2 files changed, 890 insertions(+), 804 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 4efe5203421..797047f95d3 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -573,7 +573,7 @@ def recover_with_credential_pool( rotate_status = status_code if status_code is not None else 402 next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) if next_entry is not None: - logger.info( + _ra().logger.info( "Credential %s (billing) — rotated to pool entry %s", rotate_status, getattr(next_entry, "id", "?"), @@ -588,7 +588,7 @@ def recover_with_credential_pool( rotate_status = status_code if status_code is not None else 429 next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) if next_entry is not None: - logger.info( + _ra().logger.info( "Credential %s (rate limit) — rotated to pool entry %s", rotate_status, getattr(next_entry, "id", "?"), @@ -600,7 +600,7 @@ def recover_with_credential_pool( if effective_reason == FailoverReason.auth: refreshed = pool.try_refresh_current() if refreshed is not None: - logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") + _ra().logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") agent._swap_credential(refreshed) return True, has_retried_429 # Refresh failed — rotate to next credential instead of giving up. @@ -608,7 +608,7 @@ def recover_with_credential_pool( rotate_status = status_code if status_code is not None else 401 next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) if next_entry is not None: - logger.info( + _ra().logger.info( "Credential %s (auth refresh failed) — rotated to pool entry %s", rotate_status, getattr(next_entry, "id", "?"), @@ -780,7 +780,7 @@ def drop_thinking_only_and_merge_users( else: merged.append(m) - logger.debug( + _ra().logger.debug( "Pre-call sanitizer: dropped %d thinking-only assistant turn(s), " "merged %d adjacent user message(s)", dropped, @@ -982,7 +982,7 @@ def dump_api_request_debug( try: api_key = getattr(agent.client, "api_key", None) except Exception as e: - logger.debug("Could not extract API key for debug dump: %s", e) + _ra().logger.debug("Could not extract API key for debug dump: %s", e) dump_payload: Dict[str, Any] = { "timestamp": datetime.now().isoformat(), @@ -1019,7 +1019,7 @@ def dump_api_request_debug( error_info["response_status"] = getattr(response_obj, "status_code", None) error_info["response_text"] = response_obj.text except Exception as e: - logger.debug("Could not extract error response details: %s", e) + _ra().logger.debug("Could not extract error response details: %s", e) dump_payload["error"] = error_info @@ -1166,7 +1166,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo from agent.copilot_acp_client import CopilotACPClient client = CopilotACPClient(**client_kwargs) - logger.info( + _ra().logger.info( "Copilot ACP client created (%s, shared=%s) %s", reason, shared, @@ -1182,7 +1182,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo if k in {"api_key", "base_url", "default_headers", "project_id", "timeout"} } client = GeminiCloudCodeClient(**safe_kwargs) - logger.info( + _ra().logger.info( "Gemini Cloud Code Assist client created (%s, shared=%s) %s", reason, shared, @@ -1203,7 +1203,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo if keepalive_http is not None: safe_kwargs["http_client"] = keepalive_http client = GeminiNativeClient(**safe_kwargs) - logger.info( + _ra().logger.info( "Gemini native client created (%s, shared=%s) %s", reason, shared, @@ -1234,7 +1234,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo # Uses the module-level `OpenAI` name, resolved lazily on first # access via __getattr__ below. Tests patch via `run_agent.OpenAI`. client = _ra().OpenAI(**client_kwargs) - logger.info( + _ra().logger.info( "OpenAI client created (%s, shared=%s) %s", reason, shared, @@ -1243,6 +1243,845 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo return client +def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mode=''): + """Switch the model/provider in-place for a live agent. + + Called by the /model command handlers (CLI and gateway) after + ``model_switch.switch_model()`` has resolved credentials and + validated the model. This method performs the actual runtime + swap: rebuilding clients, updating caching flags, and refreshing + the context compressor. + + The implementation mirrors ``_try_activate_fallback()`` for the + client-swap logic but also updates ``_primary_runtime`` so the + change persists across turns (unlike fallback which is + turn-scoped). + """ + from hermes_cli.providers import determine_api_mode + + # ── Determine api_mode if not provided ── + if not api_mode: + api_mode = determine_api_mode(new_provider, base_url) + + # Defense-in-depth: ensure OpenCode base_url doesn't carry a trailing + # /v1 into the anthropic_messages client, which would cause the SDK to + # hit /v1/v1/messages. `model_switch.switch_model()` already strips + # this, but we guard here so any direct callers (future code paths, + # tests) can't reintroduce the double-/v1 404 bug. + if ( + api_mode == "anthropic_messages" + and new_provider in {"opencode-zen", "opencode-go"} + and isinstance(base_url, str) + and base_url + ): + base_url = re.sub(r"/v1/?$", "", base_url) + + old_model = agent.model + old_provider = agent.provider + + # Clear the per-config context_length override so the new model's + # actual context window is resolved via get_model_context_length() + # instead of inheriting the stale value from the previous model. + agent._config_context_length = None + + # ── Swap core runtime fields ── + agent.model = new_model + agent.provider = new_provider + # Use new base_url when provided; only fall back to current when the + # new provider genuinely has no endpoint (e.g. native SDK providers). + # Without this guard the old provider's URL (e.g. Ollama's localhost + # address) would persist silently after switching to a cloud provider + # that returns an empty base_url string. + if base_url: + agent.base_url = base_url + agent.api_mode = api_mode + # Invalidate transport cache — new api_mode may need a different transport + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + if api_key: + agent.api_key = api_key + + # ── Build new client ── + if api_mode == "anthropic_messages": + from agent.anthropic_adapter import ( + build_anthropic_client, + resolve_anthropic_token, + _is_oauth_token, + ) + # Only fall back to ANTHROPIC_TOKEN when the provider is actually Anthropic. + # Other anthropic_messages providers (MiniMax, Alibaba, etc.) must use their own + # API key — falling back would send Anthropic credentials to third-party endpoints. + _is_native_anthropic = new_provider == "anthropic" + effective_key = (api_key or agent.api_key or resolve_anthropic_token() or "") if _is_native_anthropic else (api_key or agent.api_key or "") + agent.api_key = effective_key + agent._anthropic_api_key = effective_key + agent._anthropic_base_url = base_url or getattr(agent, "_anthropic_base_url", None) + agent._anthropic_client = build_anthropic_client( + effective_key, agent._anthropic_base_url, + timeout=get_provider_request_timeout(agent.provider, agent.model), + ) + agent._is_anthropic_oauth = _is_oauth_token(effective_key) if _is_native_anthropic else False + agent.client = None + agent._client_kwargs = {} + else: + effective_key = api_key or agent.api_key + effective_base = base_url or agent.base_url + agent._client_kwargs = { + "api_key": effective_key, + "base_url": effective_base, + } + _sm_timeout = get_provider_request_timeout(agent.provider, agent.model) + if _sm_timeout is not None: + agent._client_kwargs["timeout"] = _sm_timeout + agent.client = agent._create_openai_client( + dict(agent._client_kwargs), + reason="switch_model", + shared=True, + ) + + # ── Re-evaluate prompt caching ── + agent._use_prompt_caching, agent._use_native_cache_layout = ( + agent._anthropic_prompt_cache_policy( + provider=new_provider, + base_url=agent.base_url, + api_mode=api_mode, + model=new_model, + ) + ) + + # ── LM Studio: preload before probing context length ── + agent._ensure_lmstudio_runtime_loaded() + + # ── Update context compressor ── + if hasattr(agent, "context_compressor") and agent.context_compressor: + from agent.model_metadata import get_model_context_length + # Re-read custom_providers from live config so per-model + # context_length overrides are honored when switching to a + # custom provider mid-session (closes #15779). + _sm_custom_providers = None + try: + from hermes_cli.config import load_config, get_compatible_custom_providers + _sm_cfg = load_config() + _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) + except Exception: + _sm_custom_providers = None + new_context_length = get_model_context_length( + agent.model, + base_url=agent.base_url, + api_key=agent.api_key, + provider=agent.provider, + config_context_length=getattr(agent, "_config_context_length", None), + custom_providers=_sm_custom_providers, + ) + agent.context_compressor.update_model( + model=agent.model, + context_length=new_context_length, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + api_mode=agent.api_mode, + ) + + # ── Invalidate cached system prompt so it rebuilds next turn ── + agent._cached_system_prompt = None + + # ── Update _primary_runtime so the change persists across turns ── + _cc = agent.context_compressor if hasattr(agent, "context_compressor") and agent.context_compressor else None + agent._primary_runtime = { + "model": agent.model, + "provider": agent.provider, + "base_url": agent.base_url, + "api_mode": agent.api_mode, + "api_key": getattr(agent, "api_key", ""), + "client_kwargs": dict(agent._client_kwargs), + "use_prompt_caching": agent._use_prompt_caching, + "use_native_cache_layout": agent._use_native_cache_layout, + "compressor_model": getattr(_cc, "model", agent.model) if _cc else agent.model, + "compressor_base_url": getattr(_cc, "base_url", agent.base_url) if _cc else agent.base_url, + "compressor_api_key": getattr(_cc, "api_key", "") if _cc else "", + "compressor_provider": getattr(_cc, "provider", agent.provider) if _cc else agent.provider, + "compressor_context_length": _cc.context_length if _cc else 0, + "compressor_threshold_tokens": _cc.threshold_tokens if _cc else 0, + } + if api_mode == "anthropic_messages": + agent._primary_runtime.update({ + "anthropic_api_key": agent._anthropic_api_key, + "anthropic_base_url": agent._anthropic_base_url, + "is_anthropic_oauth": agent._is_anthropic_oauth, + }) + + # ── Reset fallback state ── + agent._fallback_activated = False + agent._fallback_index = 0 + + # When the user deliberately swaps primary providers (e.g. openrouter + # → anthropic), drop any fallback entries that target the OLD primary + # or the NEW one. The chain was seeded from config at agent init for + # the original provider — without pruning, a failed turn on the new + # primary silently re-activates the provider the user just rejected, + # which is exactly what was reported during TUI v2 blitz testing + # ("switched to anthropic, tui keeps trying openrouter"). + old_norm = (old_provider or "").strip().lower() + new_norm = (new_provider or "").strip().lower() + fallback_chain = list(getattr(agent, "_fallback_chain", []) or []) + if old_norm and new_norm and old_norm != new_norm: + fallback_chain = [ + entry for entry in fallback_chain + if (entry.get("provider") or "").strip().lower() not in {old_norm, new_norm} + ] + agent._fallback_chain = fallback_chain + agent._fallback_model = fallback_chain[0] if fallback_chain else None + + logging.info( + "Model switched in-place: %s (%s) -> %s (%s)", + old_model, old_provider, new_model, new_provider, + ) + + + +def invoke_tool(agent, function_name: str, function_args: dict, effective_task_id: str, + tool_call_id: Optional[str] = None, messages: list = None, + pre_tool_block_checked: bool = False) -> str: + """Invoke a single tool and return the result string. No display logic. + + Handles both agent-level tools (todo, memory, etc.) and registry-dispatched + tools. Used by the concurrent execution path; the sequential path retains + its own inline invocation for backward-compatible display handling. + """ + # Check plugin hooks for a block directive before executing anything. + block_message: Optional[str] = None + if not pre_tool_block_checked: + try: + from hermes_cli.plugins import get_pre_tool_call_block_message + block_message = get_pre_tool_call_block_message( + function_name, function_args, task_id=effective_task_id or "", + ) + except Exception: + pass + if block_message is not None: + return json.dumps({"error": block_message}, ensure_ascii=False) + + if function_name == "todo": + from tools.todo_tool import todo_tool as _todo_tool + return _todo_tool( + todos=function_args.get("todos"), + merge=function_args.get("merge", False), + store=agent._todo_store, + ) + elif function_name == "session_search": + session_db = agent._get_session_db_for_recall() + if not session_db: + from hermes_state import format_session_db_unavailable + return json.dumps({"success": False, "error": format_session_db_unavailable()}) + from tools.session_search_tool import session_search as _session_search + return _session_search( + query=function_args.get("query", ""), + role_filter=function_args.get("role_filter"), + limit=function_args.get("limit", 3), + db=session_db, + current_session_id=agent.session_id, + ) + elif function_name == "memory": + target = function_args.get("target", "memory") + from tools.memory_tool import memory_tool as _memory_tool + result = _memory_tool( + action=function_args.get("action"), + target=target, + content=function_args.get("content"), + old_text=function_args.get("old_text"), + store=agent._memory_store, + ) + # Bridge: notify external memory provider of built-in memory writes + if agent._memory_manager and function_args.get("action") in {"add", "replace"}: + try: + agent._memory_manager.on_memory_write( + function_args.get("action", ""), + target, + function_args.get("content", ""), + metadata=agent._build_memory_write_metadata( + task_id=effective_task_id, + tool_call_id=tool_call_id, + ), + ) + except Exception: + pass + return result + elif agent._memory_manager and agent._memory_manager.has_tool(function_name): + return agent._memory_manager.handle_tool_call(function_name, function_args) + elif function_name == "clarify": + from tools.clarify_tool import clarify_tool as _clarify_tool + return _clarify_tool( + question=function_args.get("question", ""), + choices=function_args.get("choices"), + callback=agent.clarify_callback, + ) + elif function_name == "delegate_task": + return agent._dispatch_delegate_task(function_args) + else: + return _ra().handle_function_call( + function_name, function_args, effective_task_id, + tool_call_id=tool_call_id, + session_id=agent.session_id or "", + enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, + skip_pre_tool_call_hook=True, + ) + + + +def repair_tool_call(agent, tool_name: str) -> str | None: + """Attempt to repair a mismatched tool name before aborting. + + Models sometimes emit variants of a tool name that differ only + in casing, separators, or class-like suffixes. Normalize + aggressively before falling back to fuzzy match: + + 1. Lowercase direct match. + 2. Lowercase + hyphens/spaces -> underscores. + 3. CamelCase -> snake_case (TodoTool -> todo_tool). + 4. Strip trailing ``_tool`` / ``-tool`` / ``tool`` suffix that + Claude-style models sometimes tack on (TodoTool_tool -> + TodoTool -> Todo -> todo). Applied twice so double-tacked + suffixes like ``TodoTool_tool`` reduce all the way. + 5. Fuzzy match (difflib, cutoff=0.7). + + See #14784 for the original reports (TodoTool_tool, Patch_tool, + BrowserClick_tool were all returning "Unknown tool" before). + + Returns the repaired name if found in valid_tool_names, else None. + """ + import re + from difflib import get_close_matches + + if not tool_name: + return None + + def _norm(s: str) -> str: + return s.lower().replace("-", "_").replace(" ", "_") + + def _camel_snake(s: str) -> str: + return re.sub(r"(? str | None: + lc = s.lower() + for suffix in ("_tool", "-tool", "tool"): + if lc.endswith(suffix): + return s[: -len(suffix)].rstrip("_-") + return None + + # Cheap fast-paths first — these cover the common case. + lowered = tool_name.lower() + if lowered in agent.valid_tool_names: + return lowered + normalized = _norm(tool_name) + if normalized in agent.valid_tool_names: + return normalized + + # Build the full candidate set for class-like emissions. + cands: set[str] = {tool_name, lowered, normalized, _camel_snake(tool_name)} + # Strip trailing tool-suffix up to twice — TodoTool_tool needs it. + for _ in range(2): + extra: set[str] = set() + for c in cands: + stripped = _strip_tool_suffix(c) + if stripped: + extra.add(stripped) + extra.add(_norm(stripped)) + extra.add(_camel_snake(stripped)) + cands |= extra + + for c in cands: + if c and c in agent.valid_tool_names: + return c + + # Fuzzy match as last resort. + matches = get_close_matches(lowered, agent.valid_tool_names, n=1, cutoff=0.7) + if matches: + return matches[0] + + return None + + + +def sanitize_api_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Fix orphaned tool_call / tool_result pairs before every LLM call. + + Runs unconditionally — not gated on whether the context compressor + is present — so orphans from session loading or manual message + manipulation are always caught. + """ + # --- Role allowlist: drop messages with roles the API won't accept --- + filtered = [] + for msg in messages: + role = msg.get("role") + if role not in _ra().AIAgent._VALID_API_ROLES: + _ra().logger.debug( + "Pre-call sanitizer: dropping message with invalid role %r", + role, + ) + continue + filtered.append(msg) + messages = filtered + + surviving_call_ids: set = set() + for msg in messages: + if msg.get("role") == "assistant": + for tc in msg.get("tool_calls") or []: + cid = _ra().AIAgent._get_tool_call_id_static(tc) + if cid: + surviving_call_ids.add(cid) + + result_call_ids: set = set() + for msg in messages: + if msg.get("role") == "tool": + cid = msg.get("tool_call_id") + if cid: + result_call_ids.add(cid) + + # 1. Drop tool results with no matching assistant call + orphaned_results = result_call_ids - surviving_call_ids + if orphaned_results: + messages = [ + m for m in messages + if not (m.get("role") == "tool" and m.get("tool_call_id") in orphaned_results) + ] + _ra().logger.debug( + "Pre-call sanitizer: removed %d orphaned tool result(s)", + len(orphaned_results), + ) + + # 2. Inject stub results for calls whose result was dropped + missing_results = surviving_call_ids - result_call_ids + if missing_results: + patched: List[Dict[str, Any]] = [] + for msg in messages: + patched.append(msg) + if msg.get("role") == "assistant": + for tc in msg.get("tool_calls") or []: + cid = _ra().AIAgent._get_tool_call_id_static(tc) + if cid in missing_results: + patched.append({ + "role": "tool", + "name": _ra().AIAgent._get_tool_call_name_static(tc), + "content": "[Result unavailable — see context summary above]", + "tool_call_id": cid, + }) + messages = patched + _ra().logger.debug( + "Pre-call sanitizer: added %d stub tool result(s)", + len(missing_results), + ) + return messages + + + +def looks_like_codex_intermediate_ack( + agent, + user_message: str, + assistant_content: str, + messages: List[Dict[str, Any]], +) -> bool: + """Detect a planning/ack message that should continue instead of ending the turn.""" + if any(isinstance(msg, dict) and msg.get("role") == "tool" for msg in messages): + return False + + assistant_text = agent._strip_think_blocks(assistant_content or "").strip().lower() + if not assistant_text: + return False + if len(assistant_text) > 1200: + return False + + has_future_ack = bool( + re.search(r"\b(i['’]ll|i will|let me|i can do that|i can help with that)\b", assistant_text) + ) + if not has_future_ack: + return False + + action_markers = ( + "look into", + "look at", + "inspect", + "scan", + "check", + "analyz", + "review", + "explore", + "read", + "open", + "run", + "test", + "fix", + "debug", + "search", + "find", + "walkthrough", + "report back", + "summarize", + ) + workspace_markers = ( + "directory", + "current directory", + "current dir", + "cwd", + "repo", + "repository", + "codebase", + "project", + "folder", + "filesystem", + "file tree", + "files", + "path", + ) + + user_text = (user_message or "").strip().lower() + user_targets_workspace = ( + any(marker in user_text for marker in workspace_markers) + or "~/" in user_text + or "/" in user_text + ) + assistant_mentions_action = any(marker in assistant_text for marker in action_markers) + assistant_targets_workspace = any( + marker in assistant_text for marker in workspace_markers + ) + return (user_targets_workspace or assistant_targets_workspace) and assistant_mentions_action + + + + +def copy_reasoning_content_for_api(agent, source_msg: dict, api_msg: dict) -> None: + """Copy provider-facing reasoning fields onto an API replay message.""" + if source_msg.get("role") != "assistant": + return + + # 1. Explicit reasoning_content already set — preserve it verbatim + # (includes DeepSeek/Kimi's own space-placeholder written at creation + # time, and any valid reasoning content from the same provider). + # + # Exception: sessions persisted BEFORE #17341 have empty-string + # placeholders pinned at creation time. DeepSeek V4 Pro rejects + # those with HTTP 400. When the active provider enforces the + # thinking-mode echo, upgrade "" → " " on replay so stale history + # doesn't 400 the user on the next turn. + existing = source_msg.get("reasoning_content") + if isinstance(existing, str): + if existing == "" and agent._needs_thinking_reasoning_pad(): + api_msg["reasoning_content"] = " " + else: + api_msg["reasoning_content"] = existing + return + + needs_thinking_pad = agent._needs_thinking_reasoning_pad() + + # 2. Cross-provider poisoned history (#15748): on DeepSeek/Kimi, + # if the source turn has tool_calls AND a 'reasoning' field but no + # 'reasoning_content' key, the 'reasoning' text was written by a + # prior provider (e.g. MiniMax) — DeepSeek's own _build_assistant_message + # pins reasoning_content at creation time for tool-call turns, so the + # shape (reasoning set, reasoning_content absent, tool_calls present) + # is unreachable from same-provider DeepSeek history after this fix. + # Inject a single space to satisfy the API without leaking another + # provider's chain of thought to DeepSeek/Kimi. Space (not "") + # because DeepSeek V4 Pro rejects empty-string reasoning_content + # in thinking mode (refs #17341). + normalized_reasoning = source_msg.get("reasoning") + if ( + needs_thinking_pad + and source_msg.get("tool_calls") + and isinstance(normalized_reasoning, str) + and normalized_reasoning + ): + api_msg["reasoning_content"] = " " + return + + # 3. Healthy session: promote 'reasoning' field to 'reasoning_content' + # for providers that use the internal 'reasoning' key. + # This must happen before the unconditional empty-string fallback so + # genuine reasoning content is not overwritten (#15812 regression in + # PR #15478). + if isinstance(normalized_reasoning, str) and normalized_reasoning: + api_msg["reasoning_content"] = normalized_reasoning + return + + # 4. DeepSeek / Kimi thinking mode: all assistant messages need + # reasoning_content. Inject a single space to satisfy the provider's + # requirement when no explicit reasoning content is present. Covers + # both tool-call turns (already-poisoned history with no reasoning + # at all) and plain text turns. Space (not "") because DeepSeek V4 + # Pro tightened validation and rejects empty string with HTTP 400 + # ("The reasoning content in the thinking mode must be passed back + # to the API"). Refs #17341. + if needs_thinking_pad: + api_msg["reasoning_content"] = " " + return + + # 5. reasoning_content was present but not a string (e.g. None after + # context compaction). Don't pass null to the API. + api_msg.pop("reasoning_content", None) + + + +def cleanup_dead_connections(agent) -> bool: + """Detect and clean up dead TCP connections on the primary client. + + Inspects the httpx connection pool for sockets in unhealthy states + (CLOSE-WAIT, errors). If any are found, force-closes all sockets + and rebuilds the primary client from scratch. + + Returns True if dead connections were found and cleaned up. + """ + client = getattr(agent, "client", None) + if client is None: + return False + try: + http_client = getattr(client, "_client", None) + if http_client is None: + return False + transport = getattr(http_client, "_transport", None) + if transport is None: + return False + pool = getattr(transport, "_pool", None) + if pool is None: + return False + connections = ( + getattr(pool, "_connections", None) + or getattr(pool, "_pool", None) + or [] + ) + dead_count = 0 + for conn in list(connections): + # Check for connections that are idle but have closed sockets + stream = ( + getattr(conn, "_network_stream", None) + or getattr(conn, "_stream", None) + ) + if stream is None: + continue + sock = getattr(stream, "_sock", None) + if sock is None: + sock = getattr(stream, "stream", None) + if sock is not None: + sock = getattr(sock, "_sock", None) + if sock is None: + continue + # Probe socket health with a non-blocking recv peek + import socket as _socket + try: + sock.setblocking(False) + data = sock.recv(1, _socket.MSG_PEEK | _socket.MSG_DONTWAIT) + if data == b"": + dead_count += 1 + except BlockingIOError: + pass # No data available — socket is healthy + except OSError: + dead_count += 1 + finally: + try: + sock.setblocking(True) + except OSError: + pass + if dead_count > 0: + _ra().logger.warning( + "Found %d dead connection(s) in client pool — rebuilding client", + dead_count, + ) + agent._replace_primary_openai_client(reason="dead_connection_cleanup") + return True + except Exception as exc: + _ra().logger.debug("Dead connection check error: %s", exc) + return False + + + +def extract_api_error_context(error: Exception) -> Dict[str, Any]: + """Extract structured rate-limit details from provider errors.""" + context: Dict[str, Any] = {} + + body = getattr(error, "body", None) + payload = None + if isinstance(body, dict): + payload = body.get("error") if isinstance(body.get("error"), dict) else body + if isinstance(payload, dict): + reason = payload.get("code") or payload.get("error") + if isinstance(reason, str) and reason.strip(): + context["reason"] = reason.strip() + message = payload.get("message") or payload.get("error_description") + if isinstance(message, str) and message.strip(): + context["message"] = message.strip() + for key in ("resets_at", "reset_at"): + value = payload.get(key) + if value not in {None, ""}: + context["reset_at"] = value + break + retry_after = payload.get("retry_after") + if retry_after not in {None, ""} and "reset_at" not in context: + try: + context["reset_at"] = time.time() + float(retry_after) + except (TypeError, ValueError): + pass + + response = getattr(error, "response", None) + headers = getattr(response, "headers", None) + if headers: + retry_after = headers.get("retry-after") or headers.get("Retry-After") + if retry_after and "reset_at" not in context: + try: + context["reset_at"] = time.time() + float(retry_after) + except (TypeError, ValueError): + pass + ratelimit_reset = headers.get("x-ratelimit-reset") + if ratelimit_reset and "reset_at" not in context: + context["reset_at"] = ratelimit_reset + + if "message" not in context: + raw_message = str(error).strip() + if raw_message: + context["message"] = raw_message[:500] + + if "reset_at" not in context: + message = context.get("message") or "" + if isinstance(message, str): + delay_match = re.search(r"quotaResetDelay[:\s\"]+(\\d+(?:\\.\\d+)?)(ms|s)", message, re.IGNORECASE) + if delay_match: + value = float(delay_match.group(1)) + seconds = value / 1000.0 if delay_match.group(2).lower() == "ms" else value + context["reset_at"] = time.time() + seconds + else: + sec_match = re.search( + r"retry\s+(?:after\s+)?(\d+(?:\.\d+)?)\s*(?:sec|secs|seconds|s\b)", + message, + re.IGNORECASE, + ) + if sec_match: + context["reset_at"] = time.time() + float(sec_match.group(1)) + + return context + + + +def apply_pending_steer_to_tool_results(agent, messages: list, num_tool_msgs: int) -> None: + """Append any pending /steer text to the last tool result in this turn. + + Called at the end of a tool-call batch, before the next API call. + The steer is appended to the last ``role:"tool"`` message's content + with a clear marker so the model understands it came from the user + and NOT from the tool itself. Role alternation is preserved — + nothing new is inserted, we only modify existing content. + + Args: + messages: The running messages list. + num_tool_msgs: Number of tool results appended in this batch; + used to locate the tail slice safely. + """ + if num_tool_msgs <= 0 or not messages: + return + steer_text = agent._drain_pending_steer() + if not steer_text: + return + # Find the last tool-role message in the recent tail. Skipping + # non-tool messages defends against future code appending + # something else at the boundary. + target_idx = None + for j in range(len(messages) - 1, max(len(messages) - num_tool_msgs - 1, -1), -1): + msg = messages[j] + if isinstance(msg, dict) and msg.get("role") == "tool": + target_idx = j + break + if target_idx is None: + # No tool result in this batch (e.g. all skipped by interrupt); + # put the steer back so the caller's fallback path can deliver + # it as a normal next-turn user message. + _lock = getattr(agent, "_pending_steer_lock", None) + if _lock is not None: + with _lock: + if agent._pending_steer: + agent._pending_steer = agent._pending_steer + "\n" + steer_text + else: + agent._pending_steer = steer_text + else: + existing = getattr(agent, "_pending_steer", None) + agent._pending_steer = (existing + "\n" + steer_text) if existing else steer_text + return + marker = f"\n\nUser guidance: {steer_text}" + existing_content = messages[target_idx].get("content", "") + if not isinstance(existing_content, str): + # Anthropic multimodal content blocks — preserve them and append + # a text block at the end. + try: + blocks = list(existing_content) if existing_content else [] + blocks.append({"type": "text", "text": marker.lstrip()}) + messages[target_idx]["content"] = blocks + except Exception: + # Fall back to string replacement if content shape is unexpected. + messages[target_idx]["content"] = f"{existing_content}{marker}" + else: + messages[target_idx]["content"] = existing_content + marker + _ra().logger.info( + "Delivered /steer to agent after tool batch (%d chars): %s", + len(steer_text), + steer_text[:120] + ("..." if len(steer_text) > 120 else ""), + ) + + + +def force_close_tcp_sockets(client: Any) -> int: + """Force-close underlying TCP sockets to prevent CLOSE-WAIT accumulation. + + When a provider drops a connection mid-stream, httpx's ``client.close()`` + performs a graceful shutdown which leaves sockets in CLOSE-WAIT until the + OS times them out (often minutes). This method walks the httpx transport + pool and issues ``socket.shutdown(SHUT_RDWR)`` + ``socket.close()`` to + force an immediate TCP RST, freeing the file descriptors. + + Returns the number of sockets force-closed. + """ + import socket as _socket + + closed = 0 + try: + http_client = getattr(client, "_client", None) + if http_client is None: + return 0 + transport = getattr(http_client, "_transport", None) + if transport is None: + return 0 + pool = getattr(transport, "_pool", None) + if pool is None: + return 0 + # httpx uses httpcore connection pools; connections live in + # _connections (list) or _pool (list) depending on version. + connections = ( + getattr(pool, "_connections", None) + or getattr(pool, "_pool", None) + or [] + ) + for conn in list(connections): + stream = ( + getattr(conn, "_network_stream", None) + or getattr(conn, "_stream", None) + ) + if stream is None: + continue + sock = getattr(stream, "_sock", None) + if sock is None: + sock = getattr(stream, "stream", None) + if sock is not None: + sock = getattr(sock, "_sock", None) + if sock is None: + continue + try: + sock.shutdown(_socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + closed += 1 + except Exception as exc: + _ra().logger.debug("Force-close TCP sockets sweep error: %s", exc) + return closed + + __all__ = [ "convert_to_trajectory_format", @@ -1257,4 +2096,14 @@ __all__ = [ "dump_api_request_debug", "anthropic_prompt_cache_policy", "create_openai_client", + "switch_model", + "invoke_tool", + "repair_tool_call", + "sanitize_api_messages", + "looks_like_codex_intermediate_ack", + "copy_reasoning_content_for_api", + "cleanup_dead_connections", + "extract_api_error_context", + "apply_pending_steer_to_tool_results", + "force_close_tcp_sockets", ] diff --git a/run_agent.py b/run_agent.py index 05d648f94e2..af1483e0e32 100644 --- a/run_agent.py +++ b/run_agent.py @@ -492,198 +492,9 @@ class AIAgent: logger.debug("LM Studio preload skipped: %s", err) def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mode=''): - """Switch the model/provider in-place for a live agent. - - Called by the /model command handlers (CLI and gateway) after - ``model_switch.switch_model()`` has resolved credentials and - validated the model. This method performs the actual runtime - swap: rebuilding clients, updating caching flags, and refreshing - the context compressor. - - The implementation mirrors ``_try_activate_fallback()`` for the - client-swap logic but also updates ``_primary_runtime`` so the - change persists across turns (unlike fallback which is - turn-scoped). - """ - from hermes_cli.providers import determine_api_mode - - # ── Determine api_mode if not provided ── - if not api_mode: - api_mode = determine_api_mode(new_provider, base_url) - - # Defense-in-depth: ensure OpenCode base_url doesn't carry a trailing - # /v1 into the anthropic_messages client, which would cause the SDK to - # hit /v1/v1/messages. `model_switch.switch_model()` already strips - # this, but we guard here so any direct callers (future code paths, - # tests) can't reintroduce the double-/v1 404 bug. - if ( - api_mode == "anthropic_messages" - and new_provider in {"opencode-zen", "opencode-go"} - and isinstance(base_url, str) - and base_url - ): - base_url = re.sub(r"/v1/?$", "", base_url) - - old_model = self.model - old_provider = self.provider - - # Clear the per-config context_length override so the new model's - # actual context window is resolved via get_model_context_length() - # instead of inheriting the stale value from the previous model. - self._config_context_length = None - - # ── Swap core runtime fields ── - self.model = new_model - self.provider = new_provider - # Use new base_url when provided; only fall back to current when the - # new provider genuinely has no endpoint (e.g. native SDK providers). - # Without this guard the old provider's URL (e.g. Ollama's localhost - # address) would persist silently after switching to a cloud provider - # that returns an empty base_url string. - if base_url: - self.base_url = base_url - self.api_mode = api_mode - # Invalidate transport cache — new api_mode may need a different transport - if hasattr(self, "_transport_cache"): - self._transport_cache.clear() - if api_key: - self.api_key = api_key - - # ── Build new client ── - if api_mode == "anthropic_messages": - from agent.anthropic_adapter import ( - build_anthropic_client, - resolve_anthropic_token, - _is_oauth_token, - ) - # Only fall back to ANTHROPIC_TOKEN when the provider is actually Anthropic. - # Other anthropic_messages providers (MiniMax, Alibaba, etc.) must use their own - # API key — falling back would send Anthropic credentials to third-party endpoints. - _is_native_anthropic = new_provider == "anthropic" - effective_key = (api_key or self.api_key or resolve_anthropic_token() or "") if _is_native_anthropic else (api_key or self.api_key or "") - self.api_key = effective_key - self._anthropic_api_key = effective_key - self._anthropic_base_url = base_url or getattr(self, "_anthropic_base_url", None) - self._anthropic_client = build_anthropic_client( - effective_key, self._anthropic_base_url, - timeout=get_provider_request_timeout(self.provider, self.model), - ) - self._is_anthropic_oauth = _is_oauth_token(effective_key) if _is_native_anthropic else False - self.client = None - self._client_kwargs = {} - else: - effective_key = api_key or self.api_key - effective_base = base_url or self.base_url - self._client_kwargs = { - "api_key": effective_key, - "base_url": effective_base, - } - _sm_timeout = get_provider_request_timeout(self.provider, self.model) - if _sm_timeout is not None: - self._client_kwargs["timeout"] = _sm_timeout - self.client = self._create_openai_client( - dict(self._client_kwargs), - reason="switch_model", - shared=True, - ) - - # ── Re-evaluate prompt caching ── - self._use_prompt_caching, self._use_native_cache_layout = ( - self._anthropic_prompt_cache_policy( - provider=new_provider, - base_url=self.base_url, - api_mode=api_mode, - model=new_model, - ) - ) - - # ── LM Studio: preload before probing context length ── - self._ensure_lmstudio_runtime_loaded() - - # ── Update context compressor ── - if hasattr(self, "context_compressor") and self.context_compressor: - from agent.model_metadata import get_model_context_length - # Re-read custom_providers from live config so per-model - # context_length overrides are honored when switching to a - # custom provider mid-session (closes #15779). - _sm_custom_providers = None - try: - from hermes_cli.config import load_config, get_compatible_custom_providers - _sm_cfg = load_config() - _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) - except Exception: - _sm_custom_providers = None - new_context_length = get_model_context_length( - self.model, - base_url=self.base_url, - api_key=self.api_key, - provider=self.provider, - config_context_length=getattr(self, "_config_context_length", None), - custom_providers=_sm_custom_providers, - ) - self.context_compressor.update_model( - model=self.model, - context_length=new_context_length, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - api_mode=self.api_mode, - ) - - # ── Invalidate cached system prompt so it rebuilds next turn ── - self._cached_system_prompt = None - - # ── Update _primary_runtime so the change persists across turns ── - _cc = self.context_compressor if hasattr(self, "context_compressor") and self.context_compressor else None - self._primary_runtime = { - "model": self.model, - "provider": self.provider, - "base_url": self.base_url, - "api_mode": self.api_mode, - "api_key": getattr(self, "api_key", ""), - "client_kwargs": dict(self._client_kwargs), - "use_prompt_caching": self._use_prompt_caching, - "use_native_cache_layout": self._use_native_cache_layout, - "compressor_model": getattr(_cc, "model", self.model) if _cc else self.model, - "compressor_base_url": getattr(_cc, "base_url", self.base_url) if _cc else self.base_url, - "compressor_api_key": getattr(_cc, "api_key", "") if _cc else "", - "compressor_provider": getattr(_cc, "provider", self.provider) if _cc else self.provider, - "compressor_context_length": _cc.context_length if _cc else 0, - "compressor_threshold_tokens": _cc.threshold_tokens if _cc else 0, - } - if api_mode == "anthropic_messages": - self._primary_runtime.update({ - "anthropic_api_key": self._anthropic_api_key, - "anthropic_base_url": self._anthropic_base_url, - "is_anthropic_oauth": self._is_anthropic_oauth, - }) - - # ── Reset fallback state ── - self._fallback_activated = False - self._fallback_index = 0 - - # When the user deliberately swaps primary providers (e.g. openrouter - # → anthropic), drop any fallback entries that target the OLD primary - # or the NEW one. The chain was seeded from config at agent init for - # the original provider — without pruning, a failed turn on the new - # primary silently re-activates the provider the user just rejected, - # which is exactly what was reported during TUI v2 blitz testing - # ("switched to anthropic, tui keeps trying openrouter"). - old_norm = (old_provider or "").strip().lower() - new_norm = (new_provider or "").strip().lower() - fallback_chain = list(getattr(self, "_fallback_chain", []) or []) - if old_norm and new_norm and old_norm != new_norm: - fallback_chain = [ - entry for entry in fallback_chain - if (entry.get("provider") or "").strip().lower() not in {old_norm, new_norm} - ] - self._fallback_chain = fallback_chain - self._fallback_model = fallback_chain[0] if fallback_chain else None - - logging.info( - "Model switched in-place: %s (%s) -> %s (%s)", - old_model, old_provider, new_model, new_provider, - ) + """Forwarder — see ``agent.agent_runtime_helpers.switch_model``.""" + from agent.agent_runtime_helpers import switch_model + return switch_model(self, new_model, new_provider, api_key, base_url, api_mode) def _safe_print(self, *args, **kwargs): """Print that silently handles broken pipes / closed stdout. @@ -1134,71 +945,9 @@ class AIAgent: assistant_content: str, messages: List[Dict[str, Any]], ) -> bool: - """Detect a planning/ack message that should continue instead of ending the turn.""" - if any(isinstance(msg, dict) and msg.get("role") == "tool" for msg in messages): - return False - - assistant_text = self._strip_think_blocks(assistant_content or "").strip().lower() - if not assistant_text: - return False - if len(assistant_text) > 1200: - return False - - has_future_ack = bool( - re.search(r"\b(i['’]ll|i will|let me|i can do that|i can help with that)\b", assistant_text) - ) - if not has_future_ack: - return False - - action_markers = ( - "look into", - "look at", - "inspect", - "scan", - "check", - "analyz", - "review", - "explore", - "read", - "open", - "run", - "test", - "fix", - "debug", - "search", - "find", - "walkthrough", - "report back", - "summarize", - ) - workspace_markers = ( - "directory", - "current directory", - "current dir", - "cwd", - "repo", - "repository", - "codebase", - "project", - "folder", - "filesystem", - "file tree", - "files", - "path", - ) - - user_text = (user_message or "").strip().lower() - user_targets_workspace = ( - any(marker in user_text for marker in workspace_markers) - or "~/" in user_text - or "/" in user_text - ) - assistant_mentions_action = any(marker in assistant_text for marker in action_markers) - assistant_targets_workspace = any( - marker in assistant_text for marker in workspace_markers - ) - return (user_targets_workspace or assistant_targets_workspace) and assistant_mentions_action - + """Forwarder — see ``agent.agent_runtime_helpers.looks_like_codex_intermediate_ack``.""" + from agent.agent_runtime_helpers import looks_like_codex_intermediate_ack + return looks_like_codex_intermediate_ack(self, user_message, assistant_content, messages) def _extract_reasoning(self, assistant_message) -> Optional[str]: """Forwarder — see ``agent.agent_runtime_helpers.extract_reasoning``.""" @@ -1547,68 +1296,9 @@ class AIAgent: @staticmethod def _extract_api_error_context(error: Exception) -> Dict[str, Any]: - """Extract structured rate-limit details from provider errors.""" - context: Dict[str, Any] = {} - - body = getattr(error, "body", None) - payload = None - if isinstance(body, dict): - payload = body.get("error") if isinstance(body.get("error"), dict) else body - if isinstance(payload, dict): - reason = payload.get("code") or payload.get("error") - if isinstance(reason, str) and reason.strip(): - context["reason"] = reason.strip() - message = payload.get("message") or payload.get("error_description") - if isinstance(message, str) and message.strip(): - context["message"] = message.strip() - for key in ("resets_at", "reset_at"): - value = payload.get(key) - if value not in {None, ""}: - context["reset_at"] = value - break - retry_after = payload.get("retry_after") - if retry_after not in {None, ""} and "reset_at" not in context: - try: - context["reset_at"] = time.time() + float(retry_after) - except (TypeError, ValueError): - pass - - response = getattr(error, "response", None) - headers = getattr(response, "headers", None) - if headers: - retry_after = headers.get("retry-after") or headers.get("Retry-After") - if retry_after and "reset_at" not in context: - try: - context["reset_at"] = time.time() + float(retry_after) - except (TypeError, ValueError): - pass - ratelimit_reset = headers.get("x-ratelimit-reset") - if ratelimit_reset and "reset_at" not in context: - context["reset_at"] = ratelimit_reset - - if "message" not in context: - raw_message = str(error).strip() - if raw_message: - context["message"] = raw_message[:500] - - if "reset_at" not in context: - message = context.get("message") or "" - if isinstance(message, str): - delay_match = re.search(r"quotaResetDelay[:\s\"]+(\\d+(?:\\.\\d+)?)(ms|s)", message, re.IGNORECASE) - if delay_match: - value = float(delay_match.group(1)) - seconds = value / 1000.0 if delay_match.group(2).lower() == "ms" else value - context["reset_at"] = time.time() + seconds - else: - sec_match = re.search( - r"retry\s+(?:after\s+)?(\d+(?:\.\d+)?)\s*(?:sec|secs|seconds|s\b)", - message, - re.IGNORECASE, - ) - if sec_match: - context["reset_at"] = time.time() + float(sec_match.group(1)) - - return context + """Forwarder — see ``agent.agent_runtime_helpers.extract_api_error_context``.""" + from agent.agent_runtime_helpers import extract_api_error_context + return extract_api_error_context(error) def _usage_summary_for_api_request_hook(self, response: Any) -> Optional[Dict[str, Any]]: """Token buckets for ``post_api_request`` plugins (no raw ``response`` object).""" @@ -1965,67 +1655,9 @@ class AIAgent: return "\n".join(lines) def _apply_pending_steer_to_tool_results(self, messages: list, num_tool_msgs: int) -> None: - """Append any pending /steer text to the last tool result in this turn. - - Called at the end of a tool-call batch, before the next API call. - The steer is appended to the last ``role:"tool"`` message's content - with a clear marker so the model understands it came from the user - and NOT from the tool itself. Role alternation is preserved — - nothing new is inserted, we only modify existing content. - - Args: - messages: The running messages list. - num_tool_msgs: Number of tool results appended in this batch; - used to locate the tail slice safely. - """ - if num_tool_msgs <= 0 or not messages: - return - steer_text = self._drain_pending_steer() - if not steer_text: - return - # Find the last tool-role message in the recent tail. Skipping - # non-tool messages defends against future code appending - # something else at the boundary. - target_idx = None - for j in range(len(messages) - 1, max(len(messages) - num_tool_msgs - 1, -1), -1): - msg = messages[j] - if isinstance(msg, dict) and msg.get("role") == "tool": - target_idx = j - break - if target_idx is None: - # No tool result in this batch (e.g. all skipped by interrupt); - # put the steer back so the caller's fallback path can deliver - # it as a normal next-turn user message. - _lock = getattr(self, "_pending_steer_lock", None) - if _lock is not None: - with _lock: - if self._pending_steer: - self._pending_steer = self._pending_steer + "\n" + steer_text - else: - self._pending_steer = steer_text - else: - existing = getattr(self, "_pending_steer", None) - self._pending_steer = (existing + "\n" + steer_text) if existing else steer_text - return - marker = f"\n\nUser guidance: {steer_text}" - existing_content = messages[target_idx].get("content", "") - if not isinstance(existing_content, str): - # Anthropic multimodal content blocks — preserve them and append - # a text block at the end. - try: - blocks = list(existing_content) if existing_content else [] - blocks.append({"type": "text", "text": marker.lstrip()}) - messages[target_idx]["content"] = blocks - except Exception: - # Fall back to string replacement if content shape is unexpected. - messages[target_idx]["content"] = f"{existing_content}{marker}" - else: - messages[target_idx]["content"] = existing_content + marker - logger.info( - "Delivered /steer to agent after tool batch (%d chars): %s", - len(steer_text), - steer_text[:120] + ("..." if len(steer_text) > 120 else ""), - ) + """Forwarder — see ``agent.agent_runtime_helpers.apply_pending_steer_to_tool_results``.""" + from agent.agent_runtime_helpers import apply_pending_steer_to_tool_results + return apply_pending_steer_to_tool_results(self, messages, num_tool_msgs) def _touch_activity(self, desc: str) -> None: """Update the last-activity timestamp and description (thread-safe).""" @@ -2383,74 +2015,9 @@ class AIAgent: @staticmethod def _sanitize_api_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Fix orphaned tool_call / tool_result pairs before every LLM call. - - Runs unconditionally — not gated on whether the context compressor - is present — so orphans from session loading or manual message - manipulation are always caught. - """ - # --- Role allowlist: drop messages with roles the API won't accept --- - filtered = [] - for msg in messages: - role = msg.get("role") - if role not in AIAgent._VALID_API_ROLES: - logger.debug( - "Pre-call sanitizer: dropping message with invalid role %r", - role, - ) - continue - filtered.append(msg) - messages = filtered - - surviving_call_ids: set = set() - for msg in messages: - if msg.get("role") == "assistant": - for tc in msg.get("tool_calls") or []: - cid = AIAgent._get_tool_call_id_static(tc) - if cid: - surviving_call_ids.add(cid) - - result_call_ids: set = set() - for msg in messages: - if msg.get("role") == "tool": - cid = msg.get("tool_call_id") - if cid: - result_call_ids.add(cid) - - # 1. Drop tool results with no matching assistant call - orphaned_results = result_call_ids - surviving_call_ids - if orphaned_results: - messages = [ - m for m in messages - if not (m.get("role") == "tool" and m.get("tool_call_id") in orphaned_results) - ] - logger.debug( - "Pre-call sanitizer: removed %d orphaned tool result(s)", - len(orphaned_results), - ) - - # 2. Inject stub results for calls whose result was dropped - missing_results = surviving_call_ids - result_call_ids - if missing_results: - patched: List[Dict[str, Any]] = [] - for msg in messages: - patched.append(msg) - if msg.get("role") == "assistant": - for tc in msg.get("tool_calls") or []: - cid = AIAgent._get_tool_call_id_static(tc) - if cid in missing_results: - patched.append({ - "role": "tool", - "name": AIAgent._get_tool_call_name_static(tc), - "content": "[Result unavailable — see context summary above]", - "tool_call_id": cid, - }) - messages = patched - logger.debug( - "Pre-call sanitizer: added %d stub tool result(s)", - len(missing_results), - ) - return messages + """Forwarder — see ``agent.agent_runtime_helpers.sanitize_api_messages``.""" + from agent.agent_runtime_helpers import sanitize_api_messages + return sanitize_api_messages(messages) @staticmethod def _is_thinking_only_assistant(msg: Dict[str, Any]) -> bool: @@ -2564,76 +2131,9 @@ class AIAgent: return unique if len(unique) < len(tool_calls) else tool_calls def _repair_tool_call(self, tool_name: str) -> str | None: - """Attempt to repair a mismatched tool name before aborting. - - Models sometimes emit variants of a tool name that differ only - in casing, separators, or class-like suffixes. Normalize - aggressively before falling back to fuzzy match: - - 1. Lowercase direct match. - 2. Lowercase + hyphens/spaces -> underscores. - 3. CamelCase -> snake_case (TodoTool -> todo_tool). - 4. Strip trailing ``_tool`` / ``-tool`` / ``tool`` suffix that - Claude-style models sometimes tack on (TodoTool_tool -> - TodoTool -> Todo -> todo). Applied twice so double-tacked - suffixes like ``TodoTool_tool`` reduce all the way. - 5. Fuzzy match (difflib, cutoff=0.7). - - See #14784 for the original reports (TodoTool_tool, Patch_tool, - BrowserClick_tool were all returning "Unknown tool" before). - - Returns the repaired name if found in valid_tool_names, else None. - """ - import re - from difflib import get_close_matches - - if not tool_name: - return None - - def _norm(s: str) -> str: - return s.lower().replace("-", "_").replace(" ", "_") - - def _camel_snake(s: str) -> str: - return re.sub(r"(? str | None: - lc = s.lower() - for suffix in ("_tool", "-tool", "tool"): - if lc.endswith(suffix): - return s[: -len(suffix)].rstrip("_-") - return None - - # Cheap fast-paths first — these cover the common case. - lowered = tool_name.lower() - if lowered in self.valid_tool_names: - return lowered - normalized = _norm(tool_name) - if normalized in self.valid_tool_names: - return normalized - - # Build the full candidate set for class-like emissions. - cands: set[str] = {tool_name, lowered, normalized, _camel_snake(tool_name)} - # Strip trailing tool-suffix up to twice — TodoTool_tool needs it. - for _ in range(2): - extra: set[str] = set() - for c in cands: - stripped = _strip_tool_suffix(c) - if stripped: - extra.add(stripped) - extra.add(_norm(stripped)) - extra.add(_camel_snake(stripped)) - cands |= extra - - for c in cands: - if c and c in self.valid_tool_names: - return c - - # Fuzzy match as last resort. - matches = get_close_matches(lowered, self.valid_tool_names, n=1, cutoff=0.7) - if matches: - return matches[0] - - return None + """Forwarder — see ``agent.agent_runtime_helpers.repair_tool_call``.""" + from agent.agent_runtime_helpers import repair_tool_call + return repair_tool_call(self, tool_name) def _invalidate_system_prompt(self): """Forwarder — see ``agent.system_prompt.invalidate_system_prompt``.""" @@ -2745,62 +2245,9 @@ class AIAgent: @staticmethod def _force_close_tcp_sockets(client: Any) -> int: - """Force-close underlying TCP sockets to prevent CLOSE-WAIT accumulation. - - When a provider drops a connection mid-stream, httpx's ``client.close()`` - performs a graceful shutdown which leaves sockets in CLOSE-WAIT until the - OS times them out (often minutes). This method walks the httpx transport - pool and issues ``socket.shutdown(SHUT_RDWR)`` + ``socket.close()`` to - force an immediate TCP RST, freeing the file descriptors. - - Returns the number of sockets force-closed. - """ - import socket as _socket - - closed = 0 - try: - http_client = getattr(client, "_client", None) - if http_client is None: - return 0 - transport = getattr(http_client, "_transport", None) - if transport is None: - return 0 - pool = getattr(transport, "_pool", None) - if pool is None: - return 0 - # httpx uses httpcore connection pools; connections live in - # _connections (list) or _pool (list) depending on version. - connections = ( - getattr(pool, "_connections", None) - or getattr(pool, "_pool", None) - or [] - ) - for conn in list(connections): - stream = ( - getattr(conn, "_network_stream", None) - or getattr(conn, "_stream", None) - ) - if stream is None: - continue - sock = getattr(stream, "_sock", None) - if sock is None: - sock = getattr(stream, "stream", None) - if sock is not None: - sock = getattr(sock, "_sock", None) - if sock is None: - continue - try: - sock.shutdown(_socket.SHUT_RDWR) - except OSError: - pass - try: - sock.close() - except OSError: - pass - closed += 1 - except Exception as exc: - logger.debug("Force-close TCP sockets sweep error: %s", exc) - return closed + """Forwarder — see ``agent.agent_runtime_helpers.force_close_tcp_sockets``.""" + from agent.agent_runtime_helpers import force_close_tcp_sockets + return force_close_tcp_sockets(client) def _close_openai_client(self, client: Any, *, reason: str, shared: bool) -> None: if client is None: @@ -2860,74 +2307,9 @@ class AIAgent: return self.client def _cleanup_dead_connections(self) -> bool: - """Detect and clean up dead TCP connections on the primary client. - - Inspects the httpx connection pool for sockets in unhealthy states - (CLOSE-WAIT, errors). If any are found, force-closes all sockets - and rebuilds the primary client from scratch. - - Returns True if dead connections were found and cleaned up. - """ - client = getattr(self, "client", None) - if client is None: - return False - try: - http_client = getattr(client, "_client", None) - if http_client is None: - return False - transport = getattr(http_client, "_transport", None) - if transport is None: - return False - pool = getattr(transport, "_pool", None) - if pool is None: - return False - connections = ( - getattr(pool, "_connections", None) - or getattr(pool, "_pool", None) - or [] - ) - dead_count = 0 - for conn in list(connections): - # Check for connections that are idle but have closed sockets - stream = ( - getattr(conn, "_network_stream", None) - or getattr(conn, "_stream", None) - ) - if stream is None: - continue - sock = getattr(stream, "_sock", None) - if sock is None: - sock = getattr(stream, "stream", None) - if sock is not None: - sock = getattr(sock, "_sock", None) - if sock is None: - continue - # Probe socket health with a non-blocking recv peek - import socket as _socket - try: - sock.setblocking(False) - data = sock.recv(1, _socket.MSG_PEEK | _socket.MSG_DONTWAIT) - if data == b"": - dead_count += 1 - except BlockingIOError: - pass # No data available — socket is healthy - except OSError: - dead_count += 1 - finally: - try: - sock.setblocking(True) - except OSError: - pass - if dead_count > 0: - logger.warning( - "Found %d dead connection(s) in client pool — rebuilding client", - dead_count, - ) - self._replace_primary_openai_client(reason="dead_connection_cleanup") - return True - except Exception as exc: - logger.debug("Dead connection check error: %s", exc) - return False + """Forwarder — see ``agent.agent_runtime_helpers.cleanup_dead_connections``.""" + from agent.agent_runtime_helpers import cleanup_dead_connections + return cleanup_dead_connections(self) @staticmethod def _api_kwargs_have_image_parts(api_kwargs: dict) -> bool: @@ -4039,74 +3421,9 @@ class AIAgent: ) def _copy_reasoning_content_for_api(self, source_msg: dict, api_msg: dict) -> None: - """Copy provider-facing reasoning fields onto an API replay message.""" - if source_msg.get("role") != "assistant": - return - - # 1. Explicit reasoning_content already set — preserve it verbatim - # (includes DeepSeek/Kimi's own space-placeholder written at creation - # time, and any valid reasoning content from the same provider). - # - # Exception: sessions persisted BEFORE #17341 have empty-string - # placeholders pinned at creation time. DeepSeek V4 Pro rejects - # those with HTTP 400. When the active provider enforces the - # thinking-mode echo, upgrade "" → " " on replay so stale history - # doesn't 400 the user on the next turn. - existing = source_msg.get("reasoning_content") - if isinstance(existing, str): - if existing == "" and self._needs_thinking_reasoning_pad(): - api_msg["reasoning_content"] = " " - else: - api_msg["reasoning_content"] = existing - return - - needs_thinking_pad = self._needs_thinking_reasoning_pad() - - # 2. Cross-provider poisoned history (#15748): on DeepSeek/Kimi, - # if the source turn has tool_calls AND a 'reasoning' field but no - # 'reasoning_content' key, the 'reasoning' text was written by a - # prior provider (e.g. MiniMax) — DeepSeek's own _build_assistant_message - # pins reasoning_content at creation time for tool-call turns, so the - # shape (reasoning set, reasoning_content absent, tool_calls present) - # is unreachable from same-provider DeepSeek history after this fix. - # Inject a single space to satisfy the API without leaking another - # provider's chain of thought to DeepSeek/Kimi. Space (not "") - # because DeepSeek V4 Pro rejects empty-string reasoning_content - # in thinking mode (refs #17341). - normalized_reasoning = source_msg.get("reasoning") - if ( - needs_thinking_pad - and source_msg.get("tool_calls") - and isinstance(normalized_reasoning, str) - and normalized_reasoning - ): - api_msg["reasoning_content"] = " " - return - - # 3. Healthy session: promote 'reasoning' field to 'reasoning_content' - # for providers that use the internal 'reasoning' key. - # This must happen before the unconditional empty-string fallback so - # genuine reasoning content is not overwritten (#15812 regression in - # PR #15478). - if isinstance(normalized_reasoning, str) and normalized_reasoning: - api_msg["reasoning_content"] = normalized_reasoning - return - - # 4. DeepSeek / Kimi thinking mode: all assistant messages need - # reasoning_content. Inject a single space to satisfy the provider's - # requirement when no explicit reasoning content is present. Covers - # both tool-call turns (already-poisoned history with no reasoning - # at all) and plain text turns. Space (not "") because DeepSeek V4 - # Pro tightened validation and rejects empty string with HTTP 400 - # ("The reasoning content in the thinking mode must be passed back - # to the API"). Refs #17341. - if needs_thinking_pad: - api_msg["reasoning_content"] = " " - return - - # 5. reasoning_content was present but not a string (e.g. None after - # context compaction). Don't pass null to the API. - api_msg.pop("reasoning_content", None) + """Forwarder — see ``agent.agent_runtime_helpers.copy_reasoning_content_for_api``.""" + from agent.agent_runtime_helpers import copy_reasoning_content_for_api + return copy_reasoning_content_for_api(self, source_msg, api_msg) @staticmethod def _sanitize_tool_calls_for_strict_api(api_msg: dict) -> dict: @@ -4251,89 +3568,9 @@ class AIAgent: def _invoke_tool(self, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False) -> str: - """Invoke a single tool and return the result string. No display logic. - - Handles both agent-level tools (todo, memory, etc.) and registry-dispatched - tools. Used by the concurrent execution path; the sequential path retains - its own inline invocation for backward-compatible display handling. - """ - # Check plugin hooks for a block directive before executing anything. - block_message: Optional[str] = None - if not pre_tool_block_checked: - try: - from hermes_cli.plugins import get_pre_tool_call_block_message - block_message = get_pre_tool_call_block_message( - function_name, function_args, task_id=effective_task_id or "", - ) - except Exception: - pass - if block_message is not None: - return json.dumps({"error": block_message}, ensure_ascii=False) - - if function_name == "todo": - from tools.todo_tool import todo_tool as _todo_tool - return _todo_tool( - todos=function_args.get("todos"), - merge=function_args.get("merge", False), - store=self._todo_store, - ) - elif function_name == "session_search": - session_db = self._get_session_db_for_recall() - if not session_db: - from hermes_state import format_session_db_unavailable - return json.dumps({"success": False, "error": format_session_db_unavailable()}) - from tools.session_search_tool import session_search as _session_search - return _session_search( - query=function_args.get("query", ""), - role_filter=function_args.get("role_filter"), - limit=function_args.get("limit", 3), - db=session_db, - current_session_id=self.session_id, - ) - elif function_name == "memory": - target = function_args.get("target", "memory") - from tools.memory_tool import memory_tool as _memory_tool - result = _memory_tool( - action=function_args.get("action"), - target=target, - content=function_args.get("content"), - old_text=function_args.get("old_text"), - store=self._memory_store, - ) - # Bridge: notify external memory provider of built-in memory writes - if self._memory_manager and function_args.get("action") in {"add", "replace"}: - try: - self._memory_manager.on_memory_write( - function_args.get("action", ""), - target, - function_args.get("content", ""), - metadata=self._build_memory_write_metadata( - task_id=effective_task_id, - tool_call_id=tool_call_id, - ), - ) - except Exception: - pass - return result - elif self._memory_manager and self._memory_manager.has_tool(function_name): - return self._memory_manager.handle_tool_call(function_name, function_args) - elif function_name == "clarify": - from tools.clarify_tool import clarify_tool as _clarify_tool - return _clarify_tool( - question=function_args.get("question", ""), - choices=function_args.get("choices"), - callback=self.clarify_callback, - ) - elif function_name == "delegate_task": - return self._dispatch_delegate_task(function_args) - else: - return handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call_id, - session_id=self.session_id or "", - enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, - skip_pre_tool_call_hook=True, - ) + """Forwarder — see ``agent.agent_runtime_helpers.invoke_tool``.""" + from agent.agent_runtime_helpers import invoke_tool + return invoke_tool(self, function_name, function_args, effective_task_id, tool_call_id, messages, pre_tool_block_checked) @staticmethod def _wrap_verbose(label: str, text: str, indent: str = " ") -> str: From 47823790b00255b54c999b9559df2617f18b1df5 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 22:55:49 -0700 Subject: [PATCH 016/142] =?UTF-8?q?refactor(run=5Fagent):=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20keyword-forward=20=5F=5Finit=5F=5F,=20drop=20dea?= =?UTF-8?q?d=20code,=20tighten=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes from PR #27248 review: 1. **__init__ forwarder is now keyword-forwarded** (daimon-nous review). Previously the run_agent.AIAgent.__init__ wrapper forwarded all 64 params positionally to agent.agent_init.init_agent, so adding a 65th param on main would require three lockstep edits (signature, init_agent signature, forwarder call) or silently shift every value. Keyword forwarding makes this trivially safe — adding a param now only needs the two signatures and one extra keyword line. 2. **Drop dead _ra() in agent/codex_runtime.py** (daimon-nous + Copilot). The lazy run_agent reference was defined but never called inside this module — the codex paths use agent.* accessors only. 3. **Drop unused imports in agent/codex_runtime.py** (Copilot): contextvars, threading, time, uuid, Optional. Carried over from run_agent.py during the original extraction. 4. **Tighten three source-introspection test guards** (Copilot): - test_memory_nudge_counter_hydration.py — was scanning the concatenated source of run_agent.py + agent/conversation_loop.py and matching self.X or agent.X form. Now asserts the hydration block lives in agent/conversation_loop.py specifically with the agent.X form — the body never moves back, so if it ever drifts a future re-introduction fails the guard. - test_run_agent.py::TestMemoryNudgeCounterPersistence — anchor on agent.iteration_budget = IterationBudget exactly (was just iteration_budget = IterationBudget) so an unrelated identifier ending in iteration_budget can't match. - test_run_agent.py::TestMemoryProviderTurnStart — assert the agent._user_turn_count form directly (the extracted body uses agent.X, not self.X — accepting either was a transitional fudge). - test_jsondecodeerror_retryable.py — scan agent/conversation_loop.py only, not the concatenation. Not addressed in this commit: * Pre-existing bugs in agent/tool_executor.py (heartbeat index mismatch when calls are blocked, _current_tool clobber in result loop, blocked-counted-as-completed in spinner summary, dead result_preview computation). These were preserved byte-for-byte from the original _execute_tool_calls_concurrent — worth a separate follow-up PR with proper tests. * _OpenAIProxy.__instancecheck__ concern — pre-existing, not flagged by any of the original test patches (nothing actually does isinstance(x, OpenAI) against the proxy instance). * agent_init.py:949 mem_config potential NameError — pre-existing; only triggers if _agent_cfg.get('memory', {}) itself raises, which it can't with a stock dict. tests/run_agent/ + tests/agent/: 4313 passed, 1 pre-existing test_auxiliary_client failure (unchanged). run_agent.py: 3821 -> 3937 lines (+116 from the keyword-forwarded init call's verbosity). Final: 16083 -> 3937 (-12146, 75% reduction). --- agent/codex_runtime.py | 13 +--- run_agent.py | 68 ++++++++++++++++++- .../test_jsondecodeerror_retryable.py | 13 ++-- .../test_memory_nudge_counter_hydration.py | 24 ++++--- tests/run_agent/test_run_agent.py | 20 +++--- 5 files changed, 98 insertions(+), 40 deletions(-) diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 73a455f6bb0..b2e9b714586 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -16,26 +16,15 @@ compatibility. from __future__ import annotations -import contextvars import json import logging import os -import threading -import time -import uuid from types import SimpleNamespace -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List logger = logging.getLogger(__name__) -def _ra(): - """Lazy ``run_agent`` reference for test-patch routing.""" - import run_agent - return run_agent - - - def run_codex_app_server_turn( agent, *, diff --git a/run_agent.py b/run_agent.py index af1483e0e32..8b68dee6857 100644 --- a/run_agent.py +++ b/run_agent.py @@ -376,7 +376,73 @@ class AIAgent: ): """Forwarder — see ``agent.agent_init.init_agent``.""" from agent.agent_init import init_agent - init_agent(self, base_url, api_key, provider, api_mode, acp_command, acp_args, command, args, model, max_iterations, tool_delay, enabled_toolsets, disabled_toolsets, save_trajectories, verbose_logging, quiet_mode, ephemeral_system_prompt, log_prefix_chars, log_prefix, providers_allowed, providers_ignored, providers_order, provider_sort, provider_require_parameters, provider_data_collection, openrouter_min_coding_score, session_id, tool_progress_callback, tool_start_callback, tool_complete_callback, thinking_callback, reasoning_callback, clarify_callback, step_callback, stream_delta_callback, interim_assistant_callback, tool_gen_callback, status_callback, max_tokens, reasoning_config, service_tier, request_overrides, prefill_messages, platform, user_id, user_name, chat_id, chat_name, chat_type, thread_id, gateway_session_key, skip_context_files, load_soul_identity, skip_memory, session_db, parent_session_id, iteration_budget, fallback_model, credential_pool, checkpoints_enabled, checkpoint_max_snapshots, checkpoint_max_total_size_mb, checkpoint_max_file_size_mb, pass_session_id) + init_agent( + self, + base_url=base_url, + api_key=api_key, + provider=provider, + api_mode=api_mode, + acp_command=acp_command, + acp_args=acp_args, + command=command, + args=args, + model=model, + max_iterations=max_iterations, + tool_delay=tool_delay, + enabled_toolsets=enabled_toolsets, + disabled_toolsets=disabled_toolsets, + save_trajectories=save_trajectories, + verbose_logging=verbose_logging, + quiet_mode=quiet_mode, + ephemeral_system_prompt=ephemeral_system_prompt, + log_prefix_chars=log_prefix_chars, + log_prefix=log_prefix, + providers_allowed=providers_allowed, + providers_ignored=providers_ignored, + providers_order=providers_order, + provider_sort=provider_sort, + provider_require_parameters=provider_require_parameters, + provider_data_collection=provider_data_collection, + openrouter_min_coding_score=openrouter_min_coding_score, + session_id=session_id, + tool_progress_callback=tool_progress_callback, + tool_start_callback=tool_start_callback, + tool_complete_callback=tool_complete_callback, + thinking_callback=thinking_callback, + reasoning_callback=reasoning_callback, + clarify_callback=clarify_callback, + step_callback=step_callback, + stream_delta_callback=stream_delta_callback, + interim_assistant_callback=interim_assistant_callback, + tool_gen_callback=tool_gen_callback, + status_callback=status_callback, + max_tokens=max_tokens, + reasoning_config=reasoning_config, + service_tier=service_tier, + request_overrides=request_overrides, + prefill_messages=prefill_messages, + platform=platform, + user_id=user_id, + user_name=user_name, + chat_id=chat_id, + chat_name=chat_name, + chat_type=chat_type, + thread_id=thread_id, + gateway_session_key=gateway_session_key, + skip_context_files=skip_context_files, + load_soul_identity=load_soul_identity, + skip_memory=skip_memory, + session_db=session_db, + parent_session_id=parent_session_id, + iteration_budget=iteration_budget, + fallback_model=fallback_model, + credential_pool=credential_pool, + checkpoints_enabled=checkpoints_enabled, + checkpoint_max_snapshots=checkpoint_max_snapshots, + checkpoint_max_total_size_mb=checkpoint_max_total_size_mb, + checkpoint_max_file_size_mb=checkpoint_max_file_size_mb, + pass_session_id=pass_session_id, + ) def _get_session_db_for_recall(self): """Return a SessionDB for recall, lazily creating it if an entrypoint forgot. diff --git a/tests/run_agent/test_jsondecodeerror_retryable.py b/tests/run_agent/test_jsondecodeerror_retryable.py index e810092613e..0bd4fc09f9f 100644 --- a/tests/run_agent/test_jsondecodeerror_retryable.py +++ b/tests/run_agent/test_jsondecodeerror_retryable.py @@ -73,17 +73,20 @@ class TestAgentLoopSourceStillHasCarveOut: revert that happens to leave the test file intact.""" def test_run_agent_excludes_jsondecodeerror_from_local_validation(self): - import run_agent import inspect from agent import conversation_loop - # The body moved into agent/conversation_loop.py; scan both for safety. - src = inspect.getsource(run_agent) + inspect.getsource(conversation_loop) + # The agent loop body lives in agent/conversation_loop.py after + # the run_agent.py refactor. Assert the carve-out is present in + # the extracted module specifically — if it ever moves back or + # disappears, this fails loudly rather than silently passing + # against a non-existent inline replica. + src = inspect.getsource(conversation_loop) # The predicate we care about must reference json.JSONDecodeError # in its exclusion tuple. We check for the specific co-occurrence # rather than the literal string so harmless reformatting doesn't # break us. assert "is_local_validation_error" in src assert "JSONDecodeError" in src, ( - "run_agent.py must carve out json.JSONDecodeError from the " - "is_local_validation_error classification — see #14782." + "agent/conversation_loop.py must carve out json.JSONDecodeError " + "from the is_local_validation_error classification — see #14782." ) diff --git a/tests/run_agent/test_memory_nudge_counter_hydration.py b/tests/run_agent/test_memory_nudge_counter_hydration.py index f3923f83442..1b9bf56005d 100644 --- a/tests/run_agent/test_memory_nudge_counter_hydration.py +++ b/tests/run_agent/test_memory_nudge_counter_hydration.py @@ -121,19 +121,21 @@ def test_production_code_contains_hydration_block(): run_conversation(). If someone deletes it, tests above still pass against the inline replica — this fails them awake. - The body now lives in agent/conversation_loop.py after the - run_agent.py refactor; check both files for safety. + After the run_agent.py refactor the agent-loop body lives in + ``agent/conversation_loop.py`` and uses ``agent.X`` rather than + ``self.X``. Assert the block is present in the extracted module + specifically — if it ever drifts back into run_agent.py or + disappears entirely, this guard fails loudly. """ from pathlib import Path repo = Path(__file__).resolve().parents[2] - src_ra = (repo / "run_agent.py").read_text(encoding="utf-8") - src_cl = (repo / "agent" / "conversation_loop.py").read_text(encoding="utf-8") - content = src_ra + src_cl + cl_path = repo / "agent" / "conversation_loop.py" + src_cl = cl_path.read_text(encoding="utf-8") # Anchor on the unique comment + the modulo line. - assert "Hydrate per-session nudge counters from persisted history" in content - # The line uses ``self.`` in run_agent.py form and ``agent.`` in the - # extracted module, accept either. - assert ( - "self._turns_since_memory = prior_user_turns % self._memory_nudge_interval" in content - or "agent._turns_since_memory = prior_user_turns % agent._memory_nudge_interval" in content + assert "Hydrate per-session nudge counters from persisted history" in src_cl, ( + f"Hydration comment missing from {cl_path}" ) + assert ( + "agent._turns_since_memory = prior_user_turns % agent._memory_nudge_interval" + in src_cl + ), f"Hydration modulo assignment missing from {cl_path}" diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 76254d4eda5..48079477535 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -5210,12 +5210,13 @@ class TestMemoryNudgeCounterPersistence: # The preamble resets many fields (retry counts, budget, etc.) # before the main loop. Find that reset block and verify our # counters aren't in it. The reset block ends at iteration_budget. - # After the run_agent.py refactor the body uses ``agent.X`` instead - # of ``self.X``, so accept either form. - preamble_end = src.index("iteration_budget = IterationBudget") + # The extracted body uses ``agent.X`` (not ``self.X``). Anchor + # exactly on ``agent.iteration_budget = IterationBudget`` so an + # unrelated identifier ending in ``iteration_budget`` (e.g. + # ``_iteration_budget`` or ``shared_iteration_budget``) can't + # match the boundary. + preamble_end = src.index("agent.iteration_budget = IterationBudget") preamble = src[:preamble_end] - assert "self._turns_since_memory = 0" not in preamble - assert "self._iters_since_skill = 0" not in preamble assert "agent._turns_since_memory = 0" not in preamble assert "agent._iters_since_skill = 0" not in preamble @@ -5316,9 +5317,6 @@ class TestMemoryProviderTurnStart: import inspect from agent.conversation_loop import run_conversation as _rc src = inspect.getsource(_rc) - # After the run_agent.py refactor the body uses ``agent.X`` instead - # of ``self.X``. Accept either spelling. - assert ( - "on_turn_start(self._user_turn_count" in src - or "on_turn_start(agent._user_turn_count" in src - ) + # The extracted body uses ``agent.X`` rather than ``self.X``; + # assert the extracted-form spelling directly. + assert "on_turn_start(agent._user_turn_count" in src From 8d4766afcae676efba0269787ddad7c769ba6c24 Mon Sep 17 00:00:00 2001 From: Sylw3ster Date: Sat, 16 May 2026 02:08:40 +0300 Subject: [PATCH 017/142] fix(api_server): coerce stringified booleans in request payloads --- gateway/platforms/api_server.py | 40 ++++++++++-- tests/gateway/test_api_server.py | 87 +++++++++++++++++++++++++++ tests/gateway/test_api_server_runs.py | 22 +++++++ 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 809d6cd8a03..ebd4f014690 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -71,6 +71,35 @@ def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: return default +_TRUE_REQUEST_BOOL_STRINGS = frozenset({"1", "true", "yes", "on"}) +_FALSE_REQUEST_BOOL_STRINGS = frozenset({"0", "false", "no", "off"}) + + +def _coerce_request_bool(value: Any, default: bool = False) -> bool: + """Normalize boolean-like API payload values. + + External clients should send real JSON booleans, but some OpenAI-compatible + frontends and middleware serialize flags like ``stream`` as strings. Using + Python truthiness on those values misroutes requests because ``"false"`` is + still truthy. Treat only explicit bool-ish scalars as booleans; everything + else falls back to the caller's default. + """ + if isinstance(value, bool): + return value + if value is None: + return default + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in _TRUE_REQUEST_BOOL_STRINGS: + return True + if normalized in _FALSE_REQUEST_BOOL_STRINGS: + return False + return default + if isinstance(value, (int, float)): + return bool(value) + return default + + def _normalize_chat_content( content: Any, *, _max_depth: int = 10, _depth: int = 0, ) -> str: @@ -1005,7 +1034,7 @@ class APIServerAdapter(BasePlatformAdapter): status=400, ) - stream = body.get("stream", False) + stream = _coerce_request_bool(body.get("stream"), default=False) # Extract system message (becomes ephemeral system prompt layered ON TOP of core) system_prompt = None @@ -2082,7 +2111,7 @@ class APIServerAdapter(BasePlatformAdapter): instructions = body.get("instructions") previous_response_id = body.get("previous_response_id") conversation = body.get("conversation") - store = body.get("store", True) + store = _coerce_request_bool(body.get("store"), default=True) # conversation and previous_response_id are mutually exclusive if conversation and previous_response_id: @@ -2165,7 +2194,7 @@ class APIServerAdapter(BasePlatformAdapter): # groups the entire conversation under one session entry. session_id = stored_session_id or str(uuid.uuid4()) - stream = bool(body.get("stream", False)) + stream = _coerce_request_bool(body.get("stream"), default=False) if stream: # Streaming branch — emit OpenAI Responses SSE events as the # agent runs so frontends can render text deltas and tool @@ -3228,7 +3257,10 @@ class APIServerAdapter(BasePlatformAdapter): status=409, ) - resolve_all = bool(body.get("all") or body.get("resolve_all")) + resolve_all = ( + _coerce_request_bool(body.get("all"), default=False) + or _coerce_request_bool(body.get("resolve_all"), default=False) + ) try: from tools.approval import resolve_gateway_approval diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 032af7109a5..7d08d64bb32 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -704,6 +704,37 @@ class TestChatCompletionsEndpoint: assert "[DONE]" in body assert "Hello!" in body + @pytest.mark.asyncio + async def test_stream_string_false_returns_json_completion(self, adapter): + """Quoted false must not route chat completions into SSE mode.""" + mock_result = { + "final_response": "Hello! How can I help you today?", + "messages": [], + "api_calls": 1, + } + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + mock_result, + {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + ) + resp = await cli.post( + "/v1/chat/completions", + json={ + "model": "hermes-agent", + "messages": [{"role": "user", "content": "Hello"}], + "stream": "false", + }, + ) + + assert resp.status == 200 + assert "text/event-stream" not in resp.headers.get("Content-Type", "") + data = await resp.json() + assert data["object"] == "chat.completion" + assert data["choices"][0]["message"]["content"] == mock_result["final_response"] + @pytest.mark.asyncio async def test_stream_task_done_callback_enqueues_eos_for_chat_completions(self, adapter): """Regression guard for #24451: completion callback must signal SSE EOS.""" @@ -1655,6 +1686,31 @@ class TestResponsesEndpoint: # The response has an ID but it shouldn't be retrievable assert adapter._response_store.get(data["id"]) is None + @pytest.mark.asyncio + async def test_store_string_false_does_not_store(self, adapter): + """Quoted false must preserve ephemeral store=false semantics.""" + mock_result = {"final_response": "OK", "messages": [], "api_calls": 1} + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + mock_result, + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + ) + resp = await cli.post( + "/v1/responses", + json={ + "model": "hermes-agent", + "input": "Hello", + "store": "false", + }, + ) + + assert resp.status == 200 + data = await resp.json() + assert adapter._response_store.get(data["id"]) is None + @pytest.mark.asyncio async def test_instructions_inherited_from_previous(self, adapter): """If no instructions provided, carry forward from previous response.""" @@ -1749,6 +1805,37 @@ class TestResponsesStreaming: assert "Hello" in body assert " world" in body + @pytest.mark.asyncio + async def test_stream_string_false_returns_json_response(self, adapter): + """Quoted false must not route Responses API requests into SSE mode.""" + mock_result = { + "final_response": "Paris is the capital of France.", + "messages": [], + "api_calls": 1, + } + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + mock_result, + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + ) + resp = await cli.post( + "/v1/responses", + json={ + "model": "hermes-agent", + "input": "What is the capital of France?", + "stream": "false", + }, + ) + + assert resp.status == 200 + assert "text/event-stream" not in resp.headers.get("Content-Type", "") + data = await resp.json() + assert data["object"] == "response" + assert data["output"][0]["content"][0]["text"] == mock_result["final_response"] + @pytest.mark.asyncio async def test_stream_task_done_callback_enqueues_eos_for_responses(self, adapter): """Regression guard for #24451 on /v1/responses streaming path.""" diff --git a/tests/gateway/test_api_server_runs.py b/tests/gateway/test_api_server_runs.py index bdb00d74a7b..8e7169a658d 100644 --- a/tests/gateway/test_api_server_runs.py +++ b/tests/gateway/test_api_server_runs.py @@ -335,6 +335,28 @@ class TestRunEvents: "approval_not_pending", } + @pytest.mark.asyncio + async def test_approval_string_false_does_not_resolve_all(self, adapter): + """Quoted false must not fan out approval resolution across the queue.""" + app = _create_runs_app(adapter) + run_id = "run_bool_parse" + adapter._run_statuses[run_id] = {"run_id": run_id, "status": "running"} + adapter._run_approval_sessions[run_id] = "session-123" + + async with TestClient(TestServer(app)) as cli: + with patch("tools.approval.resolve_gateway_approval", return_value=1) as mock_resolve: + approval_resp = await cli.post( + f"/v1/runs/{run_id}/approval", + json={"choice": "once", "all": "false"}, + ) + + assert approval_resp.status == 200 + mock_resolve.assert_called_once_with( + "session-123", + "once", + resolve_all=False, + ) + @pytest.mark.asyncio async def test_events_not_found_returns_404(self, adapter): app = _create_runs_app(adapter) From bde3c7982c30796f8709cb0041d34ab36a4d7a9c Mon Sep 17 00:00:00 2001 From: darvsum Date: Sat, 16 May 2026 13:18:01 +0800 Subject: [PATCH 018/142] fix: preserve discover_models in _normalize_custom_provider_entry The _normalize_custom_provider_entry() function was dropping the discover_models field from custom_provider entries because: 1. It was not listed in _KNOWN_KEYS, so it was logged as an unknown key and ignored. 2. The function builds the normalized dict by explicitly copying known fields, so even if the warning was suppressed, the value was not carried through. This caused downstream model_switch.py to default discover_models to True, triggering /models HTTP probes on unreachable endpoints. With 4 unreachable internal endpoints at ~6s timeout each, the /api/model/options endpoint took ~24s instead of <1s. --- hermes_cli/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hermes_cli/config.py b/hermes_cli/config.py index c41158e42ae..e4447183746 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2914,6 +2914,7 @@ def _normalize_custom_provider_entry( "api_mode", "transport", "model", "default_model", "models", "context_length", "rate_limit_delay", "request_timeout_seconds", "stale_timeout_seconds", + "discover_models", } for camel, snake in _CAMEL_ALIASES.items(): if camel in entry and snake not in entry: @@ -3004,6 +3005,10 @@ def _normalize_custom_provider_entry( if isinstance(rate_limit_delay, (int, float)) and rate_limit_delay >= 0: normalized["rate_limit_delay"] = rate_limit_delay + discover_models = entry.get("discover_models") + if isinstance(discover_models, bool): + normalized["discover_models"] = discover_models + return normalized From 75e5d0f6bd412ff4ae719a6ebd98bfd5a471f66c Mon Sep 17 00:00:00 2001 From: hueilau <33933019+hueilau@users.noreply.github.com> Date: Sat, 16 May 2026 23:02:46 -0700 Subject: [PATCH 019/142] fix: strip image parts for non-vision models with provider profiles _propare_messages_for_non_vision_model() was only called in the legacy flag path (no provider profile). Providers with registered profiles (e.g. DeepSeek, Kimi) bypassed the strip, causing HTTP 400 errors when image_url content blocks reached their non-vision APIs. This mirrors the existing behavior in the legacy path, ensuring all non-vision models get image stripping regardless of profile status. Vision-capable models are unaffected (the function is a no-op for them). --- run_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/run_agent.py b/run_agent.py index b239f2aeb60..5e0a9ec06ac 100644 --- a/run_agent.py +++ b/run_agent.py @@ -10033,6 +10033,11 @@ class AIAgent: if _ephemeral_out is not None: self._ephemeral_max_output_tokens = None + # Strip image parts for non-vision models that have provider profiles + # (e.g. DeepSeek, Kimi). The legacy path below already does this, but + # registered providers with profiles were bypassing the strip. + api_messages = self._prepare_messages_for_non_vision_model(api_messages) + return _ct.build_kwargs( model=self.model, messages=api_messages, From 5338250dab14b3e4f9dfb306446e8c55835adfad Mon Sep 17 00:00:00 2001 From: Timur00Kh <32297275+Timur00Kh@users.noreply.github.com> Date: Sun, 17 May 2026 00:28:24 +0400 Subject: [PATCH 020/142] fix(gateway): add direct_messages_topic_id for synthetic Telegram DM events When /goal loop generates synthetic MessageEvents (goal continuations, status notices), the reply anchor is unavailable (message_id=None). For Telegram DM topic lanes, the Telegram adapter requires direct_messages_topic_id to route messages correctly; without it, the adapter falls back to message_thread_id=None, sending messages to the root 'All Messages' thread instead of the active topic lane. The fix includes direct_messages_topic_id in thread metadata for all non-General Telegram DM topics, ensuring queued/synthetic messages are delivered to the correct thread even when no reply anchor exists. --- gateway/run.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gateway/run.py b/gateway/run.py index 458603c3115..56185190e26 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -12546,6 +12546,12 @@ class GatewayRunner: and getattr(source, "chat_type", None) == "dm" ): metadata["telegram_dm_topic_reply_fallback"] = True + # Telegram DM topic lanes need direct_messages_topic_id in metadata + # so synthetic/queued messages (goal continuations, status notices) + # route to the correct topic even when reply anchor is unavailable. + tid = str(thread_id) + if tid and tid not in {"", "1"}: + metadata["direct_messages_topic_id"] = tid anchor = reply_to_message_id or getattr(source, "message_id", None) if anchor is not None: metadata["telegram_reply_to_message_id"] = str(anchor) From 8bf09455dc498581fe6dea21402ee2a9238a2212 Mon Sep 17 00:00:00 2001 From: Grogger Date: Sat, 16 May 2026 12:06:09 -0400 Subject: [PATCH 021/142] fix(windows): suppress console window flash on subprocess spawns Add creationflags=CREATE_NO_WINDOW to every Windows Popen call across the terminal, process registry, code execution, and kanban worker subsystems. Prevents visible CMD windows from flashing on the user's desktop during agent operation. Also adds the _IS_WINDOWS module constant to kanban_db.py where it was missing, for consistency with the other patched files. 5 Popen sites across 4 files: - tools/environments/local.py (terminal foreground spawn) - tools/process_registry.py (background process spawn) - tools/code_execution_tool.py (sandbox + interpreter probe) - hermes_cli/kanban_db.py (kanban worker spawn) --- hermes_cli/kanban_db.py | 2 ++ tools/code_execution_tool.py | 2 ++ tools/environments/local.py | 1 + tools/process_registry.py | 1 + 4 files changed, 6 insertions(+) diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 0db694ff5b1..9d5ddad6ed0 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -93,6 +93,7 @@ from toolsets import get_toolset_names VALID_STATUSES = {"triage", "todo", "ready", "running", "blocked", "done", "archived"} VALID_WORKSPACE_KINDS = {"scratch", "worktree", "dir"} KNOWN_TOOLSET_NAMES = frozenset(name.casefold() for name in get_toolset_names()) +_IS_WINDOWS = sys.platform == "win32" # A running task's claim is valid for 15 minutes; after that the next # dispatcher tick reclaims it. Workers that outlive this window should call @@ -4024,6 +4025,7 @@ def _default_spawn( stderr=subprocess.STDOUT, env=env, start_new_session=True, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, ) except FileNotFoundError: log_f.close() diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index 3822ce539f2..bdbc4bfbe1b 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -1238,6 +1238,7 @@ def execute_code( stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, preexec_fn=None if _IS_WINDOWS else os.setsid, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, ) # --- Poll loop: watch for exit, timeout, and interrupt --- @@ -1568,6 +1569,7 @@ def _is_usable_python(python_path: str) -> bool: "import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)"], timeout=5, capture_output=True, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, ) return result.returncode == 0 except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError): diff --git a/tools/environments/local.py b/tools/environments/local.py index 3b9d65449fa..177e5efab15 100644 --- a/tools/environments/local.py +++ b/tools/environments/local.py @@ -513,6 +513,7 @@ class LocalEnvironment(BaseEnvironment): stderr=subprocess.STDOUT, stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL, preexec_fn=None if _IS_WINDOWS else os.setsid, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, cwd=_popen_cwd, ) if not _IS_WINDOWS: diff --git a/tools/process_registry.py b/tools/process_registry.py index 184939adf75..8429a71e087 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -557,6 +557,7 @@ class ProcessRegistry: stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=None if _IS_WINDOWS else os.setsid, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, ) session.process = proc From 6f50c26b2a0254275e8f79a30e8c950cece81ed5 Mon Sep 17 00:00:00 2001 From: lemassykoi <16377344+lemassykoi@users.noreply.github.com> Date: Sat, 16 May 2026 23:02:46 -0700 Subject: [PATCH 022/142] fix(model-switch): probe /models for custom providers without api_key The Telegram/Discord model picker skipped live model discovery for custom providers (llama.cpp, Ollama) unless an api_key was configured. Local providers typically don't require auth on the /models endpoint. The CLI always probes /models, so this brings the gateway picker into parity. Change: `if api_url and api_key:` -> `if api_url:` --- hermes_cli/model_switch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index fec1f33d092..a5d299165fc 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -1688,7 +1688,11 @@ def list_authenticated_providers( continue # Live model discovery from custom provider endpoints (matches # Section 3 behavior for user ``providers:`` entries). - if api_url and api_key: + # Also probes when no api_key is set (e.g. local llama.cpp / + # Ollama servers) — the /models endpoint often works without + # auth. The CLI's _model_flow_named_custom always probes, so + # the Telegram/Discord picker should do the same for parity. + if api_url: try: from hermes_cli.models import fetch_api_models From 6158964ff69c0c3ec0ee37fd5de5221b65ac7bcf Mon Sep 17 00:00:00 2001 From: draplater <6349758+draplater@users.noreply.github.com> Date: Sat, 16 May 2026 23:02:46 -0700 Subject: [PATCH 023/142] feat: inject current time into goal judge prompt The goal judge only receives the goal text and the agent's last response. It has no concept of the current time, making it impossible to evaluate time-sensitive goals like 'keep working until 5pm'. This commit adds 'Current time' to both JUDGE_USER_PROMPT_TEMPLATE and JUDGE_USER_PROMPT_WITH_SUBGOALS_TEMPLATE, computed from datetime.now().astimezone() at judge call time. --- hermes_cli/goals.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hermes_cli/goals.py b/hermes_cli/goals.py index 62ee00547c1..d6a139419a7 100644 --- a/hermes_cli/goals.py +++ b/hermes_cli/goals.py @@ -34,6 +34,7 @@ import logging import re import time from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple logger = logging.getLogger(__name__) @@ -110,6 +111,7 @@ JUDGE_SYSTEM_PROMPT = ( JUDGE_USER_PROMPT_TEMPLATE = ( "Goal:\n{goal}\n\n" "Agent's most recent response:\n{response}\n\n" + "Current time: {current_time}\n\n" "Is the goal satisfied?" ) @@ -120,6 +122,7 @@ JUDGE_USER_PROMPT_WITH_SUBGOALS_TEMPLATE = ( "Additional criteria the user added mid-loop (all must also be " "satisfied for the goal to be DONE):\n{subgoals_block}\n\n" "Agent's most recent response:\n{response}\n\n" + "Current time: {current_time}\n\n" "Decision: For each numbered criterion above, find concrete " "evidence in the agent's response that the criterion is " "satisfied. Do not accept generic phrases like 'all requirements " @@ -415,6 +418,7 @@ def judge_goal( # Build the prompt — pick the with-subgoals variant when applicable. clean_subgoals = [s.strip() for s in (subgoals or []) if s and s.strip()] + current_time = datetime.now(tz=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") if clean_subgoals: subgoals_block = "\n".join( f"- {i}. {text}" for i, text in enumerate(clean_subgoals, start=1) @@ -423,11 +427,13 @@ def judge_goal( goal=_truncate(goal, 2000), subgoals_block=_truncate(subgoals_block, 2000), response=_truncate(last_response, _JUDGE_RESPONSE_SNIPPET_CHARS), + current_time=current_time, ) else: prompt = JUDGE_USER_PROMPT_TEMPLATE.format( goal=_truncate(goal, 2000), response=_truncate(last_response, _JUDGE_RESPONSE_SNIPPET_CHARS), + current_time=current_time, ) try: From 7a7e78a360464b30ba9e9a20525681977b0f2095 Mon Sep 17 00:00:00 2001 From: pr7426 Date: Sun, 17 May 2026 02:15:45 +0800 Subject: [PATCH 024/142] fix(cron): prevent parallel job result loss on exception Replace generator-based result collection with explicit per-future handling. Each future is now processed independently with a 600s timeout. Before: _results.extend(f.result() for f in _futures) - One exception stops the generator, remaining results are lost - No timeout: one hung job blocks the entire tick After: as_completed() + per-future try/except - Each future handled independently - 600s timeout prevents indefinite blocking - Failed futures are logged and counted as failures --- cron/scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cron/scheduler.py b/cron/scheduler.py index d470e8c2c74..322fa64906f 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -1802,7 +1802,12 @@ def tick(verbose: bool = True, adapters=None, loop=None) -> int: for job in parallel_jobs: _ctx = contextvars.copy_context() _futures.append(_tick_pool.submit(_ctx.run, _process_job, job)) - _results.extend(f.result() for f in _futures) + for f in concurrent.futures.as_completed(_futures, timeout=600): + try: + _results.append(f.result()) + except Exception as exc: + logger.error("Parallel cron job future failed: %s", exc) + _results.append(False) # Best-effort sweep of MCP stdio subprocesses that survived their # session teardown during this tick. Runs AFTER every job has From a52f014a8cdefb72d81ca0e1d1208571dc3512d2 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 15 May 2026 13:45:07 +0530 Subject: [PATCH 025/142] fix(tests): mock keychain in TestReadClaudeCodeCredentials to prevent credential leakage Tests in TestReadClaudeCodeCredentials were not mocking _read_claude_code_credentials_from_keychain, which was added after the tests were written. On macOS machines with real Claude Code credentials stored in the Keychain, the function returns live credentials instead of the test fixtures, causing assertions to fail and leaking real tokens in test output. Add an autouse fixture that stubs the keychain reader to None so all tests in the class exercise only the file-based credential path. Co-Authored-By: Claude Sonnet 4.6 --- tests/agent/test_anthropic_adapter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/agent/test_anthropic_adapter.py b/tests/agent/test_anthropic_adapter.py index 0ba2ba29f51..259e9c1c523 100644 --- a/tests/agent/test_anthropic_adapter.py +++ b/tests/agent/test_anthropic_adapter.py @@ -157,6 +157,13 @@ class TestBuildAnthropicClient: class TestReadClaudeCodeCredentials: + @pytest.fixture(autouse=True) + def no_keychain(self, monkeypatch): + monkeypatch.setattr( + "agent.anthropic_adapter._read_claude_code_credentials_from_keychain", + lambda: None, + ) + def test_reads_valid_credentials(self, tmp_path, monkeypatch): cred_file = tmp_path / ".claude" / ".credentials.json" cred_file.parent.mkdir(parents=True) From 8973b00ff3665a76b69ca17e57c8cd1a39b32d53 Mon Sep 17 00:00:00 2001 From: flamiinngo Date: Sun, 17 May 2026 02:10:50 +0100 Subject: [PATCH 026/142] fix(scripts): fix UnicodeEncodeError in footgun checker on Windows The check-windows-footguns.py script outputs a checkmark (U+2713) and cross (U+2717) to report results. Windows terminals default to cp1252, which cannot encode these characters, so running the script on Windows threw a UnicodeEncodeError before any results were printed. This made the tool completely unusable on the exact platform it exists to help -- a developer on Windows trying to check their code for Windows-safety issues would just get a crash instead. Fix: reconfigure stdout and stderr to UTF-8 at the start of main(), before any output is produced. Verified on Windows 11 Home with Python 3.13 (terminal defaulting to cp1252). --- scripts/check-windows-footguns.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/check-windows-footguns.py b/scripts/check-windows-footguns.py index f424be90710..7ae7ca50c4e 100644 --- a/scripts/check-windows-footguns.py +++ b/scripts/check-windows-footguns.py @@ -551,6 +551,14 @@ def print_rules() -> None: def main(argv: list[str]) -> int: + # Windows terminals default to cp1252, which can't encode the ✓/✗ + # characters used in the output. Reconfigure streams to UTF-8 so the + # script works correctly on the very platform it is designed to help. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8") + args = parse_args(argv) if args.list: From 04bb30730a66ff17fe3dcb509d6fd572da3eb014 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:04:09 -0700 Subject: [PATCH 027/142] chore(release): AUTHOR_MAP entries for batch salvage group 3 contributors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds release-note attribution mappings for 9 contributors from group 3: - @darvsum (PR #26766) - @hueilau (PR #26498) - @Timur00Kh (PR #27114) - @Grogger (PR #27061) - @lemassykoi (PR #27042) - @draplater (PR #26707) - @pr7426 (PR #27048) - @therahul-yo (PR #26215) - @flamiinngo (PR #27205) #27154 dropped from this batch — already landed on main as 4e9cedcd4. --- scripts/release.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index 6bbc2ad4ae3..52da4c2f4b7 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1113,6 +1113,19 @@ AUTHOR_MAP = { "hermesagent26@gmail.com": "hermesagent26", # PR #26438 (kimi model-name reasoning pad) "276067471+hermesagent26@users.noreply.github.com": "hermesagent26", "71590782+kriscolab@users.noreply.github.com": "kriscolab", # PR #26926 (deepseek default_aux_model) + # batch salvage (May 2026 LHF run, group 3) + "darvsum@users.noreply.github.com": "darvsum", # PR #26766 (preserve discover_models in normalize) + "peter@Peters-Mac-mini.local": "hueilau", # PR #26498 (strip image parts for non-vision) + "33933019+hueilau@users.noreply.github.com": "hueilau", + "32297275+Timur00Kh@users.noreply.github.com": "Timur00Kh", # PR #27114 (telegram DM topic for synthetic events) + "al.bellemare@gmail.com": "Grogger", # PR #27061 (windows console flash suppress) + "clement@nousresearch.com": "lemassykoi", # PR #27042 (model-switch probe keyless providers) + "16377344+lemassykoi@users.noreply.github.com": "lemassykoi", + "draplater@icloud.com": "draplater", # PR #26707 (goal judge current time) + "6349758+draplater@users.noreply.github.com": "draplater", + "pr7426@users.noreply.github.com": "pr7426", # PR #27048 (cron parallel job loss) + "rahulnilvan43@gmail.com": "therahul-yo", # PR #26215 (mock keychain in tests) + "kingsleyemeka117@gmail.com": "flamiinngo", # PR #27205 (UnicodeEncodeError footgun checker) } From c1ae18ee815eba605c1b021e1b0b2a9c765b2d71 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sat, 16 May 2026 02:59:39 +0300 Subject: [PATCH 028/142] fix(gateway): add trust_env=True to aiohttp sessions in SMS, Slack, Teams, Google Chat adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aiohttp.ClientSession defaults to trust_env=False, which silently ignores HTTP_PROXY, HTTPS_PROXY, and ALL_PROXY environment variables. Users behind a corporate or network proxy cannot reach external APIs on any of these platforms — all outbound requests fail with connection errors. Symmetric with wecom.py (line 276), weixin.py (lines 1055/1268/1274), and matrix.py (no-proxy path) which already set this flag. Complements the open LINE fix (#26635) with the remaining gateway and plugin adapters. Changed: - gateway/platforms/sms.py: persistent Twilio session (connect) + fallback session (send) — both hit https://api.twilio.com - gateway/platforms/slack.py: ephemeral response_url POST session — hits https://hooks.slack.com/... callback URLs - plugins/platforms/teams/adapter.py: standalone send session — hits login.microsoftonline.com (token) + Bot Framework service URL - plugins/platforms/google_chat/adapter.py: standalone send session — hits https://chat.googleapis.com/v1/... WhatsApp sessions are excluded: they connect to http://127.0.0.1:{port} (local bridge) and must not be routed through a system proxy. --- gateway/platforms/slack.py | 2 +- gateway/platforms/sms.py | 2 ++ plugins/platforms/google_chat/adapter.py | 2 +- plugins/platforms/teams/adapter.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gateway/platforms/slack.py b/gateway/platforms/slack.py index 2116b569f96..5accfdb4108 100644 --- a/gateway/platforms/slack.py +++ b/gateway/platforms/slack.py @@ -482,7 +482,7 @@ class SlackAdapter(BasePlatformAdapter): "text": text, } try: - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(trust_env=True) as session: async with session.post( ctx["response_url"], json=payload, diff --git a/gateway/platforms/sms.py b/gateway/platforms/sms.py index 2cf7db69b74..9d9957d5ea1 100644 --- a/gateway/platforms/sms.py +++ b/gateway/platforms/sms.py @@ -128,6 +128,7 @@ class SmsAdapter(BasePlatformAdapter): await site.start() self._http_session = aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=30), + trust_env=True, ) self._running = True @@ -169,6 +170,7 @@ class SmsAdapter(BasePlatformAdapter): session = self._http_session or aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=30), + trust_env=True, ) try: for chunk in chunks: diff --git a/plugins/platforms/google_chat/adapter.py b/plugins/platforms/google_chat/adapter.py index d8777bf7101..1520d6664eb 100644 --- a/plugins/platforms/google_chat/adapter.py +++ b/plugins/platforms/google_chat/adapter.py @@ -3246,7 +3246,7 @@ async def _standalone_send( return {"error": "Google Chat standalone send: aiohttp not installed"} try: - async with _aiohttp.ClientSession(timeout=_aiohttp.ClientTimeout(total=30.0)) as session: + async with _aiohttp.ClientSession(timeout=_aiohttp.ClientTimeout(total=30.0), trust_env=True) as session: async with session.post( url, json=body, diff --git a/plugins/platforms/teams/adapter.py b/plugins/platforms/teams/adapter.py index c71baeb9d93..f8a1dc3d5b4 100644 --- a/plugins/platforms/teams/adapter.py +++ b/plugins/platforms/teams/adapter.py @@ -566,7 +566,7 @@ async def _standalone_send( # Per-request timeouts so a slow STS endpoint cannot starve the # subsequent activity POST of its budget. per_request_timeout = _aiohttp.ClientTimeout(total=15.0) - async with _aiohttp.ClientSession() as session: + async with _aiohttp.ClientSession(trust_env=True) as session: async with session.post( token_url, data={ From fdd455bc58b8708eb2c7e3e5d83efca3ec49e4a4 Mon Sep 17 00:00:00 2001 From: subtract0 <205509009+subtract0@users.noreply.github.com> Date: Sat, 16 May 2026 23:09:31 -0700 Subject: [PATCH 029/142] fix(gateway): avoid zsh status variable in update wrapper --- gateway/run.py | 6 +++++- tests/gateway/test_update_streaming.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gateway/run.py b/gateway/run.py index 56185190e26..81ce914b8ab 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -12837,7 +12837,11 @@ class GatewayRunner: update_cmd = ( f"PYTHONUNBUFFERED=1 {hermes_cmd_str} update --gateway" f" > {shlex.quote(str(output_path))} 2>&1; " - f"status=$?; printf '%s' \"$status\" > {shlex.quote(str(exit_code_path))}" + # Avoid `status=$?`: `status` is a read-only special parameter + # in zsh, and this command string is copied/reused in macOS/zsh + # operator wrappers. Keep the template zsh-safe even though this + # specific subprocess currently runs under bash. + f"rc=$?; printf '%s' \"$rc\" > {shlex.quote(str(exit_code_path))}" ) setsid_bin = shutil.which("setsid") if setsid_bin: diff --git a/tests/gateway/test_update_streaming.py b/tests/gateway/test_update_streaming.py index 932bd1b0579..eb0f0cfa890 100644 --- a/tests/gateway/test_update_streaming.py +++ b/tests/gateway/test_update_streaming.py @@ -237,6 +237,8 @@ class TestUpdateCommandGatewayFlag: cmd_string = call_args[-1] if isinstance(call_args, list) else str(call_args) assert "--gateway" in cmd_string assert "PYTHONUNBUFFERED" in cmd_string + assert "rc=$?" in cmd_string + assert "status=$?" not in cmd_string assert "stream progress" in result From 364a1dd290245093f76837c6074bb7d4fdc798c6 Mon Sep 17 00:00:00 2001 From: zwolniony <12735938+zwolniony@users.noreply.github.com> Date: Sat, 16 May 2026 23:09:31 -0700 Subject: [PATCH 030/142] Local: doctor uses x-goog-api-key for Google generativelanguage endpoint --- hermes_cli/doctor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 9d3b6e3c01a..07aaa2e38bc 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -1474,6 +1474,15 @@ def run_doctor(args): } if base_url_host_matches(base, "api.kimi.com"): headers["User-Agent"] = "claude-code/0.1.0" + # Google's Generative Language API (generativelanguage.googleapis.com) + # rejects ``Authorization: Bearer `` with 401 + # ``ACCESS_TOKEN_TYPE_UNSUPPORTED`` — that header is reserved for + # OAuth 2 access tokens, not plain API keys. Plain keys use + # ``x-goog-api-key`` (or ``?key=``). Without this, a perfectly valid + # GOOGLE_API_KEY/GEMINI_API_KEY always shows red in ``hermes doctor``. + if url and base_url_host_matches(url, "generativelanguage.googleapis.com"): + headers.pop("Authorization", None) + headers["x-goog-api-key"] = key r = httpx.get(url, headers=headers, timeout=10) if ( pname == "Alibaba/DashScope" From a3017508bf88e663c318495d191904020f77a0f5 Mon Sep 17 00:00:00 2001 From: Ambuj Kumar Date: Sat, 16 May 2026 02:23:25 +0530 Subject: [PATCH 031/142] fix(gateway): preserve underscores in plain-text identifiers --- gateway/platforms/helpers.py | 4 ++-- tests/gateway/test_bluebubbles.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/gateway/platforms/helpers.py b/gateway/platforms/helpers.py index 1c4f451585a..a3704bf50cf 100644 --- a/gateway/platforms/helpers.py +++ b/gateway/platforms/helpers.py @@ -168,8 +168,8 @@ class TextBatchAggregator: # Pre-compiled regexes for performance _RE_BOLD = re.compile(r"\*\*(.+?)\*\*", re.DOTALL) _RE_ITALIC_STAR = re.compile(r"\*(.+?)\*", re.DOTALL) -_RE_BOLD_UNDER = re.compile(r"__(.+?)__", re.DOTALL) -_RE_ITALIC_UNDER = re.compile(r"_(.+?)_", re.DOTALL) +_RE_BOLD_UNDER = re.compile(r"\b__(?![\s_])(.+?)(? Date: Thu, 14 May 2026 22:37:51 -0700 Subject: [PATCH 032/142] feat(gateway): extract auto-TTS markdown strip into prepare_tts_text() hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the inlined `re.sub(...)[:4000].strip()` cleanup at the auto-TTS site in `_process_message_background` into an overridable method `BasePlatformAdapter.prepare_tts_text(text: str) -> str`. The default implementation is byte-identical to the previous inline expression — strip `* _ \` # [ ] ( )` and truncate to 4000 chars — so every existing adapter (Telegram, Discord, Slack, Matrix, IRC, etc.) gets exactly the same behaviour as before. Zero behaviour change for any consumer that doesn't override the method. Why add the hook: voice-first platform adapters need stricter cleanup than text-bubble platforms. The default strips a handful of markdown sigils, which is fine when the output goes into a Discord embed or a Telegram message bubble — but read aloud by a TTS engine, URLs (`https://example.com/foo`), fenced code blocks, file paths (`/Users/x/foo.py`), and `MEDIA:` tags turn into long sequences of unintelligible characters. With this hook an adapter can drop those spans before TTS while leaving the data-channel transcript intact for visual rendering. Without the hook, voice adapters have to either - duplicate the auto-TTS flow inside their own `handle_response` pipeline, which means re-implementing the entire `extract_media`, `extract_images`, `extract_local_files`, attachment routing and error-handling sequence in `_process_message_background`, or - live with TTS speaking URLs character-by-character. Both are worse than a 7-line method addition. Example consumer: https://github.com/kortexa-ai/hermes-livekit — LiveKit WebRTC voice gateway plugin. Its `LiveKitAdapter.prepare_tts_text()` additionally strips fenced code blocks, inline code, URLs, file paths, and `MEDIA:` tags before TTS synthesis, while the full response still reaches connected clients via the data channel. Drop-in installable via `pip install git+https://github.com/kortexa-ai/hermes-livekit.git`. Carved out of #3894 (LiveKit WebRTC gateway PR) so the generic hook can land independently of the LiveKit platform itself. --- gateway/platforms/base.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 7b3147e21f4..96b56d29cc7 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -2014,6 +2014,13 @@ class BasePlatformAdapter(ABC): text = f"{caption}\n{text}" return await self.send(chat_id=chat_id, content=text, reply_to=reply_to, metadata=metadata) + def prepare_tts_text(self, text: str) -> str: + """Prepare text for TTS. Override to filter tool output, code, etc. + + Default strips markdown formatting and truncates to 4000 chars. + """ + return re.sub(r'[*_`#\[\]()]', '', text)[:4000].strip() + async def play_tts( self, chat_id: str, @@ -3144,7 +3151,7 @@ class BasePlatformAdapter(ABC): from tools.tts_tool import text_to_speech_tool, check_tts_requirements if check_tts_requirements(): import json as _json - speech_text = re.sub(r'[*_`#\[\]()]', '', text_content)[:4000].strip() + speech_text = self.prepare_tts_text(text_content) if not speech_text: raise ValueError("Empty text after markdown cleanup") tts_result_str = await asyncio.to_thread( From b389796ae3a33256ff1b4077acc1169831fb63e1 Mon Sep 17 00:00:00 2001 From: zccyman Date: Thu, 14 May 2026 07:49:52 +0800 Subject: [PATCH 033/142] fix(auxiliary): resolve api_key_env alias in named custom provider path of resolve_provider_client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In resolve_provider_client(), the named custom provider code path at ~line 2914 only checked the ``key_env`` field when looking for an environment-variable-based API key. The documented ``api_key_env`` snake_case alias was silently ignored, causing custom providers configured with ``api_key_env`` to fall through to the ``no-key-required`` placeholder — which produces a confusing 401 (``****ired`` mask) on auth-required remote endpoints. This mirrors the same fix already applied to run_agent.py in commit 6ddc48b05 (fix(fallback): resolve api_key_env in fallback chain entries). Also adds a logger.warning() when the placeholder is reached, so future alias gaps are easier to debug. Closes #25091 --- agent/auxiliary_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index cfc44e5f2a6..102ff79f1ce 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -3049,10 +3049,17 @@ def resolve_provider_client( if custom_entry: custom_base = custom_entry.get("base_url", "").strip() custom_key = custom_entry.get("api_key", "").strip() - custom_key_env = custom_entry.get("key_env", "").strip() + custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip() if not custom_key and custom_key_env: custom_key = os.getenv(custom_key_env, "").strip() custom_key = custom_key or "no-key-required" + if custom_key == "no-key-required": + logger.warning( + "resolve_provider_client: named custom provider %r has no resolvable " + "api_key — request will be sent with placeholder no-key-required " + "and will 401 on auth-required endpoints", + custom_entry.get("name") or provider, + ) # An explicit per-task api_mode override (from _resolve_task_provider_model) # wins; otherwise fall back to what the provider entry declared. entry_api_mode = (api_mode or custom_entry.get("api_mode") or "").strip() From 5631345b12aa5fa7ead11203624e646b42c8936f Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 16 May 2026 16:41:03 +0900 Subject: [PATCH 034/142] [agent] fix: harden api server response headers --- gateway/platforms/api_server.py | 5 +++++ tests/gateway/test_api_server.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index ebd4f014690..0668896e170 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -510,7 +510,12 @@ else: body_limit_middleware = None # type: ignore[assignment] _SECURITY_HEADERS = { + "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", + "Permissions-Policy": "camera=(), microphone=(), geolocation=()", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "X-XSS-Protection": "0", "Referrer-Policy": "no-referrer", } diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 7d08d64bb32..aae5f550532 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -445,7 +445,12 @@ class TestHealthEndpoint: async with TestClient(TestServer(app)) as cli: resp = await cli.get("/health") assert resp.status == 200 + assert resp.headers.get("Content-Security-Policy") == "default-src 'none'; frame-ancestors 'none'" + assert resp.headers.get("Permissions-Policy") == "camera=(), microphone=(), geolocation=()" + assert resp.headers.get("Strict-Transport-Security") == "max-age=31536000; includeSubDomains" assert resp.headers.get("X-Content-Type-Options") == "nosniff" + assert resp.headers.get("X-Frame-Options") == "DENY" + assert resp.headers.get("X-XSS-Protection") == "0" assert resp.headers.get("Referrer-Policy") == "no-referrer" @pytest.mark.asyncio From 52c89715a29198d838dac54e229aba9cf328e408 Mon Sep 17 00:00:00 2001 From: phoenixshen <1594534+phoenixshen@users.noreply.github.com> Date: Sat, 16 May 2026 23:09:31 -0700 Subject: [PATCH 035/142] fix: respect user-configured vision model for OpenRouter _OPENROUTER_MODEL hardcoded 'google/gemini-3-flash-preview' which returns 404 on OpenRouter, breaking all vision tasks for users who rely on the OpenRouter default. Additionally, _try_openrouter() ignored the user-configured auxiliary.vision.model entirely. Changes: - Update _OPENROUTER_MODEL default to google/gemini-2.5-flash (valid) - Add optional 'model' parameter to _try_openrouter() - Pass configured model from _resolve_strict_vision_backend() through to _try_openrouter() This allows users who set auxiliary.vision.model (e.g. x-ai/grok-4.3) to have it actually used, while maintaining backward compatibility. --- agent/auxiliary_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 102ff79f1ce..e02fa1911f7 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -424,7 +424,7 @@ NOUS_EXTRA_BODY = _nous_extra_body() auxiliary_is_nous: bool = False # Default auxiliary models per provider -_OPENROUTER_MODEL = "google/gemini-3-flash-preview" +_OPENROUTER_MODEL = "google/gemini-2.5-flash" _NOUS_MODEL = "google/gemini-3-flash-preview" _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1" _ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com" @@ -1473,7 +1473,7 @@ def _resolve_api_key_provider() -> Tuple[Optional[OpenAI], Optional[str]]: -def _try_openrouter(explicit_api_key: str = None) -> Tuple[Optional[OpenAI], Optional[str]]: +def _try_openrouter(explicit_api_key: str = None, model: str = None) -> Tuple[Optional[OpenAI], Optional[str]]: pool_present, entry = _select_pool_entry("openrouter") if pool_present: or_key = explicit_api_key or _pool_runtime_api_key(entry) @@ -1483,7 +1483,7 @@ def _try_openrouter(explicit_api_key: str = None) -> Tuple[Optional[OpenAI], Opt base_url = _pool_runtime_base_url(entry, OPENROUTER_BASE_URL) or OPENROUTER_BASE_URL logger.debug("Auxiliary client: OpenRouter via pool") return OpenAI(api_key=or_key, base_url=base_url, - default_headers=build_or_headers()), _OPENROUTER_MODEL + default_headers=build_or_headers()), model or _OPENROUTER_MODEL or_key = explicit_api_key or os.getenv("OPENROUTER_API_KEY") if not or_key: @@ -1491,7 +1491,7 @@ def _try_openrouter(explicit_api_key: str = None) -> Tuple[Optional[OpenAI], Opt return None, None logger.debug("Auxiliary client: OpenRouter") return OpenAI(api_key=or_key, base_url=OPENROUTER_BASE_URL, - default_headers=build_or_headers()), _OPENROUTER_MODEL + default_headers=build_or_headers()), model or _OPENROUTER_MODEL def _describe_openrouter_unavailable() -> str: @@ -3407,7 +3407,7 @@ def _resolve_strict_vision_backend( if provider == "copilot": return resolve_provider_client("copilot", model, is_vision=True) if provider == "openrouter": - return _try_openrouter() + return _try_openrouter(model=model) if provider == "nous": return _try_nous(vision=True) if provider == "openai-codex": From 35b7befc67315da5d4ce6b6a3daa4d9ba2f57c1c Mon Sep 17 00:00:00 2001 From: AhmetArif0 <147827411+AhmetArif0@users.noreply.github.com> Date: Sat, 16 May 2026 02:06:31 +0300 Subject: [PATCH 036/142] fix(line): add trust_env=True to all _LineClient aiohttp sessions _LineClient's five aiohttp.ClientSession() calls omit trust_env=True, silently bypassing HTTP_PROXY / HTTPS_PROXY / ALL_PROXY. Result: every LINE API call (reply, push, loading, fetch_content, get_bot_user_id) ignores the system proxy. Fix: add trust_env=True to all five session constructions. Symmetric with the wecom and weixin adapters which already set this flag. No behavior change for users not behind a proxy. --- plugins/platforms/line/adapter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index db5d3564d32..907f16be4ff 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -447,7 +447,7 @@ class _LineClient: async def reply(self, reply_token: str, messages: List[Dict[str, Any]]) -> None: import aiohttp timeout = aiohttp.ClientTimeout(total=self._timeout) - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.post( LINE_REPLY_URL, headers=self._headers, @@ -460,7 +460,7 @@ class _LineClient: async def push(self, chat_id: str, messages: List[Dict[str, Any]]) -> None: import aiohttp timeout = aiohttp.ClientTimeout(total=self._timeout) - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.post( LINE_PUSH_URL, headers=self._headers, @@ -479,7 +479,7 @@ class _LineClient: clamped = max(5, min(60, (seconds // 5) * 5 or 5)) try: timeout = aiohttp.ClientTimeout(total=5.0) - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: await session.post( LINE_LOADING_URL, headers=self._headers, @@ -493,7 +493,7 @@ class _LineClient: import aiohttp url = LINE_CONTENT_URL_FMT.format(message_id=message_id) timeout = aiohttp.ClientTimeout(total=30.0) - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.get(url, headers={"Authorization": f"Bearer {self._token}"}) as resp: if resp.status >= 400: raise RuntimeError(f"LINE content {resp.status}") @@ -504,7 +504,7 @@ class _LineClient: import aiohttp timeout = aiohttp.ClientTimeout(total=10.0) try: - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.get(LINE_BOT_INFO_URL, headers=self._headers) as resp: if resp.status >= 400: return None From 7322816efa601737722c74147194f1f5ffd3ad07 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:10:34 -0700 Subject: [PATCH 037/142] chore(release): AUTHOR_MAP entries for batch salvage group 4 contributors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds release-note attribution mappings for 9 contributors from group 4: - @EloquentBrush0x (PR #26657) - @subtract0 (PR #25658) - @zwolniony (PR #26961) - @that-ambuj (PR #26582) - @zccyman (PR #25294) - @lidge-jun (PR #26814) - @phoenixshen (PR #26768) - @AhmetArif0 (PR #26635) - (francip already mapped from prior PR #26134 attribution) #27147 dropped from this batch — already landed on main as 4b17c2411. --- scripts/release.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index 52da4c2f4b7..c388116cff6 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1126,6 +1126,17 @@ AUTHOR_MAP = { "pr7426@users.noreply.github.com": "pr7426", # PR #27048 (cron parallel job loss) "rahulnilvan43@gmail.com": "therahul-yo", # PR #26215 (mock keychain in tests) "kingsleyemeka117@gmail.com": "flamiinngo", # PR #27205 (UnicodeEncodeError footgun checker) + # batch salvage (May 2026 LHF run, group 4) + "283442588+EloquentBrush0x@users.noreply.github.com": "EloquentBrush0x", # PR #26657 (trust_env aiohttp) + "205509009+subtract0@users.noreply.github.com": "subtract0", # PR #25658 (zsh $status -> $rc) + "patryk@jarmakowicz.me": "zwolniony", # PR #26961 (gemini x-goog-api-key) + "12735938+zwolniony@users.noreply.github.com": "zwolniony", + "ambuj@dodopayments.com": "that-ambuj", # PR #26582 (preserve underscores) + "zccyman@163.com": "zccyman", # PR #25294 (custom provider api_key_env alias) + "bitkyc08@gmail.com": "lidge-jun", # PR #26814 (api server browser security headers) + "sp_ps@Mac-mini.lan": "phoenixshen", # PR #26768 (respect user-configured vision model) + "1594534+phoenixshen@users.noreply.github.com": "phoenixshen", + "147827411+AhmetArif0@users.noreply.github.com": "AhmetArif0", # PR #26635 (line proxy env vars) } From 8f3bc17db9ebe1d9108ae69b14fcc3f06734554b Mon Sep 17 00:00:00 2001 From: Matthew Lai Date: Thu, 14 May 2026 00:43:49 +0100 Subject: [PATCH 038/142] feat(agent): Added gemma 4 to reasoning allowlist (cherry picked from commit 7244116b687f6e5ff5e869c99cdbb1b09c822799) --- run_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/run_agent.py b/run_agent.py index 8b68dee6857..3afbd584529 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3338,6 +3338,7 @@ class AIAgent: "openai/", "x-ai/", "google/gemini-2", + "google/gemma-4", "qwen/qwen3", "tencent/hy3-preview", "xiaomi/", From 94b3131be7115709c516a79be7c3d01dd71761a8 Mon Sep 17 00:00:00 2001 From: hermesagent26 <276067471+hermesagent26@users.noreply.github.com> Date: Sat, 16 May 2026 22:51:17 -0700 Subject: [PATCH 039/142] fix(run_agent): detect kimi models via model name for reasoning pad MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit previously only checked provider ID and base URL. When kimi-k2.6 is served via ollama-cloud (or any third-party provider), provider is not 'kimi-coding' and base URL is not api.kimi.com — so reasoning_content pad was never injected. This caused HTTP 400 from Ollama Cloud's Go backend: 'invalid message content type: map[string]interface {}'. Fix: add model-name detection ('kimi' in model.lower()) so any route serving a kimi model gets the required reasoning_content echo-back. Refs the 400/401 Telegram errors where kimi-k2.6 via ollama-cloud consistently failed after tool-call turns. (cherry picked from commit 9a9f8a6d9945c9bf3118c557f85ad1956de4f553) --- run_agent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run_agent.py b/run_agent.py index 3afbd584529..31677ff73fe 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3448,12 +3448,16 @@ class AIAgent: Kimi ``/coding`` and Moonshot thinking mode both require ``reasoning_content`` on every assistant tool-call message; omitting it causes the next replay to fail with HTTP 400. + + Also detects Kimi models served through third-party providers (e.g. + ollama-cloud) by matching ``kimi`` in the model name. """ return ( self.provider in {"kimi-coding", "kimi-coding-cn"} or base_url_host_matches(self.base_url, "api.kimi.com") or base_url_host_matches(self.base_url, "moonshot.ai") or base_url_host_matches(self.base_url, "moonshot.cn") + or "kimi" in (self.model or "").lower() ) def _needs_deepseek_tool_reasoning(self) -> bool: From a77ca9295e96af9f0da522b1bd3afe1965ef21ee Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:20:27 -0700 Subject: [PATCH 040/142] perf(run_agent): accumulate length-continuation prefix via list+join Original commit 4f8aaf104 by InB4DevOps targeted run_conversation() in the pre-refactor run_agent.py. Re-applied to the extracted location in agent/conversation_loop.py. Co-authored-by: InB4DevOps --- agent/conversation_loop.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index c95f1b63385..6fd9a6aec1d 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -464,7 +464,7 @@ def run_conversation( codex_ack_continuations = 0 length_continue_retries = 0 truncated_tool_call_retries = 0 - truncated_response_prefix = "" + truncated_response_parts: List[str] = [] compression_attempts = 0 _turn_exit_reason = "unknown" # Diagnostic: why the loop ended @@ -1357,7 +1357,7 @@ def run_conversation( interim_msg = agent._build_assistant_message(assistant_message, finish_reason) messages.append(interim_msg) if assistant_message.content: - truncated_response_prefix += assistant_message.content + truncated_response_parts.append(assistant_message.content) if length_continue_retries < 3: agent._vprint( @@ -1378,7 +1378,7 @@ def run_conversation( restart_with_length_continuation = True break - partial_response = agent._strip_think_blocks(truncated_response_prefix).strip() + partial_response = agent._strip_think_blocks("".join(truncated_response_parts)).strip() agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) return { @@ -3582,9 +3582,9 @@ def run_conversation( codex_ack_continuations = 0 - if truncated_response_prefix: - final_response = truncated_response_prefix + final_response - truncated_response_prefix = "" + if truncated_response_parts: + final_response = "".join(truncated_response_parts) + final_response + truncated_response_parts = [] length_continue_retries = 0 final_response = agent._strip_think_blocks(final_response).strip() From 7d221aa1f288a96a11485845923266499f5a3abb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:21:51 -0700 Subject: [PATCH 041/142] =?UTF-8?q?fix(langfuse):=20complete=20observabili?= =?UTF-8?q?ty=20fix=20=E2=80=94=20port=20to=20extracted=20conversation=5Fl?= =?UTF-8?q?oop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit db84a78e6 by kshitij targeted run_conversation()'s pre_api_request and post_api_request hooks in pre-refactor run_agent.py. Re-applied to the extracted location in agent/conversation_loop.py. Co-authored-by: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Co-authored-by: xxxigm Co-authored-by: Brian Conklin --- agent/conversation_loop.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 6fd9a6aec1d..51ae06e9900 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -925,16 +925,30 @@ def run_conversation( try: from hermes_cli.plugins import invoke_hook as _invoke_hook + request_messages = api_kwargs.get("messages") + if not isinstance(request_messages, list): + request_messages = api_kwargs.get("input") + if not isinstance(request_messages, list): + request_messages = api_messages + # Shallow-copy the outer list so plugins that retain the + # reference for async snapshotting don't observe later + # mutations of api_messages. The inner dicts are not + # mutated by the agent loop, so a shallow copy is + # sufficient; a deepcopy would walk every tool result + # and base64 image on every API call. _invoke_hook( "pre_api_request", task_id=effective_task_id, session_id=agent.session_id or "", + user_message=original_user_message, + conversation_history=list(messages), platform=agent.platform or "", model=agent.model, provider=agent.provider, base_url=agent.base_url, api_mode=agent.api_mode, api_call_count=api_call_count, + request_messages=list(request_messages) if isinstance(request_messages, list) else [], message_count=len(api_messages), tool_count=len(agent.tools or []), approx_input_tokens=approx_tokens, @@ -2839,7 +2853,9 @@ def run_conversation( finish_reason=finish_reason, message_count=len(api_messages), response_model=getattr(response, "model", None), + response=response, usage=agent._usage_summary_for_api_request_hook(response), + assistant_message=assistant_message, assistant_content_chars=len(_assistant_text), assistant_tool_call_count=len(_assistant_tool_calls), ) From b07524e53aed5e8629b98ce3bbf3a54a27d596f4 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:23:38 -0700 Subject: [PATCH 042/142] =?UTF-8?q?feat(xai-oauth):=20add=20xAI=20Grok=20O?= =?UTF-8?q?Auth=20(SuperGrok=20Subscription)=20provider=20=E2=80=94=20port?= =?UTF-8?q?=20to=20extracted=20modules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit b62c99797 by Jaaneek targeted six locations in pre-refactor run_agent.py. Re-applied to the extracted post-PR locations: - api_mode dispatch → agent/agent_init.py - is_xai_responses build_api_kwargs → agent/chat_completion_helpers.py - codex_auth_retry block + 401 hint → agent/conversation_loop.py - _try_refresh_codex_client_credentials body → run_agent.py (kept) The non-run_agent.py portions of the commit (auxiliary_client, codex transport, hermes_cli/auth, tools/xai_http, tests, docs) merged cleanly from main via the prior merge commit. Co-authored-by: Jaaneek --- agent/agent_init.py | 2 +- agent/chat_completion_helpers.py | 2 +- agent/conversation_loop.py | 19 +++++++---- run_agent.py | 55 +++++++++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/agent/agent_init.py b/agent/agent_init.py index acae61487c4..6b85f9ef1df 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -228,7 +228,7 @@ def init_agent( agent.api_mode = api_mode elif agent.provider == "openai-codex": agent.api_mode = "codex_responses" - elif agent.provider == "xai": + elif agent.provider in {"xai", "xai-oauth"}: agent.api_mode = "codex_responses" elif (provider_name is None) and ( agent._base_url_hostname == "chatgpt.com" diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 9616fefe0e4..d163557b8fa 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -284,7 +284,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: and "/backend-api/codex" in agent._base_url_lower ) ) - is_xai_responses = agent.provider == "xai" or agent._base_url_hostname == "api.x.ai" + is_xai_responses = agent.provider in {"xai", "xai-oauth"} or agent._base_url_hostname == "api.x.ai" _msgs_for_codex = agent._prepare_messages_for_non_vision_model(api_messages) return _ct.build_kwargs( model=agent.model, diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 51ae06e9900..e121c4b2a74 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -1957,13 +1957,14 @@ def run_conversation( if ( agent.api_mode == "codex_responses" - and agent.provider == "openai-codex" + and agent.provider in {"openai-codex", "xai-oauth"} and status_code == 401 and not codex_auth_retry_attempted ): codex_auth_retry_attempted = True if agent._try_refresh_codex_client_credentials(force=True): - agent._vprint(f"{agent.log_prefix}🔐 Codex auth refreshed after 401. Retrying request...") + _label = "xAI OAuth" if agent.provider == "xai-oauth" else "Codex" + agent._vprint(f"{agent.log_prefix}🔐 {_label} auth refreshed after 401. Retrying request...") continue if ( agent.api_mode == "chat_completions" @@ -2603,11 +2604,15 @@ def run_conversation( agent._vprint(f"{agent.log_prefix} 🌐 Endpoint: {_base}", force=True) # Actionable guidance for common auth errors if classified.is_auth or classified.reason == FailoverReason.billing: - if _provider == "openai-codex" and status_code == 401: - agent._vprint(f"{agent.log_prefix} 💡 Codex OAuth token was rejected (HTTP 401). Your token may have been", force=True) - agent._vprint(f"{agent.log_prefix} refreshed by another client (Codex CLI, VS Code). To fix:", force=True) - agent._vprint(f"{agent.log_prefix} 1. Run `codex` in your terminal to generate fresh tokens.", force=True) - agent._vprint(f"{agent.log_prefix} 2. Then run `hermes auth` to re-authenticate.", force=True) + if _provider in {"openai-codex", "xai-oauth"} and status_code == 401: + if _provider == "openai-codex": + agent._vprint(f"{agent.log_prefix} 💡 Codex OAuth token was rejected (HTTP 401). Your token may have been", force=True) + agent._vprint(f"{agent.log_prefix} refreshed by another client (Codex CLI, VS Code). To fix:", force=True) + agent._vprint(f"{agent.log_prefix} 1. Run `codex` in your terminal to generate fresh tokens.", force=True) + agent._vprint(f"{agent.log_prefix} 2. Then run `hermes auth` to re-authenticate.", force=True) + else: + agent._vprint(f"{agent.log_prefix} 💡 xAI OAuth token was rejected (HTTP 401). To fix:", force=True) + agent._vprint(f"{agent.log_prefix} re-authenticate with xAI Grok OAuth (SuperGrok Subscription) from `hermes model`.", force=True) else: agent._vprint(f"{agent.log_prefix} 💡 Your API key was rejected by the provider. Check:", force=True) agent._vprint(f"{agent.log_prefix} • Is the key valid? Run: hermes setup", force=True) diff --git a/run_agent.py b/run_agent.py index 31677ff73fe..c976eba9690 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2449,15 +2449,60 @@ class AIAgent: return run_codex_create_stream_fallback(self, api_kwargs, client) def _try_refresh_codex_client_credentials(self, *, force: bool = True) -> bool: - if self.api_mode != "codex_responses" or self.provider != "openai-codex": + if self.api_mode != "codex_responses" or self.provider not in {"openai-codex", "xai-oauth"}: + return False + + # Guard against silent account swap. + # + # When an agent is using a non-singleton credential — e.g. a manual + # pool entry (``hermes auth add xai-oauth``) whose tokens belong to + # a different account than the loopback_pkce singleton, or an agent + # constructed with an explicit ``api_key=`` arg — force-refreshing + # the singleton here and adopting its tokens silently re-routes the + # rest of the conversation onto the singleton's account. The + # credential pool's reactive recovery (``_recover_with_credential_pool``) + # is the right channel for that case; this path is the + # singleton-only fallback used when the pool can't recover, and + # MUST only fire when the agent really is on singleton tokens. + try: + if self.provider == "openai-codex": + from hermes_cli.auth import resolve_codex_runtime_credentials + + singleton_now = resolve_codex_runtime_credentials( + refresh_if_expiring=False, + ) + else: + from hermes_cli.auth import resolve_xai_oauth_runtime_credentials + + singleton_now = resolve_xai_oauth_runtime_credentials( + refresh_if_expiring=False, + ) + except Exception as exc: + logger.debug("%s singleton read failed: %s", self.provider, exc) + return False + + singleton_key = str(singleton_now.get("api_key") or "").strip() + active_key = str(self.api_key or "").strip() + if singleton_key and active_key and singleton_key != active_key: + logger.debug( + "%s singleton tokens differ from the active api_key; " + "skipping singleton force-refresh to avoid silent account swap. " + "Reactive credential rotation should go through the pool.", + self.provider, + ) return False try: - from hermes_cli.auth import resolve_codex_runtime_credentials + if self.provider == "openai-codex": + from hermes_cli.auth import resolve_codex_runtime_credentials - creds = resolve_codex_runtime_credentials(force_refresh=force) + creds = resolve_codex_runtime_credentials(force_refresh=force) + else: + from hermes_cli.auth import resolve_xai_oauth_runtime_credentials + + creds = resolve_xai_oauth_runtime_credentials(force_refresh=force) except Exception as exc: - logger.debug("Codex credential refresh failed: %s", exc) + logger.debug("%s credential refresh failed: %s", self.provider, exc) return False api_key = creds.get("api_key") @@ -2472,7 +2517,7 @@ class AIAgent: self._client_kwargs["api_key"] = self.api_key self._client_kwargs["base_url"] = self.base_url - if not self._replace_primary_openai_client(reason="codex_credential_refresh"): + if not self._replace_primary_openai_client(reason=f"{self.provider}_credential_refresh"): return False return True From 27df249564b3ce6fa4d1db883df0329bfda01593 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:25:11 -0700 Subject: [PATCH 043/142] =?UTF-8?q?feat(nvidia):=20add=20NIM=20billing=20o?= =?UTF-8?q?rigin=20header=20=E2=80=94=20port=20to=20extracted=20modules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 13c3d4b4e by kchantharuan touched __init__ and _apply_client_headers_for_base_url in pre-refactor run_agent.py. Re-applied to: - __init__: agent/agent_init.py (3 hunks — NVIDIA branch + _custom_headers fallback in routed-client and fallback-client paths) - _apply_client_headers_for_base_url: still in run_agent.py (1 hunk) build_nvidia_nim_headers was already present in agent/auxiliary_client.py from the prior merge — no additional port needed. Co-authored-by: kchantharuan --- agent/agent_init.py | 22 +++++++++++++++++----- run_agent.py | 8 +++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/agent/agent_init.py b/agent/agent_init.py index 6b85f9ef1df..d5798f163ff 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -617,6 +617,9 @@ def init_agent( if base_url_host_matches(effective_base, "openrouter.ai"): from agent.auxiliary_client import build_or_headers client_kwargs["default_headers"] = build_or_headers() + elif base_url_host_matches(effective_base, "integrate.api.nvidia.com"): + from agent.auxiliary_client import build_nvidia_nim_headers + client_kwargs["default_headers"] = build_nvidia_nim_headers(effective_base) elif base_url_host_matches(effective_base, "api.routermint.com"): client_kwargs["default_headers"] = _ra()._routermint_headers() elif base_url_host_matches(effective_base, "api.githubcopilot.com"): @@ -655,9 +658,15 @@ def init_agent( } if _provider_timeout is not None: client_kwargs["timeout"] = _provider_timeout - # Preserve any default_headers the router set - if hasattr(_routed_client, '_default_headers') and _routed_client._default_headers: - client_kwargs["default_headers"] = dict(_routed_client._default_headers) + # Preserve provider-specific headers the router set. The + # OpenAI SDK stores caller-provided default_headers in + # _custom_headers; older/mocked clients may expose + # _default_headers instead. + _routed_headers = getattr(_routed_client, "_custom_headers", None) + if not _routed_headers: + _routed_headers = getattr(_routed_client, "_default_headers", None) + if _routed_headers: + client_kwargs["default_headers"] = dict(_routed_headers) else: # When the user explicitly chose a non-OpenRouter provider # but no credentials were found, fail fast with a clear @@ -706,8 +715,11 @@ def init_agent( } if _provider_timeout is not None: client_kwargs["timeout"] = _provider_timeout - if hasattr(_fb_client, "_default_headers") and _fb_client._default_headers: - client_kwargs["default_headers"] = dict(_fb_client._default_headers) + _fb_headers = getattr(_fb_client, "_custom_headers", None) + if not _fb_headers: + _fb_headers = getattr(_fb_client, "_default_headers", None) + if _fb_headers: + client_kwargs["default_headers"] = dict(_fb_headers) _fb_resolved = True break if not _fb_resolved: diff --git a/run_agent.py b/run_agent.py index c976eba9690..f4157807e04 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2644,12 +2644,18 @@ class AIAgent: return True def _apply_client_headers_for_base_url(self, base_url: str) -> None: - from agent.auxiliary_client import _AI_GATEWAY_HEADERS, build_or_headers + from agent.auxiliary_client import ( + _AI_GATEWAY_HEADERS, + build_nvidia_nim_headers, + build_or_headers, + ) if base_url_host_matches(base_url, "openrouter.ai"): self._client_kwargs["default_headers"] = build_or_headers() elif base_url_host_matches(base_url, "ai-gateway.vercel.sh"): self._client_kwargs["default_headers"] = dict(_AI_GATEWAY_HEADERS) + elif base_url_host_matches(base_url, "integrate.api.nvidia.com"): + self._client_kwargs["default_headers"] = build_nvidia_nim_headers(base_url) elif base_url_host_matches(base_url, "api.routermint.com"): self._client_kwargs["default_headers"] = _routermint_headers() elif base_url_host_matches(base_url, "api.githubcopilot.com"): From 6362e71973c18b407651157f818e279122ce41f6 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:28:05 -0700 Subject: [PATCH 044/142] fix(xai-oauth): recover from prelude SSE errors, gate reasoning replay, surface entitlement 403s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 31ba2b0cb by Teknium targeted run_codex_stream() at its pre-refactor location in run_agent.py. Re-applied: - Prelude error retry/fallback → agent/codex_runtime.py (in run_codex_stream where the body now lives) - _decorate_xai_entitlement_error helper + _summarize_api_error wrapping → run_agent.py (these methods remained on AIAgent as @staticmethod's; cherry-pick applied them cleanly) The xai-oauth provider gate, encrypted_content drop on replay, etc. landed in agent/codex_responses_adapter.py via the prior merge from main. Closes #8133, #14634 Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/codex_runtime.py | 38 +++++++++++++++++++++++++++++++++---- run_agent.py | 43 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index b2e9b714586..547fbb9ce07 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -284,18 +284,48 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta except RuntimeError as exc: err_text = str(exc) missing_completed = "response.completed" in err_text - if missing_completed and attempt < max_stream_retries: + # The OpenAI SDK's Responses streaming state machine raises + # ``RuntimeError("Expected to have received `response.created` + # before ``")`` when the first SSE event from the + # server is anything other than ``response.created`` — and it + # discards the event's payload before we can read it. Three + # real-world backends emit a different first frame: + # + # * xAI on grok-4.x OAuth — sends ``error`` (issues + # reported around the May 2026 SuperGrok rollout when + # multi-turn conversations replay encrypted reasoning + # content the OAuth tier rejects) + # * codex-lb relays — send ``codex.rate_limits`` (#14634) + # * custom Responses relays — send ``response.in_progress`` + # (#8133) + # + # In all three cases the underlying byte stream is still + # readable: a non-stream ``responses.create(stream=True)`` + # fallback succeeds and surfaces the real provider error as + # a normal exception with body+status_code attached, which + # ``_summarize_api_error`` can then translate into a useful + # user-facing line. Treat ``response.created`` prelude + # errors the same way we already treat ``response.completed`` + # postlude errors. + prelude_error = ( + "Expected to have received `response.created`" in err_text + or "Expected to have received \"response.created\"" in err_text + ) + if (missing_completed or prelude_error) and attempt < max_stream_retries: logger.debug( - "Responses stream closed before completion (attempt %s/%s); retrying. %s", + "Responses stream %s (attempt %s/%s); retrying. %s", + "prelude rejected" if prelude_error else "closed before completion", attempt + 1, max_stream_retries + 1, agent._client_log_context(), ) continue - if missing_completed: + if missing_completed or prelude_error: logger.debug( - "Responses stream did not emit response.completed; falling back to create(stream=True). %s", + "Responses stream %s; falling back to create(stream=True). %s err=%s", + "rejected before response.created" if prelude_error else "did not emit response.completed", agent._client_log_context(), + err_text, ) return agent._run_codex_create_stream_fallback(api_kwargs, client=active_client) raise diff --git a/run_agent.py b/run_agent.py index f4157807e04..80577a19be3 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1287,6 +1287,45 @@ class AIAgent: trajectory = self._convert_to_trajectory_format(messages, user_query, completed) _save_trajectory_to_file(trajectory, self.model, completed) + @staticmethod + def _decorate_xai_entitlement_error(detail: str) -> str: + """Append a friendly hint when xAI's OAuth surface returns an + entitlement-shaped error. + + xAI's ``/v1/responses`` endpoint replies to OAuth tokens that lack a + SuperGrok / X Premium subscription with HTTP 403 carrying a body like:: + + {"code": "The caller does not have permission to execute the + specified operation", "error": "You have either run out of + available resources or do not have an active Grok subscription. + Manage subscriptions at https://grok.com/..."} + + The raw text is useful but the action the user needs to take (subscribe + on grok.com, or switch providers with ``/model``) isn't obvious from + the wire format. Detect the entitlement shape and append a hint. + + Matched once per detail string — won't double-decorate if the upstream + already concatenated the same text. + """ + if not detail: + return detail + lower = detail.lower() + is_entitlement = ( + "do not have an active grok subscription" in lower + or ("out of available resources" in lower and "grok" in lower) + or ("does not have permission" in lower and "grok" in lower) + ) + if not is_entitlement: + return detail + hint = ( + " — xAI OAuth account lacks SuperGrok / X Premium entitlement for " + "this model. Subscribe at https://grok.com or run `/model` to " + "switch providers." + ) + if hint.strip() in detail: + return detail + return f"{detail}{hint}" + @staticmethod def _summarize_api_error(error: Exception) -> str: """Extract a human-readable one-liner from an API error. @@ -1320,12 +1359,12 @@ class AIAgent: if msg: status_code = getattr(error, "status_code", None) prefix = f"HTTP {status_code}: " if status_code else "" - return f"{prefix}{msg[:300]}" + return AIAgent._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}") # Fallback: truncate the raw string but give more room than 200 chars status_code = getattr(error, "status_code", None) prefix = f"HTTP {status_code}: " if status_code else "" - return f"{prefix}{raw[:500]}" + return AIAgent._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}") def _mask_api_key_for_logs(self, key: Optional[str]) -> Optional[str]: if not key: From 408aa4fbc4839b1f770849e4e28c700b6617d07e Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:30:37 -0700 Subject: [PATCH 045/142] =?UTF-8?q?port(refactor):=20deepseek=20thinking-m?= =?UTF-8?q?ode=20(068c24f8a=20+=20cd9470f41)=20=E2=80=94=20no=20net=20chan?= =?UTF-8?q?ge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original 068c24f8a (DeepSeek thinking via legacy chat_completions path) was reverted by cd9470f41 (rewired to DeepSeekProfile.build_api_kwargs_extras). Both commits' run_agent.py edits cancel out at the extracted-module level. The active fix lives in plugins/model-providers/deepseek/__init__.py (merged cleanly from main via the prior merge commit). Co-authored-by: twebefy Co-authored-by: teknium1 <127238744+teknium1@users.noreply.github.com> From 6975a2d9ae20c5131c4fd3b3758dc9eade8cc6a0 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:33:18 -0700 Subject: [PATCH 046/142] =?UTF-8?q?fix(xai-oauth):=20entitlement-403=20cha?= =?UTF-8?q?in=20=E2=80=94=20final=20state=20(ce0e189d3=20+=209818b9a1a=20+?= =?UTF-8?q?=206784c8079=20+=20dffb602f3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapses the four-commit xAI entitlement-403 chain to its final on-main state, ported to the post-refactor module layout: - Added _is_entitlement_failure on AIAgent (run_agent.py) — detects Grok subscription-shape 403s on (401|403|None) status codes. - Added entitlement-skip branch to recover_with_credential_pool (agent/agent_runtime_helpers.py) — breaks the refresh-loop that Don's 100-iteration trace exposed when a Premium+ user hit a real entitlement issue. - Removed _decorate_xai_entitlement_error and unwrapped its two _summarize_api_error call sites — xAI's own body text already points users at grok.com/?_s=usage so we surface that verbatim (dffb602f3 reasoning: X Premium subs DO now work per xAI's 2026-05-16 announcement, so editorialising would misdirect). - grok-4.3 1M context entry landed in agent/model_metadata.py via the prior merge — no additional port needed. Tests already on disk (tests/run_agent/test_codex_xai_oauth_recovery.py) assert _is_entitlement_failure shape and verbatim body surfacing. Closes #27110. Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/agent_runtime_helpers.py | 9 +++++ run_agent.py | 69 +++++++++++++++++----------------- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 797047f95d3..ea48163ba0b 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -598,6 +598,15 @@ def recover_with_credential_pool( return False, True if effective_reason == FailoverReason.auth: + if agent._is_entitlement_failure(error_context, status_code): + _ra().logger.info( + "Credential %s — entitlement-shaped 403 from %s; " + "skipping pool refresh (account lacks subscription, " + "not a transient auth failure).", + status_code if status_code is not None else "auth", + agent.provider or "provider", + ) + return False, has_retried_429 refreshed = pool.try_refresh_current() if refreshed is not None: _ra().logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") diff --git a/run_agent.py b/run_agent.py index 80577a19be3..1cb0ae761e6 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1288,43 +1288,42 @@ class AIAgent: _save_trajectory_to_file(trajectory, self.model, completed) @staticmethod - def _decorate_xai_entitlement_error(detail: str) -> str: - """Append a friendly hint when xAI's OAuth surface returns an - entitlement-shaped error. + def _is_entitlement_failure( + error_context: Optional[Dict[str, Any]], + status_code: Optional[int], + ) -> bool: + """Detect subscription/entitlement 403s that masquerade as auth failures. - xAI's ``/v1/responses`` endpoint replies to OAuth tokens that lack a - SuperGrok / X Premium subscription with HTTP 403 carrying a body like:: + Returned True only when the body text matches a known entitlement + shape AND the status is 401/403. Refreshing an OAuth token cannot + fix an unsubscribed account, so callers should surface the error + instead of looping the credential pool. - {"code": "The caller does not have permission to execute the - specified operation", "error": "You have either run out of - available resources or do not have an active Grok subscription. - Manage subscriptions at https://grok.com/..."} + Current matches: + * xAI OAuth: "do not have an active Grok subscription" / + "out of available resources" / "does not have permission" + "grok" - The raw text is useful but the action the user needs to take (subscribe - on grok.com, or switch providers with ``/model``) isn't obvious from - the wire format. Detect the entitlement shape and append a hint. - - Matched once per detail string — won't double-decorate if the upstream - already concatenated the same text. + Extend here for new providers as we discover them (Anthropic's + Claude Max OAuth entitlement errors look distinct enough today that + the existing 1M-context-beta branch handles them; revisit if other + subscription tiers start producing the same loop signature). """ - if not detail: - return detail - lower = detail.lower() - is_entitlement = ( - "do not have an active grok subscription" in lower - or ("out of available resources" in lower and "grok" in lower) - or ("does not have permission" in lower and "grok" in lower) - ) - if not is_entitlement: - return detail - hint = ( - " — xAI OAuth account lacks SuperGrok / X Premium entitlement for " - "this model. Subscribe at https://grok.com or run `/model` to " - "switch providers." - ) - if hint.strip() in detail: - return detail - return f"{detail}{hint}" + if status_code not in (401, 403, None): + return False + if not isinstance(error_context, dict): + return False + message = str(error_context.get("message") or "").lower() + reason = str(error_context.get("reason") or "").lower() + haystack = f"{message} {reason}" + if not haystack.strip(): + return False + if "do not have an active grok subscription" in haystack: + return True + if "out of available resources" in haystack and "grok" in haystack: + return True + if "does not have permission" in haystack and "grok" in haystack: + return True + return False @staticmethod def _summarize_api_error(error: Exception) -> str: @@ -1359,12 +1358,12 @@ class AIAgent: if msg: status_code = getattr(error, "status_code", None) prefix = f"HTTP {status_code}: " if status_code else "" - return AIAgent._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}") + return f"{prefix}{msg[:300]}" # Fallback: truncate the raw string but give more room than 200 chars status_code = getattr(error, "status_code", None) prefix = f"HTTP {status_code}: " if status_code else "" - return AIAgent._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}") + return f"{prefix}{raw[:500]}" def _mask_api_key_for_logs(self, key: Optional[str]) -> Optional[str]: if not key: From f885be030cc2521a4dd20122c66f411f9c1377e5 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:33:59 -0700 Subject: [PATCH 047/142] =?UTF-8?q?fix(auxiliary):=20resolve=20xai=20oauth?= =?UTF-8?q?=20compression=20from=20pool=20=E2=80=94=20port=20to=20conversa?= =?UTF-8?q?tion=5Fcompression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 97a32afdc by helix4u targeted _check_compression_model_feasibility in pre-refactor run_agent.py. The function body now lives in agent/conversation_compression.py — re-applied the configured-but-unavailable provider message there. Co-authored-by: helix4u <4317663+helix4u@users.noreply.github.com> --- agent/conversation_compression.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 90c637ee4fa..bc70623997d 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -80,11 +80,20 @@ def check_compression_model_feasibility(agent: Any) -> None: except Exception: _aux_cfg_provider = "" if client is None or not aux_model: - msg = ( - "⚠ No auxiliary LLM provider configured — context " - "compression will drop middle turns without a summary. " - "Run `hermes setup` or set OPENROUTER_API_KEY." - ) + if _aux_cfg_provider and _aux_cfg_provider != "auto": + msg = ( + "⚠ Configured auxiliary compression provider " + f"'{_aux_cfg_provider}' is unavailable — context " + "compression will drop middle turns without a summary. " + "Check auxiliary.compression in config.yaml and " + "reauthenticate that provider." + ) + else: + msg = ( + "⚠ No auxiliary LLM provider configured — context " + "compression will drop middle turns without a summary. " + "Run `hermes setup` or set OPENROUTER_API_KEY." + ) agent._compression_warning = msg agent._emit_status(msg) logger.warning( From fe4c87eb28907c467f60335d75680a50e77b15c9 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:35:54 -0700 Subject: [PATCH 048/142] =?UTF-8?q?fix(agent):=20retry=20malformed=20anthr?= =?UTF-8?q?opic=20stream=20parser=20errors=20=E2=80=94=20port=20to=20extra?= =?UTF-8?q?cted=20modules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 9c304a7f5 by helix4u targeted _flatten_exception_chain, _summarize_api_error, and the _call streaming retry loop in pre-refactor run_agent.py. Re-applied to: - New _is_provider_stream_parse_error helper → run_agent.py (next to _flatten_exception_chain in the AIAgent class) - _summarize_api_error early-return for the malformed-streaming ValueError → run_agent.py (kept method body) - _call streaming retry: _is_stream_parse_err flag wired into _is_transient AND the post-exhaustion branch + dedicated malformed-streaming user-status string → agent/chat_completion_helpers.py (the _call body now lives there) Co-authored-by: helix4u <4317663+helix4u@users.noreply.github.com> --- agent/chat_completion_helpers.py | 13 +++++++++++-- run_agent.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index d163557b8fa..0b3c394832f 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1632,6 +1632,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _is_conn_err = isinstance( e, (_httpx.ConnectError, _httpx.RemoteProtocolError, ConnectionError) ) + _is_stream_parse_err = agent._is_provider_stream_parse_error(e) # If the stream died AFTER some tokens were delivered: # normally we don't retry (the user already saw text, @@ -1671,7 +1672,10 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= for phrase in _SSE_PREVIEW_PHRASES ) _is_transient = ( - _is_timeout or _is_conn_err or _is_sse_conn_err_preview + _is_timeout + or _is_conn_err + or _is_sse_conn_err_preview + or _is_stream_parse_err ) _can_silent_retry = ( _partial_tool_in_flight @@ -1769,7 +1773,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= for phrase in _SSE_CONN_PHRASES ) - if _is_timeout or _is_conn_err or _is_sse_conn_err: + if _is_timeout or _is_conn_err or _is_sse_conn_err or _is_stream_parse_err: # Transient network / timeout error. Retry the # streaming request with a fresh connection first. if _stream_attempt < _max_stream_retries: @@ -1811,6 +1815,11 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= diag=request_client_holder.get("diag"), ) agent._emit_status( + "❌ Provider returned malformed streaming data after " + f"{_max_stream_retries + 1} attempts. " + "The provider may be experiencing issues — " + "try again in a moment." + if _is_stream_parse_err else "❌ Connection to provider failed after " f"{_max_stream_retries + 1} attempts. " "The provider may be experiencing issues — " diff --git a/run_agent.py b/run_agent.py index 1cb0ae761e6..f843603a1e5 100644 --- a/run_agent.py +++ b/run_agent.py @@ -700,6 +700,24 @@ class AIAgent: from agent.stream_diag import flatten_exception_chain return flatten_exception_chain(error) + def _is_provider_stream_parse_error(self, error: BaseException) -> bool: + """Return True for malformed provider streaming data from SDK parsers. + + Some Anthropic-compatible streaming providers can send a malformed + event-stream frame. The Anthropic SDK surfaces that as a plain + ``ValueError`` such as ``expected ident at line 1 column 149``. That + is provider wire-format trouble, not local request validation, so it + should follow the same retry path as a truncated JSON body. + """ + if getattr(self, "api_mode", None) != "anthropic_messages": + return False + if not isinstance(error, ValueError): + return False + if isinstance(error, (UnicodeEncodeError, json.JSONDecodeError)): + return False + message = str(error).strip().lower() + return "expected ident at line" in message + def _log_stream_retry( self, *, @@ -1335,6 +1353,12 @@ class AIAgent: """ raw = str(error) + if ( + isinstance(error, ValueError) + and "expected ident at line" in raw.lower() + ): + return f"Malformed provider streaming response: {raw[:300]}" + # Cloudflare / proxy HTML pages: grab the for a clean summary if "<!DOCTYPE" in raw or "<html" in raw: m = re.search(r"<title[^>]*>([^<]+)", raw, re.IGNORECASE) From 3fbedd732e5179759d797927fd0f2cf4324682b2 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:36:37 -0700 Subject: [PATCH 049/142] =?UTF-8?q?feat:=20add=20supports=5Fparallel=5Ftoo?= =?UTF-8?q?l=5Fcalls=20for=20MCP=20servers=20(#26825)=20=E2=80=94=20port?= =?UTF-8?q?=20to=20tool=5Fdispatch=5Fhelpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 395e9dd9e by Teknium targeted module-level _is_mcp_tool_parallel_safe and _should_parallelize_tool_batch helpers in pre-refactor run_agent.py. Both helpers now live in agent/tool_dispatch_helpers.py — re-applied to that module. The tools/mcp_tool.py portion (the public is_mcp_tool_parallel_safe API + _parallel_safe_servers tracking) merged cleanly from main via the prior merge commit. Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/tool_dispatch_helpers.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/agent/tool_dispatch_helpers.py b/agent/tool_dispatch_helpers.py index 289e10fb027..30aa8869db9 100644 --- a/agent/tool_dispatch_helpers.py +++ b/agent/tool_dispatch_helpers.py @@ -87,6 +87,19 @@ def _is_destructive_command(cmd: str) -> bool: return False +def _is_mcp_tool_parallel_safe(tool_name: str) -> bool: + """Check if an MCP tool comes from a server with parallel tool calls enabled. + + Lazy-imports from ``tools.mcp_tool`` to avoid circular dependencies. + Returns False if the MCP module is not available. + """ + try: + from tools.mcp_tool import is_mcp_tool_parallel_safe + return is_mcp_tool_parallel_safe(tool_name) + except Exception: + return False + + def _should_parallelize_tool_batch(tool_calls) -> bool: """Return True when a tool-call batch is safe to run concurrently.""" if len(tool_calls) <= 1: @@ -126,7 +139,9 @@ def _should_parallelize_tool_batch(tool_calls) -> bool: continue if tool_name not in _PARALLEL_SAFE_TOOLS: - return False + # Check if it's an MCP tool from a server that opted into parallel calls. + if not _is_mcp_tool_parallel_safe(tool_name): + return False return True From df22d29522ced894ab79ff66e4496c2c93be65c4 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:38:45 -0700 Subject: [PATCH 050/142] =?UTF-8?q?fix(copilot):=20GitHub=20Models=20413?= =?UTF-8?q?=20hint=20=E2=80=94=20port=20to=20extracted=20conversation=5Flo?= =?UTF-8?q?op?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commits 4ded3ede3 (@konsisumer) + 374dc81c2 (Teknium) added a 413 hint to run_agent.py's agent loop. Final-state version (the sharpened 374dc81c2 wording) ported to agent/conversation_loop.py, where the payload_too_large branch now lives. The deprecation detection + _URL_TO_PROVIDER changes from both commits landed in agent/copilot_acp_client.py and agent/model_metadata.py via the prior merge. Closes #10648 Co-authored-by: konsisumer Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/conversation_loop.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index e121c4b2a74..8096b754298 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2333,6 +2333,39 @@ def run_conversation( classified.reason == FailoverReason.payload_too_large ) + # Actionable hint for GitHub Models (Azure) 413 errors. + # The free tier enforces a hard 8K token cap per request, + # which Hermes' system prompt + tool schemas alone exceed. + # Compression can't help — the floor is the system prompt + # itself, not the conversation — so surface a clear "not + # compatible" message instead of looping into three futile + # compression attempts. + if ( + status_code == 413 + and isinstance(agent.base_url, str) + and "models.inference.ai.azure.com" in agent.base_url + ): + agent._vprint( + f"{agent.log_prefix} 💡 GitHub Models free tier (models.inference.ai.azure.com) caps every", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} request at ~8K tokens. Hermes' system prompt + tool schemas baseline", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} exceeds that floor, so this endpoint cannot run an agentic loop.", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} Use the `copilot` provider with a Copilot subscription token (`hermes", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} setup` → GitHub Copilot), or pick any other provider.", + force=True, + ) + if is_payload_too_large: compression_attempts += 1 if compression_attempts > max_compression_attempts: From 80fa92a491c67ae98c43ea723487db640d99857f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:39:41 -0700 Subject: [PATCH 051/142] =?UTF-8?q?fix(codex):=20rotate=20pool=20on=20usag?= =?UTF-8?q?e=20limit=20429=20=E2=80=94=20port=20to=20extracted=20modules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit e51d74ab9 by Maxim Esipov targeted _extract_api_error_context and _recover_with_credential_pool in pre-refactor run_agent.py. Both bodies now live in agent/agent_runtime_helpers.py — re-applied to that module: - extract_api_error_context: payload.get('type') added to the reason fallback chain (Codex error bodies use 'type' instead of 'code'/'error') - recover_with_credential_pool: usage_limit_reached detection in the rate_limit branch — skip the retry-once-then-rotate dance and rotate immediately when the body says the per-account usage limit hit. Co-authored-by: Maxim Esipov --- agent/agent_runtime_helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index ea48163ba0b..bac21f14061 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -583,7 +583,15 @@ def recover_with_credential_pool( return False, has_retried_429 if effective_reason == FailoverReason.rate_limit: - if not has_retried_429: + usage_limit_reached = False + if error_context: + context_reason = str(error_context.get("reason") or "").lower() + context_message = str(error_context.get("message") or "").lower() + usage_limit_reached = ( + "usage_limit_reached" in context_reason + or "usage limit has been reached" in context_message + ) + if not has_retried_429 and not usage_limit_reached: return False, True rotate_status = status_code if status_code is not None else 429 next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) @@ -1910,7 +1918,7 @@ def extract_api_error_context(error: Exception) -> Dict[str, Any]: if isinstance(body, dict): payload = body.get("error") if isinstance(body.get("error"), dict) else body if isinstance(payload, dict): - reason = payload.get("code") or payload.get("error") + reason = payload.get("code") or payload.get("type") or payload.get("error") if isinstance(reason, str) and reason.strip(): context["reason"] = reason.strip() message = payload.get("message") or payload.get("error_description") From aa05ffba530fde599b6515120578364cce682ac7 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:41:09 -0700 Subject: [PATCH 052/142] fix(xai): surface provider 'error' SSE frame in Codex fallback stream (#27184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 2b193907d by Teknium added a new module-level _StreamErrorEvent class and threaded its raise into _run_codex_create_stream_fallback in pre-refactor run_agent.py. - _StreamErrorEvent class → run_agent.py (module-level, next to _qwen_portal_headers; class needs to be top-level for the codex runtime to import it) - The fallback event-loop's 'type=error' handler → agent/codex_runtime.py where run_codex_create_stream_fallback now lives. Imports _StreamErrorEvent lazily from run_agent to avoid circular import. Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/codex_runtime.py | 29 +++++++++++++++++++++++++++++ run_agent.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 547fbb9ce07..02b788f5777 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -356,6 +356,35 @@ def run_codex_create_stream_fallback(agent, api_kwargs: dict, client: Any = None if not event_type and isinstance(event, dict): event_type = event.get("type") + # ``error`` SSE frames carry the provider's real failure + # reason (subscription / quota / model-not-available / + # rejected-reasoning-replay) but never appear in the + # ``{completed, incomplete, failed}`` terminal set, so the + # raw loop below would silently consume them and end with + # "did not emit a terminal response". xAI in particular + # emits ``type=error`` as the FIRST frame for OAuth + # accounts whose Grok subscription is missing/exhausted — + # the SDK's stream helper raises ``RuntimeError(Expected + # to have received response.created before error)`` which + # the caller catches and routes here, expecting this + # fallback to surface the message. Synthesize an + # APIError-shaped exception so ``_summarize_api_error`` + # and the credential-pool entitlement detector see the + # real text instead of a generic RuntimeError. + if event_type == "error": + err_message = getattr(event, "message", None) + if not err_message and isinstance(event, dict): + err_message = event.get("message") + err_code = getattr(event, "code", None) + if not err_code and isinstance(event, dict): + err_code = event.get("code") + err_param = getattr(event, "param", None) + if not err_param and isinstance(event, dict): + err_param = event.get("param") + err_message = (err_message or "stream emitted error event").strip() + from run_agent import _StreamErrorEvent + raise _StreamErrorEvent(err_message, code=err_code, param=err_param) + # Collect output items and text deltas for backfill if event_type == "response.output_item.done": done_item = getattr(event, "item", None) diff --git a/run_agent.py b/run_agent.py index f843603a1e5..8471afccddf 100644 --- a/run_agent.py +++ b/run_agent.py @@ -284,6 +284,45 @@ def _qwen_portal_headers() -> dict: } +class _StreamErrorEvent(Exception): + """Synthesized provider error surfaced from a Responses ``error`` SSE frame. + + Some Codex-style Responses backends (xAI for subscription/quota + failures, custom relays under malformed-tool-call conditions) emit a + standalone ``type=error`` frame instead of routing the failure + through ``response.failed`` or returning an HTTP 4xx. The fallback + streaming path raises this exception so ``_summarize_api_error`` and + ``_extract_api_error_context`` see a familiar ``.body`` / + ``.status_code`` shape and the entitlement detector can match the + underlying provider message ("do not have an active Grok + subscription", etc.). + """ + + def __init__( + self, + message: str, + *, + code: Optional[str] = None, + param: Optional[str] = None, + status_code: Optional[int] = None, + ) -> None: + super().__init__(message) + self.message = message + self.code = code + self.param = param + self.status_code = status_code + # OpenAI SDK-shaped body so _extract_api_error_context / + # _summarize_api_error / classify_api_error all pick it up. + self.body: Dict[str, Any] = { + "error": { + "message": message, + "code": code, + "param": param, + "type": "error", + } + } + + class AIAgent: """ AI Agent with tool calling capabilities. From 4ab9a06a51268a2864cc66ee36ef34bf6f9ef6e8 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:41:45 -0700 Subject: [PATCH 053/142] fix(agent): reset _fallback_index at turn start even when no fallback activated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 33528b428 by konsisumer targeted _restore_primary_runtime in pre-refactor run_agent.py. The body now lives in agent/agent_runtime_helpers.restore_primary_runtime — re-applied there. Fixes #20465 Co-authored-by: konsisumer --- agent/agent_runtime_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index bac21f14061..b5c70392946 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -819,6 +819,14 @@ def restore_primary_runtime(agent) -> bool: ``gateway/run.py``), so this restoration IS needed there too. """ if not agent._fallback_activated: + # Reset the chain index even when no fallback was activated this + # turn. Without this, a turn where _try_activate_fallback() was + # called but returned False (chain exhausted or provider not + # configured) leaves _fallback_index >= len(_fallback_chain) while + # _fallback_activated stays False. The next turn skips this block + # entirely, stranding the index and silently blocking all future + # fallback attempts for the session. Fixes #20465. + agent._fallback_index = 0 return False if getattr(agent, "_rate_limited_until", 0) > time.monotonic(): From b5bcffe1674fa9ab3ba7a754c07ab77bedde83a8 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:42:16 -0700 Subject: [PATCH 054/142] fix(fallback): forward custom_providers to fallback model context-length detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 21078ebce by PaTTeeL targeted _try_activate_fallback in pre-refactor run_agent.py. The body now lives in agent/chat_completion_helpers.try_activate_fallback — re-applied there. Co-authored-by: PaTTeeL <9150277+PaTTeeL@users.noreply.github.com> --- agent/chat_completion_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 0b3c394832f..66302536263 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -850,6 +850,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool agent.model, base_url=agent.base_url, api_key=agent.api_key, provider=agent.provider, config_context_length=getattr(agent, "_config_context_length", None), + custom_providers=agent._custom_providers, ) agent.context_compressor.update_model( model=agent.model, From 4ece521bcf37401686e73f1d08ebaa87caaae05a Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:42:49 -0700 Subject: [PATCH 055/142] fix(run_agent): isolate background review fork from external memory plugins (#27190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 973f27e95 by Teknium targeted _spawn_background_review in pre-refactor run_agent.py. The body now lives in agent/background_review._spawn_background_review — re-applied there. Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com> --- agent/background_review.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/agent/background_review.py b/agent/background_review.py index 0319bbfa046..83292029c6c 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -363,6 +363,21 @@ def _run_review_in_thread( # owns the loop and the agent-loop tools dispatch. if _parent_api_mode == "codex_app_server": _parent_api_mode = "codex_responses" + # skip_memory=True keeps the review fork from + # touching external memory plugins (honcho, mem0, + # supermemory, etc.). Without it, the fork's + # __init__ rebuilds its own _memory_manager from + # config, scoped to the parent's session_id, and + # run_conversation() then leaks the harness prompt + # into the user's real memory namespace via three + # ingestion sites: on_turn_start (cadence + turn + # message), prefetch_all (recall query), and + # sync_all (harness prompt + review output recorded + # as a (user, assistant) turn pair). Built-in + # MEMORY.md / USER.md state is re-bound from the + # parent below so memory(action="add") writes from + # the review still land on disk; the review just + # has zero side effects on external providers. review_agent = AIAgent( model=agent.model, max_iterations=16, @@ -374,6 +389,7 @@ def _run_review_in_thread( api_key=_parent_runtime.get("api_key") or None, credential_pool=getattr(agent, "_credential_pool", None), parent_session_id=agent.session_id, + skip_memory=True, ) review_agent._memory_write_origin = "background_review" review_agent._memory_write_context = "background_review" From 36ad8336f9fcf2fe03f43782281cc7a555cbc6ed Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:43:09 -0700 Subject: [PATCH 056/142] fix(run_agent): guard memory provider init against empty/whitespace string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 8d756a421 by austrian_guy targeted __init__ in pre-refactor run_agent.py. The body now lives in agent/agent_init.init_agent — re-applied there. Co-authored-by: austrian_guy <33156212+ether-btc@users.noreply.github.com> --- agent/agent_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent_init.py b/agent/agent_init.py index d5798f163ff..df8fe229e7b 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -960,7 +960,7 @@ def init_agent( try: _mem_provider_name = mem_config.get("provider", "") if mem_config else "" - if _mem_provider_name: + if _mem_provider_name and _mem_provider_name.strip(): from agent.memory_manager import MemoryManager as _MemoryManager from plugins.memory import load_memory_provider as _load_mem agent._memory_manager = _MemoryManager() From 563b4d9e51a46cc421e327b351cb7efe1ccb151b Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:43:37 -0700 Subject: [PATCH 057/142] fix: strip image parts for non-vision models with provider profiles + getattr-safe _custom_providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit 75e5d0f6b by hueilau targeted _build_api_kwargs in pre-refactor run_agent.py. The body now lives in agent/chat_completion_helpers.build_api_kwargs — re-applied there. Also: switch the custom_providers forward (from 21078ebce) to use getattr() — tests build a bare AIAgent via __new__ and would otherwise hit AttributeError on _custom_providers. Co-authored-by: hueilau <33933019+hueilau@users.noreply.github.com> --- agent/chat_completion_helpers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 66302536263..1bf1ebc651e 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -377,6 +377,11 @@ def build_api_kwargs(agent, api_messages: list) -> dict: if _ephemeral_out is not None: agent._ephemeral_max_output_tokens = None + # Strip image parts for non-vision models that have provider profiles + # (e.g. DeepSeek, Kimi). The legacy path below already does this, but + # registered providers with profiles were bypassing the strip. + api_messages = agent._prepare_messages_for_non_vision_model(api_messages) + return _ct.build_kwargs( model=agent.model, messages=api_messages, @@ -850,7 +855,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool agent.model, base_url=agent.base_url, api_key=agent.api_key, provider=agent.provider, config_context_length=getattr(agent, "_config_context_length", None), - custom_providers=agent._custom_providers, + custom_providers=getattr(agent, "_custom_providers", None), ) agent.context_compressor.update_model( model=agent.model, From 519657aa98d4969ec9e23c70c074d1982ef3ccf1 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 17 May 2026 00:28:24 -0700 Subject: [PATCH 058/142] fix(matrix): warn on clock-skew silent message drops (#12614) (#27330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 5-second startup-grace filter in _on_room_message silently drops events where event_ts < startup_ts - 5. When the host clock is set ahead of real time, the comparison flips against every live event and the bot 'connects but never replies' — exactly the symptom in #12614. Reporter Schnurzel700 chased this for several weeks before tracing it to their Debian VM's clock being out of sync. The current /1000.0 millisecond->second conversion is correct (mautrix returns ms); the failure mode is purely environmental. Add a one-shot WARNING that fires when: - we are >30s past startup (initial-sync replay window closed), AND - 3 consecutive drops share the same skew within 60s (a constant clock offset, not varied-age backfill from an invited room). State is reset in connect() so reconnects after fixing NTP rearm the detector. Includes the NTP fix instruction in the warning message itself and a new Troubleshooting entry in the Matrix docs. 5 new tests cover the happy path, initial-sync backfill, under- threshold drops, varied-age backfill, and the reconnect rearm path. --- gateway/platforms/matrix.py | 59 ++++++ tests/gateway/test_matrix.py | 204 ++++++++++++++++++++ website/docs/user-guide/messaging/matrix.md | 17 ++ 3 files changed, 280 insertions(+) diff --git a/gateway/platforms/matrix.py b/gateway/platforms/matrix.py index 95dc73201c5..50d383f6f22 100644 --- a/gateway/platforms/matrix.py +++ b/gateway/platforms/matrix.py @@ -348,6 +348,17 @@ class MatrixAdapter(BasePlatformAdapter): self._sync_task: Optional[asyncio.Task] = None self._closing = False self._startup_ts: float = 0.0 + # Clock-skew detection: count grace-check drops that happen well + # after startup (i.e. not initial-sync backfill). If the host's + # system clock is set ahead of real time, the startup grace check + # `event_ts < startup_ts - 5` silently drops every live message. + # See #12614 — the symptom is "bot joins rooms but never replies". + # Drops only count when their skew matches the first sampled drop + # (within 60s), so varied-age backfill from freshly-invited rooms + # doesn't trip the heuristic. + self._late_grace_drops: int = 0 + self._late_grace_skew: float = 0.0 + self._clock_skew_warned: bool = False # Cache: room_id → bool (is DM) self._dm_rooms: Dict[str, bool] = {} @@ -842,6 +853,11 @@ class MatrixAdapter(BasePlatformAdapter): # Initial sync to catch up, then start background sync. self._startup_ts = time.time() + # Reset clock-skew detector for each connect cycle so a reconnect + # after the user fixes NTP doesn't inherit stale counters. + self._late_grace_drops = 0 + self._late_grace_skew = 0.0 + self._clock_skew_warned = False self._closing = False try: @@ -1542,6 +1558,49 @@ class MatrixAdapter(BasePlatformAdapter): ) event_ts = raw_ts / 1000.0 if raw_ts else 0.0 if event_ts and event_ts < self._startup_ts - _STARTUP_GRACE_SECONDS: + # If we are well past startup but events are still being dropped + # by the grace check, the host clock is probably set ahead of + # real time — every live event then looks "older than startup". + # Warn once so users can fix NTP instead of chasing a ghost. + # See #12614 (Schnurzel700, April 2026). + # + # Filter out backfill (events legitimately old) by requiring: + # - we are >30s past startup (initial-sync replay window closed) + # - the skew is *consistent* across consecutive drops, which is + # the signature of a constant clock offset rather than a + # variable-age room history. Backfill from a freshly invited + # room can deliver events spanning hours/days — those skews + # will be all over the place and reset the counter. + if not self._clock_skew_warned and ( + time.time() - self._startup_ts > 30 + ): + skew = self._startup_ts - event_ts + # Sanity bound: malformed events with negative or absurd + # timestamps shouldn't count. + if 5 < skew < 86400: + if self._late_grace_drops == 0: + self._late_grace_skew = skew + self._late_grace_drops = 1 + elif abs(skew - self._late_grace_skew) < 60: + # Consistent offset → likely real clock skew. + self._late_grace_drops += 1 + else: + # Varied skew → likely backfill, restart sampling. + self._late_grace_skew = skew + self._late_grace_drops = 1 + if self._late_grace_drops >= 3: + logger.warning( + "Matrix: dropped %d consecutive live events as " + "'too old' more than 30s after startup (skew " + "≈ %.0fs). The host system clock is likely set " + "ahead of real time, which causes the startup " + "grace filter to silently discard every incoming " + "message. Run `timedatectl set-ntp true` (or " + "sync NTP) and restart the bot.", + self._late_grace_drops, + skew, + ) + self._clock_skew_warned = True return # Extract content from the event. diff --git a/tests/gateway/test_matrix.py b/tests/gateway/test_matrix.py index c329441531d..a0fb8f086d8 100644 --- a/tests/gateway/test_matrix.py +++ b/tests/gateway/test_matrix.py @@ -2257,6 +2257,210 @@ class TestMatrixOnRoomMessageFilter: ev = self._mk_event(sender="@alice:example.org", body="hello bot") await self.adapter._on_room_message(ev) self.adapter._handle_text_message.assert_awaited_once() + + +class TestMatrixClockSkewWarning: + """Clock-skew detector for #12614. + + Reporter's host clock was set ~2 hours ahead of real time. The grace + filter `event_ts < startup_ts - 5` then drops every live event because + server timestamps look "older than startup". When this happens well + after startup (>30s), the adapter logs a one-shot WARNING pointing the + user at NTP instead of failing silently. + """ + + def setup_method(self): + self.adapter = _make_adapter() + self.adapter._user_id = "@bot:example.org" + self.adapter._handle_text_message = AsyncMock() + self.adapter._handle_media_message = AsyncMock() + + @staticmethod + def _mk_event(sender, ts_ms, event_id=None): + ev = MagicMock() + ev.room_id = "!room:example.org" + ev.sender = sender + ev.event_id = event_id or f"$evt-{sender}-{ts_ms}" + ev.timestamp = ts_ms + ev.server_timestamp = ts_ms + ev.content = {"msgtype": "m.text", "body": "hi"} + return ev + + @pytest.mark.asyncio + async def test_late_drops_emit_one_shot_clock_skew_warning(self, caplog): + import logging + import time as _t + + # Simulate the reporter's environment: host clock is ~2 hours ahead + # of server time. Startup happened "in the future" relative to the + # real-world events we're now receiving. + now = _t.time() + self.adapter._startup_ts = now - 60 # bot started 60s ago (wall clock) + # Server events are dated 2h before startup_ts (skewed clock). + skewed_event_ts_ms = int((self.adapter._startup_ts - 7200) * 1000) + + with caplog.at_level(logging.WARNING, logger="gateway.platforms.matrix"): + for i in range(5): + ev = self._mk_event( + sender=f"@alice{i}:example.org", ts_ms=skewed_event_ts_ms + ) + await self.adapter._on_room_message(ev) + + # Handler should never be invoked — all events failed the grace check. + self.adapter._handle_text_message.assert_not_called() + # Exactly one WARNING from THIS logger should be emitted. Filter by + # logger name so unrelated stdlib/library warnings can't satisfy the + # assertion. + skew_warnings = [ + r for r in caplog.records + if r.name == "gateway.platforms.matrix" + and r.levelname == "WARNING" + and "set-ntp" in r.getMessage() + ] + assert len(skew_warnings) == 1, ( + f"expected exactly 1 clock-skew warning, got {len(skew_warnings)}" + ) + msg = skew_warnings[0].getMessage() + assert "7200" in msg, f"skew value missing from message: {msg!r}" + # Pin the counter so a regression in the gating logic (e.g. warning + # at threshold 1 or 5, or not stopping after warn) is caught. + assert self.adapter._late_grace_drops == 3 + assert self.adapter._clock_skew_warned is True + + @pytest.mark.asyncio + async def test_initial_sync_drops_do_not_warn(self, caplog): + """During the first 30s after startup, old events are normal backfill.""" + import logging + import time as _t + + now = _t.time() + # Startup was 1s ago — we're still in the initial-sync window. + self.adapter._startup_ts = now - 1 + old_ts_ms = int((self.adapter._startup_ts - 3600) * 1000) + + with caplog.at_level(logging.WARNING, logger="gateway.platforms.matrix"): + for i in range(5): + ev = self._mk_event( + sender=f"@alice{i}:example.org", ts_ms=old_ts_ms + ) + await self.adapter._on_room_message(ev) + + # Backfill drops are silent — no clock-skew warning fired. + assert self.adapter._clock_skew_warned is False + skew_warnings = [ + r for r in caplog.records + if r.name == "gateway.platforms.matrix" + and "set-ntp" in r.getMessage() + ] + assert skew_warnings == [] + + @pytest.mark.asyncio + async def test_fewer_than_three_late_drops_do_not_warn(self, caplog): + """A single delayed backfill event after 30s shouldn't trigger NTP advice.""" + import logging + import time as _t + + now = _t.time() + self.adapter._startup_ts = now - 120 # extra slack vs the 30s gate + old_ts_ms = int((self.adapter._startup_ts - 3600) * 1000) + + with caplog.at_level(logging.WARNING, logger="gateway.platforms.matrix"): + for i in range(2): # only 2 late drops — under the threshold + ev = self._mk_event( + sender=f"@alice{i}:example.org", ts_ms=old_ts_ms + ) + await self.adapter._on_room_message(ev) + + assert self.adapter._late_grace_drops == 2 + assert self.adapter._clock_skew_warned is False + + @pytest.mark.asyncio + async def test_varied_backfill_skews_do_not_warn(self, caplog): + """Backfill from a freshly-invited room delivers events of varied age. + + A genuine clock-skew bug produces drops with a *constant* offset + (every event is ~X seconds older than wall clock). Joining an old + room post-startup delivers events spanning hours-to-days; those + skews vary wildly and must NOT trigger the NTP warning. + """ + import logging + import time as _t + + now = _t.time() + self.adapter._startup_ts = now - 120 + # Each event has a different age, ranging from 1h to 30d ago. + ages_in_hours = [1, 24, 168, 720, 4] # 1h, 1d, 1w, 30d, 4h + with caplog.at_level(logging.WARNING, logger="gateway.platforms.matrix"): + for i, hrs in enumerate(ages_in_hours): + ts_ms = int((self.adapter._startup_ts - hrs * 3600) * 1000) + ev = self._mk_event( + sender=f"@alice{i}:example.org", ts_ms=ts_ms + ) + await self.adapter._on_room_message(ev) + + # The varied-skew guard should keep the counter from reaching 3. + assert self.adapter._late_grace_drops < 3 + assert self.adapter._clock_skew_warned is False + skew_warnings = [ + r for r in caplog.records + if r.name == "gateway.platforms.matrix" + and "set-ntp" in r.getMessage() + ] + assert skew_warnings == [] + + @pytest.mark.asyncio + async def test_state_reset_allows_warning_to_fire_again(self, caplog): + """After the reset block at top of connect() runs, the warning is rearmed. + + Reconnect lifecycle: the user fixes NTP, restarts the bot, and the + new connect() call resets _late_grace_drops / _clock_skew_warned at + the top. This test exercises the rearm path by: + 1. Tripping the warning once (state: warned=True). + 2. Running the same reset block connect() runs. + 3. Tripping the warning a second time — the second warning should + fire because the state was cleared. + """ + import logging + import time as _t + + now = _t.time() + self.adapter._startup_ts = now - 60 + skewed_ms = int((self.adapter._startup_ts - 7200) * 1000) + + with caplog.at_level(logging.WARNING, logger="gateway.platforms.matrix"): + for i in range(3): + ev = self._mk_event( + sender=f"@alice{i}:example.org", ts_ms=skewed_ms, + event_id=f"$first-{i}", + ) + await self.adapter._on_room_message(ev) + assert self.adapter._clock_skew_warned is True + + # Mirror the reset block in connect() (matrix.py around line 855). + self.adapter._startup_ts = _t.time() - 60 + self.adapter._late_grace_drops = 0 + self.adapter._late_grace_skew = 0.0 + self.adapter._clock_skew_warned = False + + # Same skewed-clock scenario should warn AGAIN after reset. + skewed_ms2 = int((self.adapter._startup_ts - 7200) * 1000) + for i in range(3): + ev = self._mk_event( + sender=f"@bob{i}:example.org", ts_ms=skewed_ms2, + event_id=f"$second-{i}", + ) + await self.adapter._on_room_message(ev) + + skew_warnings = [ + r for r in caplog.records + if r.name == "gateway.platforms.matrix" + and "set-ntp" in r.getMessage() + ] + assert len(skew_warnings) == 2, ( + f"expected 2 warnings (one per connect cycle), got {len(skew_warnings)}" + ) + + # --------------------------------------------------------------------------- # DM auto-thread # --------------------------------------------------------------------------- diff --git a/website/docs/user-guide/messaging/matrix.md b/website/docs/user-guide/messaging/matrix.md index 255806c01ba..b03f7a655d4 100644 --- a/website/docs/user-guide/messaging/matrix.md +++ b/website/docs/user-guide/messaging/matrix.md @@ -357,6 +357,23 @@ To find a Room ID: in Element, go to the room → **Settings** → **Advanced** **Fix**: Invite the bot to the room — it auto-joins on invite. Verify your User ID is in `MATRIX_ALLOWED_USERS` (use the full `@user:server` format). Restart the gateway. +### Bot joins rooms but silently drops every message (clock skew) + +**Cause**: The host's system clock is set ahead of real time. The Matrix adapter applies a 5-second startup-grace filter (`event_ts < startup_ts - 5`) to ignore events replayed from initial sync. When the wall clock is ahead, every incoming event looks "older than startup" and is dropped before reaching the message handler — the bot appears connected but never replies. See [#12614](https://github.com/NousResearch/hermes-agent/issues/12614). + +**Symptom**: Gateway log shows `Matrix: dropped N live events as 'too old' more than 30s after startup`. + +**Fix**: Sync the host clock with NTP and restart the bot: + +```bash +# Debian/Ubuntu +sudo timedatectl set-ntp true +timedatectl status # confirm "System clock synchronized: yes" + +# macOS +sudo sntp -sS time.apple.com +``` + ### "Failed to authenticate" / "whoami failed" on startup **Cause**: The access token or homeserver URL is incorrect. From ad1aa1a037a0603b09593dfdce1efd8111936c8f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 02:19:38 -0700 Subject: [PATCH 059/142] feat(x_search): auto-enable toolset when xAI OAuth or XAI_API_KEY is configured (#27376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The x_search toolset is gated on xAI credentials (SuperGrok OAuth or XAI_API_KEY), but it was staying off-by-default even for users who had already configured those credentials — they had to also click through `hermes tools` → X (Twitter) Search to flip it on. The HASS_TOKEN → homeassistant rule already handles the parallel case cleanly; x_search needs the same treatment. Why a separate code path from HASS_TOKEN: `ha_*` tools live inside the `hermes-cli` composite, so the subset-inference loop picks them up and the HASS branch just unmasks default_off. `x_search` is its own one-tool toolset NOT in the composite, so the subset loop never adds it — it has to be injected directly. * Add `_xai_credentials_present()` — side-effect-free check for stored xAI OAuth tokens or XAI_API_KEY (dotenv or env). No network. * In `_get_platform_tools()` else branch (no explicit user config), inject `x_search` and carve a parallel hole in default_off. * Auto-enable does NOT fire when the user has saved an explicit toolset list via `hermes tools` — that list stays authoritative. * `agent.disabled_toolsets: [x_search]` still wins (global override). Tests: 4 new in test_tools_config.py covering OAuth path, API-key path, no-creds path, and explicit-config-respect. All pass alongside existing 70/70 in that file. --- hermes_cli/tools_config.py | 58 +++++++++++++++++++++++++-- tests/hermes_cli/test_tools_config.py | 56 ++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 06ba32bea9e..3114ed12a4c 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -88,12 +88,40 @@ CONFIGURABLE_TOOLSETS = [ # who want it opt in via `hermes tools` → Video Generation, which walks # them through provider + model selection. # -# X search is off by default — gated on xAI credentials (SuperGrok OAuth -# or XAI_API_KEY). Users opt in via `hermes tools` → X (Twitter) Search, -# which walks them through credential setup. The tool's check_fn means -# the schema won't appear to the model even if enabled without credentials. +# X search is off by default for users without xAI credentials, but +# auto-enables when SuperGrok OAuth tokens are stored OR XAI_API_KEY is +# set — mirroring the HASS_TOKEN → homeassistant auto-enable below. The +# `hermes tools` → X (Twitter) Search setup walks users through credential +# setup. The tool's check_fn means the schema still won't appear to the +# model if the credential later goes missing or expires. _DEFAULT_OFF_TOOLSETS = {"moa", "homeassistant", "spotify", "discord", "discord_admin", "video", "video_gen", "x_search"} + +def _xai_credentials_present() -> bool: + """Cheap, side-effect-free check for usable xAI credentials. + + Used to auto-enable the ``x_search`` toolset when the user has either + completed xAI Grok OAuth (SuperGrok subscription) or set + ``XAI_API_KEY``. Does NOT hit the network — only inspects the local + auth store and environment. The tool's runtime ``check_fn`` still + gates schema registration if creds later expire or get revoked. + """ + try: + from hermes_cli.auth import _read_xai_oauth_tokens + + _read_xai_oauth_tokens() + return True + except Exception: + pass + try: + from tools.xai_http import get_env_value as _xai_get_env_value + + if str(_xai_get_env_value("XAI_API_KEY") or "").strip(): + return True + except Exception: + pass + return bool(str(os.environ.get("XAI_API_KEY") or "").strip()) + # Platform-scoped toolsets: only appear in the `hermes tools` checklist for # these platforms, and only resolve/save for these platforms. A toolset # absent from this map is available on every platform (current behaviour). @@ -1129,6 +1157,23 @@ def _get_platform_tools( if ts_tools and ts_tools.issubset(all_tool_names): enabled_toolsets.add(ts_key) + # Auto-enable ``x_search`` when xAI credentials are configured. + # Unlike ``homeassistant`` (whose ``ha_*`` tools live inside the + # platform composite and thus pass the subset check above), + # ``x_search`` is its own one-tool toolset that the composite does + # NOT include, so the subset loop never picks it up. Inject it + # directly here, mirroring the HASS_TOKEN → ``homeassistant`` rule + # below: once you have working creds, you don't have to also click + # through ``hermes tools`` to flip the toolset on. Only fires when + # the user has not yet saved an explicit toolset list — once they + # do, the saved list is authoritative. + x_search_auto_enabled = ( + _toolset_allowed_for_platform("x_search", platform) + and _xai_credentials_present() + ) + if x_search_auto_enabled: + enabled_toolsets.add("x_search") + default_off = set(_DEFAULT_OFF_TOOLSETS) # Legacy safety: if the platform's own name matches a default-off # toolset (e.g. `homeassistant` platform + `homeassistant` toolset), @@ -1146,6 +1191,11 @@ def _get_platform_tools( # regressed after #14798 made cron honor per-platform tool config. if "homeassistant" in default_off and os.getenv("HASS_TOKEN"): default_off.remove("homeassistant") + # Symmetric carve-out for x_search auto-enable (see the inject + # block above). Without this, the default_off subtraction would + # strip the entry we just added. + if x_search_auto_enabled and "x_search" in default_off: + default_off.remove("x_search") enabled_toolsets -= default_off # Recover non-configurable platform toolsets (e.g. discord, feishu_doc, diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index 8a94ce4302f..d6b18f1608a 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -125,6 +125,62 @@ def test_get_platform_tools_homeassistant_toolset_off_for_cron_when_hass_token_m assert "homeassistant" not in cron_enabled +def test_get_platform_tools_x_search_auto_enabled_when_xai_oauth_present(monkeypatch): + """x_search toolset auto-enables across platforms when xAI Grok OAuth + tokens are present, mirroring the HASS_TOKEN → homeassistant rule. + + The user already authenticated via SuperGrok OAuth; they shouldn't have + to also click through `hermes tools` → X (Twitter) Search to flip the + toolset on. Tool's check_fn still gates schema registration if creds + later go missing. + """ + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr( + "hermes_cli.tools_config._xai_credentials_present", lambda: True + ) + + for plat in ("cli", "cron", "telegram"): + enabled = _get_platform_tools({}, plat) + assert "x_search" in enabled, f"x_search missing for {plat}" + + +def test_get_platform_tools_x_search_auto_enabled_when_xai_api_key_present(monkeypatch): + """x_search toolset auto-enables when XAI_API_KEY is set, even without + OAuth tokens — the API-key path is a supported credential source.""" + monkeypatch.setenv("XAI_API_KEY", "fake-xai-key") + + cli_enabled = _get_platform_tools({}, "cli") + assert "x_search" in cli_enabled + + +def test_get_platform_tools_x_search_off_when_no_xai_credentials(monkeypatch): + """Without any xAI credentials, x_search stays off — preserves the + "don't ship the schema to users who can't use it" default.""" + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr( + "hermes_cli.tools_config._xai_credentials_present", lambda: False + ) + + cli_enabled = _get_platform_tools({}, "cli") + assert "x_search" not in cli_enabled + + +def test_get_platform_tools_x_search_respects_explicit_config(monkeypatch): + """Once the user has saved an explicit toolset list via `hermes tools`, + that list is authoritative — x_search auto-enable does NOT fire even + when xAI creds exist. The saved list represents deliberate choices.""" + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr( + "hermes_cli.tools_config._xai_credentials_present", lambda: True + ) + + # User explicitly opted into spotify but not x_search via `hermes tools`. + config = {"platform_toolsets": {"cli": ["hermes-cli", "spotify"]}} + enabled = _get_platform_tools(config, "cli") + assert "x_search" not in enabled + assert "spotify" in enabled + + def test_get_platform_tools_expands_composite_when_mixed_with_configurable(): """``[hermes-cli, spotify]`` (composite + configurable) must keep the full ``hermes-cli`` toolset alongside the explicit Spotify opt-in. The From a9ba636d535faa6aaf928f5c6c575209bb292f58 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sat, 16 May 2026 02:33:35 +0300 Subject: [PATCH 060/142] fix(tools): run post_setup in _reconfigure_provider() for env-var providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _configure_provider() calls _run_post_setup() after collecting env vars (line 2286). _reconfigure_provider() did not — providers with both env_vars and post_setup (Browserbase, Browser Use, Firecrawl, Camofox) skipped the installation step on reconfiguration. Fix: mirror the _configure_provider() call. post_setup hooks are idempotent (check before installing), so no behaviour change for users who already have the dependencies installed. --- hermes_cli/tools_config.py | 3 +++ tests/hermes_cli/test_tools_config.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 3114ed12a4c..9120102d646 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -2599,6 +2599,9 @@ def _reconfigure_provider(provider: dict, config: dict): else: _print_info(" Kept current") + if provider.get("post_setup"): + _run_post_setup(provider["post_setup"]) + # Imagegen backends prompt for model selection on reconfig too. plugin_name = provider.get("image_gen_plugin_name") if plugin_name: diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index d6b18f1608a..89dc33258a0 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -1045,3 +1045,30 @@ def test_reconfigure_browser_provider_overwrites_stale_use_gateway(): provider = {"name": "Browserbase", "browser_provider": "browserbase", "env_vars": []} _reconfigure_provider(provider, config) assert config["browser"]["use_gateway"] is False + + +@pytest.mark.parametrize("provider_name,post_setup_key", [ + ("Browserbase", "agent_browser"), + ("Browser Use", "agent_browser"), + ("Firecrawl", "agent_browser"), + ("Camofox", "camofox"), +]) +def test_reconfigure_provider_runs_post_setup_for_env_var_providers( + monkeypatch, provider_name, post_setup_key +): + """_reconfigure_provider() must call _run_post_setup() for providers that have + both env_vars and post_setup — parity with _configure_provider() line 2286.""" + called = [] + monkeypatch.setattr("hermes_cli.tools_config._run_post_setup", lambda key: called.append(key)) + monkeypatch.setattr("hermes_cli.tools_config.get_env_value", lambda k: None) + monkeypatch.setattr("hermes_cli.tools_config._prompt", lambda *a, **kw: "") + monkeypatch.setattr("hermes_cli.tools_config.save_env_value", lambda k, v: None) + + provider = next( + p + for p in TOOL_CATEGORIES["browser"]["providers"] + if p["name"] == provider_name + ) + _reconfigure_provider(provider, {}) + + assert called == [post_setup_key] From cc59880ab01c2ca737cc3bad99de7cde8fd32f22 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 16 May 2026 23:09:08 -0700 Subject: [PATCH 061/142] chore(release): map EloquentBrush0x email for #26642 salvage --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index c388116cff6..1b9e4bcd8f3 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1096,6 +1096,7 @@ AUTHOR_MAP = { "4296245+matthewlai@users.noreply.github.com": "matthewlai", "109617724+0xchainer@users.noreply.github.com": "0xchainer", # PR #27154/27138/27147 salvage "201800237+kronexoi@users.noreply.github.com": "kronexoi", # PR #27167 salvage (Teams port fallback) + "283442588+EloquentBrush0x@users.noreply.github.com": "EloquentBrush0x", # PR #26642 salvage (post_setup parity) # batch salvage (May 2026 LHF run, group 2) "shellybotmoyer@example.com": "shellybotmoyer", # PR #26661 (kanban --severity >=) "coulson@shellybotmoyer.com": "shellybotmoyer", # PR #25576 (credential_pool ISO rehydrate) From ad00777f042d9c2ca23f1575ef1036a5b59d6195 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sat, 16 May 2026 03:28:52 +0300 Subject: [PATCH 062/142] fix(mcp-oauth): print SSH tunnel hint in _redirect_handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Hermes runs on a remote host over SSH, MCP OAuth loopback flows silently fail: the OAuth provider redirects the user's browser to http://127.0.0.1:/callback, which reaches the callback server on the *remote* machine — not the local machine where the browser is running. _redirect_handler already detected SSH (via _can_open_browser) and printed "Headless environment detected — open the URL manually." but gave no guidance on how to actually reach the callback server. Users got silent timeouts or "Could not establish connection" errors. This is the same bug fixed for xAI-oauth and Spotify in #26592, which added _print_loopback_ssh_hint() in hermes_cli/auth.py. mcp_oauth.py uses the identical loopback callback pattern (http://127.0.0.1:/callback via _configure_callback_port / _wait_for_callback) but was missing the hint. Fix: when SSH_CLIENT or SSH_TTY is set and _oauth_port is available, print the ssh -N -L port-forward command and the OAuth-over-SSH guide URL to stderr, consistent with the rest of _redirect_handler's output. Tests: 4 new cases in TestRedirectHandlerSshHint covering SSH_CLIENT, SSH_TTY, local session (no hint), and missing _oauth_port (no hint). --- tests/tools/test_mcp_oauth.py | 61 +++++++++++++++++++++++++++++++++++ tools/mcp_oauth.py | 17 ++++++++++ 2 files changed, 78 insertions(+) diff --git a/tests/tools/test_mcp_oauth.py b/tests/tools/test_mcp_oauth.py index 2dfebd80b9c..e12149a45d3 100644 --- a/tests/tools/test_mcp_oauth.py +++ b/tests/tools/test_mcp_oauth.py @@ -10,6 +10,8 @@ from unittest.mock import patch, MagicMock, AsyncMock import pytest +import asyncio + from tools.mcp_oauth import ( HermesTokenStorage, OAuthNonInteractiveError, @@ -20,6 +22,7 @@ from tools.mcp_oauth import ( _is_interactive, _wait_for_callback, _make_callback_handler, + _redirect_handler, ) @@ -241,6 +244,64 @@ class TestUtilities: assert _can_open_browser() is True +class TestRedirectHandlerSshHint: + """_redirect_handler must print an SSH tunnel hint on remote sessions.""" + + def _run(self, coro): + return asyncio.get_event_loop().run_until_complete(coro) + + def test_ssh_hint_shown_on_ssh_session(self, monkeypatch, capsys): + import tools.mcp_oauth as mco + monkeypatch.setattr(mco, "_oauth_port", 49200) + monkeypatch.setenv("SSH_CLIENT", "1.2.3.4 1234 22") + monkeypatch.delenv("SSH_TTY", raising=False) + monkeypatch.setattr(mco, "_can_open_browser", lambda: False) + + self._run(_redirect_handler("https://example.com/auth?foo=bar")) + + err = capsys.readouterr().err + assert "49200" in err + assert "ssh -N -L" in err + assert "Remote session detected" in err + + def test_ssh_hint_shown_via_ssh_tty(self, monkeypatch, capsys): + import tools.mcp_oauth as mco + monkeypatch.setattr(mco, "_oauth_port", 49201) + monkeypatch.delenv("SSH_CLIENT", raising=False) + monkeypatch.setenv("SSH_TTY", "/dev/pts/1") + monkeypatch.setattr(mco, "_can_open_browser", lambda: False) + + self._run(_redirect_handler("https://example.com/auth")) + + err = capsys.readouterr().err + assert "49201" in err + assert "ssh -N -L" in err + + def test_no_ssh_hint_on_local_session(self, monkeypatch, capsys): + import tools.mcp_oauth as mco + monkeypatch.setattr(mco, "_oauth_port", 49202) + monkeypatch.delenv("SSH_CLIENT", raising=False) + monkeypatch.delenv("SSH_TTY", raising=False) + monkeypatch.setattr(mco, "_can_open_browser", lambda: True) + monkeypatch.setattr("webbrowser.open", lambda url, **kw: True) + + self._run(_redirect_handler("https://example.com/auth")) + + err = capsys.readouterr().err + assert "ssh -N -L" not in err + + def test_no_ssh_hint_when_port_not_set(self, monkeypatch, capsys): + import tools.mcp_oauth as mco + monkeypatch.setattr(mco, "_oauth_port", None) + monkeypatch.setenv("SSH_CLIENT", "1.2.3.4 1234 22") + monkeypatch.setattr(mco, "_can_open_browser", lambda: False) + + self._run(_redirect_handler("https://example.com/auth")) + + err = capsys.readouterr().err + assert "ssh -N -L" not in err + + # --------------------------------------------------------------------------- # Path traversal protection # --------------------------------------------------------------------------- diff --git a/tools/mcp_oauth.py b/tools/mcp_oauth.py index d7bf135da47..8d48eedf0e8 100644 --- a/tools/mcp_oauth.py +++ b/tools/mcp_oauth.py @@ -401,6 +401,23 @@ async def _redirect_handler(authorization_url: str) -> None: ) print(msg, file=sys.stderr) + # On a remote SSH session the OAuth provider redirects to + # http://127.0.0.1:/callback, which reaches the callback server on + # the *remote* machine — not the user's local machine where the browser + # opened. Print a port-forward hint so the user knows to tunnel first. + if _oauth_port and (os.getenv("SSH_CLIENT") or os.getenv("SSH_TTY")): + print( + f" Remote session detected. The OAuth provider will redirect your browser to\n" + f" http://127.0.0.1:{_oauth_port}/callback\n" + f" which the callback listener on THIS machine is waiting on. If your browser\n" + f" is on a different machine, forward the port first in a separate terminal:\n" + f"\n" + f" ssh -N -L {_oauth_port}:127.0.0.1:{_oauth_port} @\n" + f"\n" + f" Then open the URL above. See: https://hermes-agent.nousresearch.com/docs/guides/oauth-over-ssh\n", + file=sys.stderr, + ) + if _can_open_browser(): try: opened = webbrowser.open(authorization_url) From 5fba236644a9c2aa18501fdef1484e5b6fecfb85 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:41 -0700 Subject: [PATCH 063/142] =?UTF-8?q?chore:=20ruff=20auto-fix=20PLR6201=20re?= =?UTF-8?q?sweep=20=E2=80=94=20tuple=20=E2=86=92=20set=20in=20membership?= =?UTF-8?q?=20tests=20(#27355)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six days after #23937 (608 fixes) the codebase had accumulated 241 new PLR6201 violations. Same mechanical `x in (...)` → `x in {...}` fix, same zero-risk profile: set lookup is O(1) vs O(n) for tuple and the two are semantically equivalent for hashable scalar membership tests. All 241 instances fixed via `ruff check --select PLR6201 --fix --unsafe-fixes`, zero remaining. Every changed value is a hashable scalar (str/int/None/enum/signal); no risk of unhashable runtime errors. No behavior change. Test plan: - 119 files changed, +244/-244 (net zero) — exactly one-line edits - `ruff check` clean afterward - Compile checks pass on the largest touched files (cli.py, run_agent.py, gateway/run.py, gateway/platforms/discord.py, model_tools.py) - Subset broad test run on tests/gateway/ tests/hermes_cli/ tests/agent/ tests/tools/: 18187 passed, 59 pre-existing failures (verified against origin/main with the same shape — identical failure count, identical category — all xdist test-order flakes unrelated to this change) Follows the same template as PR #23937 ([tracker: #23972](https://github.com/NousResearch/hermes-agent/issues/23972)). --- agent/lsp/client.py | 2 +- agent/lsp/install.py | 2 +- agent/lsp/manager.py | 2 +- agent/lsp/reporter.py | 2 +- agent/lsp/servers.py | 2 +- agent/transports/codex_app_server_session.py | 6 ++-- cli.py | 6 ++-- gateway/platforms/discord.py | 10 +++---- gateway/run.py | 2 +- hermes_cli/auth.py | 4 +-- hermes_cli/codex_runtime_switch.py | 4 +-- hermes_cli/dep_ensure.py | 2 +- hermes_cli/proxy/cli.py | 2 +- hermes_cli/proxy/server.py | 2 +- hermes_cli/runtime_provider.py | 2 +- hermes_cli/session_recap.py | 2 +- .../meme-generation/scripts/generate_meme.py | 4 +-- .../devops/watchers/scripts/watch_rss.py | 2 +- .../finance/stocks/scripts/stocks_client.py | 2 +- .../fitness-nutrition/scripts/body_calc.py | 6 ++-- .../scripts/openclaw_to_hermes.py | 20 ++++++------- .../telephony/scripts/telephony.py | 2 +- .../scripts/show_snapshot.py | 2 +- .../domain-intel/scripts/domain_intel.py | 2 +- .../osint-investigation/scripts/_http.py | 2 +- .../scripts/fetch_icij_offshore.py | 2 +- plugins/disk-cleanup/__init__.py | 2 +- plugins/google_meet/__init__.py | 2 +- plugins/google_meet/cli.py | 6 ++-- plugins/google_meet/meet_bot.py | 4 +-- plugins/google_meet/node/cli.py | 2 +- plugins/google_meet/realtime/openai_client.py | 2 +- plugins/google_meet/tools.py | 4 +-- plugins/kanban/dashboard/plugin_api.py | 6 ++-- plugins/memory/byterover/__init__.py | 4 +-- plugins/memory/hindsight/__init__.py | 12 ++++---- plugins/memory/honcho/__init__.py | 4 +-- plugins/memory/honcho/cli.py | 30 +++++++++---------- plugins/memory/honcho/client.py | 4 +-- plugins/memory/openviking/__init__.py | 10 +++---- plugins/memory/supermemory/__init__.py | 8 ++--- plugins/model-providers/deepseek/__init__.py | 4 +-- .../model-providers/kimi-coding/__init__.py | 2 +- plugins/platforms/google_chat/adapter.py | 10 +++---- plugins/platforms/irc/adapter.py | 14 ++++----- plugins/platforms/line/adapter.py | 8 ++--- plugins/platforms/simplex/adapter.py | 10 +++---- plugins/platforms/teams/adapter.py | 2 +- plugins/teams_pipeline/cli.py | 10 +++---- plugins/teams_pipeline/meetings.py | 4 +-- plugins/teams_pipeline/models.py | 2 +- plugins/teams_pipeline/runtime.py | 2 +- run_agent.py | 4 +-- skills/creative/comfyui/scripts/_common.py | 10 +++---- .../comfyui/scripts/extract_schema.py | 6 ++-- skills/creative/comfyui/scripts/fetch_logs.py | 2 +- .../comfyui/scripts/hardware_check.py | 2 +- .../creative/comfyui/scripts/run_workflow.py | 6 ++-- skills/creative/comfyui/scripts/ws_monitor.py | 2 +- .../comfyui/tests/test_cloud_integration.py | 2 +- .../comfyui/tests/test_extract_schema.py | 2 +- .../google-workspace/scripts/google_api.py | 2 +- .../productivity/maps/scripts/maps_client.py | 10 +++---- .../scripts/extract_marker.py | 2 +- .../scripts/extract_pymupdf.py | 2 +- skills/research/arxiv/scripts/search_arxiv.py | 2 +- .../research/polymarket/scripts/polymarket.py | 2 +- tests/agent/lsp/_mock_lsp_server.py | 2 +- .../agent/lsp/test_install_and_lint_fixes.py | 4 +-- tests/agent/test_anthropic_adapter.py | 4 +-- tests/agent/test_auxiliary_main_first.py | 2 +- tests/agent/test_context_compressor.py | 6 ++-- .../agent/test_deepseek_anthropic_thinking.py | 2 +- tests/cli/test_cli_init.py | 2 +- tests/cli/test_reasoning_command.py | 8 ++--- tests/cron/test_cron_no_agent.py | 4 +-- tests/gateway/conftest.py | 2 +- tests/gateway/test_allowlist_startup_check.py | 4 +-- tests/gateway/test_config_cwd_bridge.py | 4 +-- tests/gateway/test_discord_system_messages.py | 2 +- .../test_platform_connected_checkers.py | 4 +-- tests/gateway/test_qqbot.py | 2 +- tests/gateway/test_restart_resume_pending.py | 2 +- tests/gateway/test_session_boundary_hooks.py | 2 +- .../test_session_model_override_routing.py | 2 +- tests/gateway/test_transcript_offset.py | 2 +- tests/hermes_cli/test_auth_nous_provider.py | 2 +- tests/hermes_cli/test_cmd_update.py | 2 +- tests/hermes_cli/test_codex_runtime_switch.py | 2 +- tests/hermes_cli/test_install_cua_driver.py | 4 +-- .../test_kanban_core_functionality.py | 2 +- tests/hermes_cli/test_memory_reset.py | 4 +-- tests/hermes_cli/test_models.py | 4 +-- .../test_opencode_go_in_model_list.py | 2 +- .../hermes_cli/test_update_stale_dashboard.py | 2 +- tests/hermes_cli/test_web_server.py | 10 +++---- tests/honcho_plugin/test_session.py | 2 +- tests/plugins/test_achievements_plugin.py | 2 +- tests/plugins/video_gen/test_xai_plugin.py | 2 +- .../test_anthropic_truncation_continuation.py | 4 +-- tests/skills/test_openclaw_migration.py | 2 +- tests/stress/test_atypical_scenarios.py | 8 ++--- tests/test_live_system_guard_self_test.py | 2 +- tests/test_timezone.py | 2 +- tests/test_tui_gateway_server.py | 2 +- tests/tools/test_browser_homebrew_paths.py | 12 ++++---- tests/tools/test_code_execution_modes.py | 2 +- tests/tools/test_discord_tool.py | 2 +- tests/tools/test_hidden_dir_filter.py | 2 +- tests/tools/test_managed_modal_environment.py | 2 +- .../test_mcp_cancelled_error_propagation.py | 2 +- tests/tools/test_singularity_preflight.py | 2 +- tests/tools/test_skill_manager_tool.py | 2 +- tests/tools/test_skills_hub.py | 2 +- tests/tui_gateway/test_entry_sys_path.py | 10 +++---- tools/lazy_deps.py | 2 +- tools/mcp_tool.py | 2 +- tools/video_generation_tool.py | 4 +-- tools/x_search_tool.py | 2 +- 119 files changed, 244 insertions(+), 244 deletions(-) diff --git a/agent/lsp/client.py b/agent/lsp/client.py index 8f380fc7a60..06a92ae351b 100644 --- a/agent/lsp/client.py +++ b/agent/lsp/client.py @@ -232,7 +232,7 @@ class LSPClient: the process is killed and the client is left in state ``"error"`` — re-call ``start()`` to retry. """ - if self._state in ("running", "starting"): + if self._state in {"running", "starting"}: return self._state = "starting" try: diff --git a/agent/lsp/install.py b/agent/lsp/install.py index 0aaa22be744..d4a80ec195e 100644 --- a/agent/lsp/install.py +++ b/agent/lsp/install.py @@ -151,7 +151,7 @@ def try_install(pkg: str, strategy: str = "auto") -> Optional[str]: same path (or ``None``) without reinstalling. Concurrent calls are serialized. """ - if strategy not in ("auto",): + if strategy not in {"auto",}: # Only ``auto`` triggers an actual install. In manual/off, # we still check whether the binary already exists. recipe = INSTALL_RECIPES.get(pkg, {}) diff --git a/agent/lsp/manager.py b/agent/lsp/manager.py index 7f5feaa170f..4f16188de0b 100644 --- a/agent/lsp/manager.py +++ b/agent/lsp/manager.py @@ -162,7 +162,7 @@ class LSPService: idle_timeout: float = DEFAULT_IDLE_TIMEOUT, ) -> None: self._enabled = enabled - self._wait_mode = wait_mode if wait_mode in ("document", "full") else "document" + self._wait_mode = wait_mode if wait_mode in {"document", "full"} else "document" self._wait_timeout = wait_timeout self._install_strategy = install_strategy self._binary_overrides = binary_overrides or {} diff --git a/agent/lsp/reporter.py b/agent/lsp/reporter.py index fedad0d19b3..0eba96ba1ff 100644 --- a/agent/lsp/reporter.py +++ b/agent/lsp/reporter.py @@ -28,7 +28,7 @@ def format_diagnostic(d: Dict[str, Any]) -> str: col = int(start.get("character", 0)) + 1 msg = str(d.get("message") or "").rstrip() code = d.get("code") - code_part = f" [{code}]" if code not in (None, "") else "" + code_part = f" [{code}]" if code not in {None, ""} else "" source = d.get("source") source_part = f" ({source})" if source else "" return f"{sev} [{line}:{col}] {msg}{code_part}{source_part}" diff --git a/agent/lsp/servers.py b/agent/lsp/servers.py index 00ad4c40005..144b5cb2c11 100644 --- a/agent/lsp/servers.py +++ b/agent/lsp/servers.py @@ -237,7 +237,7 @@ def _spawn_pyright(root: str, ctx: ServerContext) -> Optional[SpawnSpec]: return None # If we got the cli ``pyright``, the langserver is its sibling. base = os.path.basename(bin_path) - if base in ("pyright", "pyright.exe"): + if base in {"pyright", "pyright.exe"}: sibling = os.path.join(os.path.dirname(bin_path), "pyright-langserver") if os.path.exists(sibling): bin_path = sibling diff --git a/agent/transports/codex_app_server_session.py b/agent/transports/codex_app_server_session.py index f0cd0a196c4..a72599ae719 100644 --- a/agent/transports/codex_app_server_session.py +++ b/agent/transports/codex_app_server_session.py @@ -541,7 +541,7 @@ class CodexAppServerSession: turn_status = ( (note.get("params") or {}).get("turn") or {} ).get("status") - if turn_status and turn_status not in ("completed", "interrupted"): + if turn_status and turn_status not in {"completed", "interrupted"}: err_obj = ( (note.get("params") or {}).get("turn") or {} ).get("error") @@ -775,9 +775,9 @@ def _approval_choice_to_codex_decision(choice: str) -> str: (verified against codex-rs/app-server-protocol/src/protocol/v2/item.rs on codex 0.130.0). """ - if choice in ("once",): + if choice in {"once",}: return "accept" - if choice in ("session", "always"): + if choice in {"session", "always"}: return "acceptForSession" return "decline" diff --git a/cli.py b/cli.py index 42b1482578e..e8e38965f53 100644 --- a/cli.py +++ b/cli.py @@ -1396,7 +1396,7 @@ def _detect_light_mode() -> bool: last = cfgbg.split(";")[-1] if ";" in cfgbg else cfgbg if last.isdigit(): bg = int(last) - if bg in (7, 15): + if bg in {7, 15}: result = True _LIGHT_MODE_CACHE = result return result @@ -7706,7 +7706,7 @@ class HermesCLI: # google-gemini/gemini-cli#19332. _rest = cmd_original.split(None, 1) _args = (_rest[1] if len(_rest) > 1 else "").strip().lower() - if _args in ("--delete", "-d"): + if _args in {"--delete", "-d"}: self._delete_session_on_exit = True elif _args: _cprint(f" {_DIM}✗ Unknown argument: {_escape(_args)}. Use /exit --delete to also remove session history.{_RST}") @@ -13835,7 +13835,7 @@ class HermesCLI: if _errno == errno.EIO: pass # suppress broken-stdout I/O errors on interrupt (#13710) elif ( - _errno in (errno.EINVAL, errno.EBADF) + _errno in {errno.EINVAL, errno.EBADF} or "is not registered" in _msg or "Bad file descriptor" in _msg or "Invalid argument" in _msg diff --git a/gateway/platforms/discord.py b/gateway/platforms/discord.py index 9b8285e2a36..f79678bc61a 100644 --- a/gateway/platforms/discord.py +++ b/gateway/platforms/discord.py @@ -3639,18 +3639,18 @@ class DiscordAdapter(BasePlatformAdapter): configured = self.config.extra.get("thread_require_mention") if configured is not None: if isinstance(configured, str): - return configured.lower() not in ("false", "0", "no", "off") + return configured.lower() not in {"false", "0", "no", "off"} return bool(configured) - return os.getenv("DISCORD_THREAD_REQUIRE_MENTION", "false").lower() in ("true", "1", "yes", "on") + return os.getenv("DISCORD_THREAD_REQUIRE_MENTION", "false").lower() in {"true", "1", "yes", "on"} def _discord_history_backfill(self) -> bool: """Return whether history backfill is enabled for shared sessions.""" configured = self.config.extra.get("history_backfill") if configured is not None: if isinstance(configured, str): - return configured.lower() not in ("false", "0", "no", "off") + return configured.lower() not in {"false", "0", "no", "off"} return bool(configured) - return os.getenv("DISCORD_HISTORY_BACKFILL", "true").lower() in ("true", "1", "yes") + return os.getenv("DISCORD_HISTORY_BACKFILL", "true").lower() in {"true", "1", "yes"} def _discord_history_backfill_limit(self) -> int: """Return the max number of messages to scan backwards for context. @@ -3737,7 +3737,7 @@ class DiscordAdapter(BasePlatformAdapter): break # Skip system messages (pins, joins, thread renames, etc.) - if msg.type not in (discord.MessageType.default, discord.MessageType.reply): + if msg.type not in {discord.MessageType.default, discord.MessageType.reply}: continue # Respect DISCORD_ALLOW_BOTS for other bots. diff --git a/gateway/run.py b/gateway/run.py index 81ce914b8ab..db7066281c3 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -8863,7 +8863,7 @@ class GatewayRunner: lines.append("Failed/paused: (none)") return "\n".join(lines) - if action in ("pause", "resume"): + if action in {"pause", "resume"}: if not target: return f"Usage: /platform {action} " platform = _resolve_platform(target) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 6cabb61570d..6752b65829f 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -2610,7 +2610,7 @@ def _print_loopback_ssh_hint(redirect_uri: str, *, docs_url: str | None = None) return host = parsed.hostname or "" port = parsed.port - if host not in ("127.0.0.1", "::1", "localhost") or not port: + if host not in {"127.0.0.1", "::1", "localhost"} or not port: return print() print("Remote session detected. Your browser will redirect to") @@ -5246,7 +5246,7 @@ def _login_xai_oauth( reuse = input("Use existing credentials? [Y/n]: ").strip().lower() except (EOFError, KeyboardInterrupt): reuse = "y" - if reuse in ("", "y", "yes"): + if reuse in {"", "y", "yes"}: config_path = _update_config_for_provider( "xai-oauth", existing.get("base_url", DEFAULT_XAI_OAUTH_BASE_URL), diff --git a/hermes_cli/codex_runtime_switch.py b/hermes_cli/codex_runtime_switch.py index b3adda12b54..98b40b1e8f2 100644 --- a/hermes_cli/codex_runtime_switch.py +++ b/hermes_cli/codex_runtime_switch.py @@ -48,9 +48,9 @@ def parse_args(arg_string: str) -> tuple[Optional[str], list[str]]: if not raw: return None, [] # Accept human-friendly synonyms - if raw in ("on", "codex", "enable"): + if raw in {"on", "codex", "enable"}: return "codex_app_server", [] - if raw in ("off", "default", "disable", "hermes"): + if raw in {"off", "default", "disable", "hermes"}: return "auto", [] if raw in VALID_RUNTIMES: return raw, [] diff --git a/hermes_cli/dep_ensure.py b/hermes_cli/dep_ensure.py index 3312726c36d..1067b428f7b 100644 --- a/hermes_cli/dep_ensure.py +++ b/hermes_cli/dep_ensure.py @@ -91,7 +91,7 @@ def ensure_dependency(dep: str, interactive: bool = True) -> bool: reply = input(f"{desc} is not installed. Install now? [Y/n] ").strip().lower() except (EOFError, KeyboardInterrupt): return False - if reply not in ("", "y", "yes"): + if reply not in {"", "y", "yes"}: return False result = subprocess.run( diff --git a/hermes_cli/proxy/cli.py b/hermes_cli/proxy/cli.py index 83c2d34035b..c35b14f7835 100644 --- a/hermes_cli/proxy/cli.py +++ b/hermes_cli/proxy/cli.py @@ -114,7 +114,7 @@ def cmd_proxy(args: Any) -> int: return cmd_proxy_start(args) if sub == "status": return cmd_proxy_status(args) - if sub in ("providers", "list"): + if sub in {"providers", "list"}: return cmd_proxy_list_providers(args) # No subcommand → print short help. print( diff --git a/hermes_cli/proxy/server.py b/hermes_cli/proxy/server.py index 48de784afe4..fa497f13291 100644 --- a/hermes_cli/proxy/server.py +++ b/hermes_cli/proxy/server.py @@ -76,7 +76,7 @@ def _filter_response_headers(headers) -> dict: if key.lower() in _HOP_BY_HOP_HEADERS: continue # aiohttp recomputes Content-Encoding/Content-Length on stream — let it. - if key.lower() in ("content-encoding", "content-length"): + if key.lower() in {"content-encoding", "content-length"}: continue out[key] = value return out diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index c0baf14db92..c186f1d6e7c 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -209,7 +209,7 @@ def _maybe_apply_codex_app_server_runtime( Returns the (possibly-rewritten) api_mode.""" if not model_cfg: return api_mode - if provider not in ("openai", "openai-codex"): + if provider not in {"openai", "openai-codex"}: return api_mode runtime = str(model_cfg.get("openai_runtime") or "").strip().lower() if runtime == "codex_app_server": diff --git a/hermes_cli/session_recap.py b/hermes_cli/session_recap.py index d67f737d799..111da117485 100644 --- a/hermes_cli/session_recap.py +++ b/hermes_cli/session_recap.py @@ -171,7 +171,7 @@ def _recent_window( cut = 0 for i in range(len(messages) - 1, -1, -1): msg = messages[i] - if isinstance(msg, Mapping) and msg.get("role") in ("user", "assistant"): + if isinstance(msg, Mapping) and msg.get("role") in {"user", "assistant"}: count += 1 if count >= window: cut = i diff --git a/optional-skills/creative/meme-generation/scripts/generate_meme.py b/optional-skills/creative/meme-generation/scripts/generate_meme.py index 288c3838367..807fee71165 100644 --- a/optional-skills/creative/meme-generation/scripts/generate_meme.py +++ b/optional-skills/creative/meme-generation/scripts/generate_meme.py @@ -358,7 +358,7 @@ def generate_meme(template_id: str, texts: list[str], output_path: str) -> str: img = _overlay_on_image(img, texts, fields) output = Path(output_path) - if output.suffix.lower() in (".jpg", ".jpeg"): + if output.suffix.lower() in {".jpg", ".jpeg"}: img = img.convert("RGB") img.save(str(output), quality=95) return str(output) @@ -378,7 +378,7 @@ def generate_from_image( result = _overlay_on_image(img, texts, fields) output = Path(output_path) - if output.suffix.lower() in (".jpg", ".jpeg"): + if output.suffix.lower() in {".jpg", ".jpeg"}: result = result.convert("RGB") result.save(str(output), quality=95) return str(output) diff --git a/optional-skills/devops/watchers/scripts/watch_rss.py b/optional-skills/devops/watchers/scripts/watch_rss.py index cc729f91b13..6e09630404f 100755 --- a/optional-skills/devops/watchers/scripts/watch_rss.py +++ b/optional-skills/devops/watchers/scripts/watch_rss.py @@ -43,7 +43,7 @@ def _parse_feed(xml_bytes: bytes): entries = [] for item in root.iter(): tag = _strip_ns(item.tag) - if tag not in ("item", "entry"): + if tag not in {"item", "entry"}: continue # ElementTree Elements without children are *falsy* — use `is not None`. children = {_strip_ns(c.tag): c for c in item} diff --git a/optional-skills/finance/stocks/scripts/stocks_client.py b/optional-skills/finance/stocks/scripts/stocks_client.py index 7b98fd9dc66..c0bf97dce4a 100755 --- a/optional-skills/finance/stocks/scripts/stocks_client.py +++ b/optional-skills/finance/stocks/scripts/stocks_client.py @@ -125,7 +125,7 @@ def fetch_url(url: str, headers: dict | None = None, retries: int = MAX_RETRIES) return json.loads(raw.decode("utf-8", errors="replace")) except urllib.error.HTTPError as e: last_err = e - if e.code in (404, 400): + if e.code in {404, 400}: break # no point retrying wait = BACKOFF_BASE ** attempt time.sleep(wait) diff --git a/optional-skills/health/fitness-nutrition/scripts/body_calc.py b/optional-skills/health/fitness-nutrition/scripts/body_calc.py index 2d07129cecc..2ce65fd336e 100644 --- a/optional-skills/health/fitness-nutrition/scripts/body_calc.py +++ b/optional-skills/health/fitness-nutrition/scripts/body_calc.py @@ -95,11 +95,11 @@ def one_rep_max(weight, reps): def macros(tdee_kcal, goal): goal = goal.lower() - if goal in ("cut", "lose", "deficit"): + if goal in {"cut", "lose", "deficit"}: cals = tdee_kcal - 500 p, f, c = 0.40, 0.30, 0.30 label = "Fat Loss (-500 kcal)" - elif goal in ("bulk", "gain", "surplus"): + elif goal in {"bulk", "gain", "surplus"}: cals = tdee_kcal + 400 p, f, c = 0.30, 0.25, 0.45 label = "Lean Bulk (+400 kcal)" @@ -184,7 +184,7 @@ def main(): int(sys.argv[4]), sys.argv[5], int(sys.argv[6]), ) - elif cmd in ("1rm", "orm"): + elif cmd in {"1rm", "orm"}: one_rep_max(float(sys.argv[2]), int(sys.argv[3])) elif cmd == "macros": diff --git a/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py b/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py index 6ebb1d75400..d9d53a97a24 100644 --- a/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py +++ b/optional-skills/migration/openclaw-migration/scripts/openclaw_to_hermes.py @@ -610,7 +610,7 @@ def _is_secret_key(key: str) -> bool: normalized = _normalize_secret_key(key) if normalized == "token" or normalized.endswith("token"): return True - if normalized in ("auth", "authorization"): + if normalized in {"auth", "authorization"}: return True return any(marker in normalized for marker in _SECRET_KEY_MARKERS) @@ -831,7 +831,7 @@ class Migrator: # Flip the config-block flag when a conflict/error occurs on a # config.yaml write. Later config-mutating options will skip rather # than attempting a partial write. - if status in (STATUS_CONFLICT, STATUS_ERROR) and destination is not None: + if status in {STATUS_CONFLICT, STATUS_ERROR} and destination is not None: dest_str = str(destination) if dest_str.endswith("config.yaml") or dest_str.endswith("config.yml"): self._config_apply_blocked = True @@ -1526,7 +1526,7 @@ class Migrator: api_key = resolve_secret_input(raw_key, openclaw_env) if not api_key: # Warn if a SecretRef with file/exec source was silently unresolvable - if isinstance(raw_key, dict) and raw_key.get("source") in ("file", "exec"): + if isinstance(raw_key, dict) and raw_key.get("source") in {"file", "exec"}: self.record( "provider-keys", self.source_root / "openclaw.json", @@ -1736,7 +1736,7 @@ class Migrator: tts_data: Dict[str, Any] = {} provider = tts.get("provider") - if isinstance(provider, str) and provider in ("elevenlabs", "openai", "edge", "microsoft"): + if isinstance(provider, str) and provider in {"elevenlabs", "openai", "edge", "microsoft"}: # OpenClaw renamed "edge" to "microsoft"; Hermes still uses "edge" tts_data["provider"] = "edge" if provider == "microsoft" else provider @@ -2304,11 +2304,11 @@ class Migrator: if defaults.get("thinkingDefault"): # Map OpenClaw thinking -> Hermes reasoning_effort thinking = defaults["thinkingDefault"] - if thinking in ("always", "high", "xhigh"): + if thinking in {"always", "high", "xhigh"}: agent_cfg["reasoning_effort"] = "high" - elif thinking in ("auto", "medium", "adaptive"): + elif thinking in {"auto", "medium", "adaptive"}: agent_cfg["reasoning_effort"] = "medium" - elif thinking in ("off", "low", "none", "minimal"): + elif thinking in {"off", "low", "none", "minimal"}: agent_cfg["reasoning_effort"] = "low" changes = True @@ -2626,8 +2626,8 @@ class Migrator: if not isinstance(ch_cfg, dict): continue complex_keys = {k: v for k, v in ch_cfg.items() - if k not in ("botToken", "appToken", "allowFrom", "enabled") - and v and k not in ("requireMention", "autoThread")} + if k not in {"botToken", "appToken", "allowFrom", "enabled"} + and v and k not in {"requireMention", "autoThread"}} if complex_keys: complex_archive[ch_name] = complex_keys @@ -2671,7 +2671,7 @@ class Migrator: # Archive remaining browser settings advanced = {k: v for k, v in browser.items() - if k not in ("cdpUrl", "headless") and v} + if k not in {"cdpUrl", "headless"} and v} if advanced and self.archive_dir: if self.execute: self.archive_dir.mkdir(parents=True, exist_ok=True) diff --git a/optional-skills/productivity/telephony/scripts/telephony.py b/optional-skills/productivity/telephony/scripts/telephony.py index c9233647f3f..188b6be2ad9 100644 --- a/optional-skills/productivity/telephony/scripts/telephony.py +++ b/optional-skills/productivity/telephony/scripts/telephony.py @@ -109,7 +109,7 @@ def _config_lookup(*paths: tuple[str, ...], default: str = "") -> str: node = None break node = node.get(key) - if node not in (None, "") and not isinstance(node, dict): + if node not in {None, ""} and not isinstance(node, dict): return str(node) return default diff --git a/optional-skills/research/darwinian-evolver/scripts/show_snapshot.py b/optional-skills/research/darwinian-evolver/scripts/show_snapshot.py index 10e3a03dca9..5dd559570dd 100644 --- a/optional-skills/research/darwinian-evolver/scripts/show_snapshot.py +++ b/optional-skills/research/darwinian-evolver/scripts/show_snapshot.py @@ -51,7 +51,7 @@ def main() -> int: field = args.field if field is None: for k, v in vars(org).items(): - if isinstance(v, str) and not k.startswith("_") and k not in ("id",): + if isinstance(v, str) and not k.startswith("_") and k not in {"id",}: field = k break val = getattr(org, field, None) if field else None diff --git a/optional-skills/research/domain-intel/scripts/domain_intel.py b/optional-skills/research/domain-intel/scripts/domain_intel.py index 1a69f6528f2..c25e9286d40 100644 --- a/optional-skills/research/domain-intel/scripts/domain_intel.py +++ b/optional-skills/research/domain-intel/scripts/domain_intel.py @@ -185,7 +185,7 @@ def whois_lookup(domain): for key, pat in patterns.items(): matches = re.findall(pat, raw, re.IGNORECASE) if matches: - if key in ("name_servers", "status"): + if key in {"name_servers", "status"}: result[key] = list(dict.fromkeys(m.strip().lower() for m in matches)) else: result[key] = matches[0].strip() diff --git a/optional-skills/research/osint-investigation/scripts/_http.py b/optional-skills/research/osint-investigation/scripts/_http.py index 5da62310b9f..0936548a92a 100644 --- a/optional-skills/research/osint-investigation/scripts/_http.py +++ b/optional-skills/research/osint-investigation/scripts/_http.py @@ -60,7 +60,7 @@ def get( f"HTTP 429 rate-limited by {urllib.parse.urlsplit(url).netloc}. " f"Slow down or supply a real API key. Body: {body[:300]}" ) from e - if e.code in (500, 502, 503, 504) and attempt < max_retries: + if e.code in {500, 502, 503, 504} and attempt < max_retries: retry_after = e.headers.get("Retry-After") if e.headers else None wait = float(retry_after) if (retry_after and retry_after.isdigit()) else backoff ** (attempt + 1) time.sleep(wait) diff --git a/optional-skills/research/osint-investigation/scripts/fetch_icij_offshore.py b/optional-skills/research/osint-investigation/scripts/fetch_icij_offshore.py index 8d050b62bf1..3108681e20c 100644 --- a/optional-skills/research/osint-investigation/scripts/fetch_icij_offshore.py +++ b/optional-skills/research/osint-investigation/scripts/fetch_icij_offshore.py @@ -122,7 +122,7 @@ def fetch( with zipfile.ZipFile(zip_path) as zf: for node_type, csv_substring in targets: - relevant_needles = [n for (k, n) in needles if k in (node_type, "Entity", "Officer")] or [] + relevant_needles = [n for (k, n) in needles if k in {node_type, "Entity", "Officer"}] or [] # Only scan a CSV if we have a needle that could plausibly match it, # or if we have ONLY a jurisdiction filter. applicable_needles = [n for (k, n) in needles if k == node_type] diff --git a/plugins/disk-cleanup/__init__.py b/plugins/disk-cleanup/__init__.py index 0a4b6c7ae16..71d44b1c891 100644 --- a/plugins/disk-cleanup/__init__.py +++ b/plugins/disk-cleanup/__init__.py @@ -222,7 +222,7 @@ def _fmt_summary(summary: Dict[str, Any]) -> str: def _handle_slash(raw_args: str) -> Optional[str]: argv = raw_args.strip().split() - if not argv or argv[0] in ("help", "-h", "--help"): + if not argv or argv[0] in {"help", "-h", "--help"}: return _HELP_TEXT sub = argv[0] diff --git a/plugins/google_meet/__init__.py b/plugins/google_meet/__init__.py index feca75667b5..df401e1a680 100644 --- a/plugins/google_meet/__init__.py +++ b/plugins/google_meet/__init__.py @@ -72,7 +72,7 @@ def register(ctx) -> None: # tested path there and guest-join Chromium is flakier. Refuse to register # rather than half-working. system = platform.system().lower() - if system not in ("linux", "darwin"): + if system not in {"linux", "darwin"}: logger.info( "google_meet plugin: platform=%s not supported (linux/macos only)", system, diff --git a/plugins/google_meet/cli.py b/plugins/google_meet/cli.py index b7d8097fc76..0e9b08881b3 100644 --- a/plugins/google_meet/cli.py +++ b/plugins/google_meet/cli.py @@ -159,7 +159,7 @@ def _cmd_setup() -> int: print("---------------------") system = _p.system() - system_ok = system in ("Linux", "Darwin") + system_ok = system in {"Linux", "Darwin"} print(f" platform : {system} [{'ok' if system_ok else 'unsupported'}]") try: @@ -231,7 +231,7 @@ def _cmd_install(*, realtime: bool, assume_yes: bool) -> int: import subprocess as _sp system = _p.system() - if system not in ("Linux", "Darwin"): + if system not in {"Linux", "Darwin"}: print(f"google_meet install: {system} is not supported (linux/macos only)") return 1 @@ -242,7 +242,7 @@ def _cmd_install(*, realtime: bool, assume_yes: bool) -> int: ans = input(f"{prompt} [y/N] ").strip().lower() except EOFError: return False - return ans in ("y", "yes") + return ans in {"y", "yes"} print("google_meet install") print("-------------------") diff --git a/plugins/google_meet/meet_bot.py b/plugins/google_meet/meet_bot.py index eb9318ae4a5..9040d9a789a 100644 --- a/plugins/google_meet/meet_bot.py +++ b/plugins/google_meet/meet_bot.py @@ -447,7 +447,7 @@ def _mac_audio_device_index(device_name: str) -> str: def run_bot() -> int: # noqa: C901 — orchestration, explicit branches url = os.environ.get("HERMES_MEET_URL", "").strip() out_dir_env = os.environ.get("HERMES_MEET_OUT_DIR", "").strip() - headed = os.environ.get("HERMES_MEET_HEADED", "").lower() in ("1", "true", "yes") + headed = os.environ.get("HERMES_MEET_HEADED", "").lower() in {"1", "true", "yes"} auth_state = os.environ.get("HERMES_MEET_AUTH_STATE", "").strip() guest_name = os.environ.get("HERMES_MEET_GUEST_NAME", "Hermes Agent") duration_s = _parse_duration(os.environ.get("HERMES_MEET_DURATION", "")) @@ -808,7 +808,7 @@ def _looks_like_human_speaker(speaker: str, bot_guest_name: str) -> bool: if not speaker or not speaker.strip(): return False spk = speaker.strip().lower() - if spk in ("unknown", "you", bot_guest_name.strip().lower()): + if spk in {"unknown", "you", bot_guest_name.strip().lower()}: return False return True diff --git a/plugins/google_meet/node/cli.py b/plugins/google_meet/node/cli.py index 4e10161e0cc..255b851ba6a 100644 --- a/plugins/google_meet/node/cli.py +++ b/plugins/google_meet/node/cli.py @@ -103,7 +103,7 @@ def node_command(args: argparse.Namespace) -> int: print(f"removed {args.name!r}" if ok else f"no such node: {args.name!r}") return 0 if ok else 1 - if cmd in ("status", "ping"): + if cmd in {"status", "ping"}: entry = reg.get(args.name) if entry is None: print(f"no such node: {args.name!r}", file=sys.stderr) diff --git a/plugins/google_meet/realtime/openai_client.py b/plugins/google_meet/realtime/openai_client.py index e9738d106ae..24527603e52 100644 --- a/plugins/google_meet/realtime/openai_client.py +++ b/plugins/google_meet/realtime/openai_client.py @@ -183,7 +183,7 @@ class RealtimeSession: rid = (frame.get("response") or {}).get("id") if rid: self._last_response_id = rid - elif ftype in ("response.done", "response.completed", "response.cancelled"): + elif ftype in {"response.done", "response.completed", "response.cancelled"}: break elif ftype == "error": err = frame.get("error") or frame diff --git a/plugins/google_meet/tools.py b/plugins/google_meet/tools.py index 9af804288c7..034116b88af 100644 --- a/plugins/google_meet/tools.py +++ b/plugins/google_meet/tools.py @@ -36,7 +36,7 @@ def check_meet_requirements() -> bool: handlers relax the requirement when a node is addressed. """ import platform as _p - if _p.system().lower() not in ("linux", "darwin"): + if _p.system().lower() not in {"linux", "darwin"}: return False try: import playwright # noqa: F401 @@ -238,7 +238,7 @@ def handle_meet_join(args: Dict[str, Any], **_kw) -> str: if not url: return _err("url is required") mode = (args.get("mode") or "transcribe").strip().lower() - if mode not in ("transcribe", "realtime"): + if mode not in {"transcribe", "realtime"}: return _err(f"mode must be 'transcribe' or 'realtime' (got {mode!r})") node = args.get("node") diff --git a/plugins/kanban/dashboard/plugin_api.py b/plugins/kanban/dashboard/plugin_api.py index 7b0cb1d791a..08824e3807b 100644 --- a/plugins/kanban/dashboard/plugin_api.py +++ b/plugins/kanban/dashboard/plugin_api.py @@ -628,7 +628,7 @@ def update_task(task_id: str, payload: UpdateTaskBody, board: Optional[str] = Qu status_code=400, detail="Cannot set status to 'running' directly; use the dispatcher/claim path", ) - elif s in ("todo", "triage"): + elif s in {"todo", "triage"}: ok = _set_status_direct(conn, task_id, s) else: raise HTTPException(status_code=400, detail=f"unknown status: {s}") @@ -742,7 +742,7 @@ def _set_status_direct( (task_id, run_id, json.dumps({"status": new_status}), int(time.time())), ) # If we re-opened something, children may have gone stale. - if new_status in ("done", "ready"): + if new_status in {"done", "ready"}: kanban_db.recompute_ready(conn) return True @@ -868,7 +868,7 @@ def bulk_update(payload: BulkTaskBody, board: Optional[str] = Query(None)): ok = kanban_db.unblock_task(conn, tid) else: ok = _set_status_direct(conn, tid, "ready") - elif s in ("todo", "running", "triage"): + elif s in {"todo", "running", "triage"}: ok = _set_status_direct(conn, tid, s) else: entry.update(ok=False, error=f"unknown status {s!r}") diff --git a/plugins/memory/byterover/__init__.py b/plugins/memory/byterover/__init__.py index 1870e9ab865..eafd9b2cfe5 100644 --- a/plugins/memory/byterover/__init__.py +++ b/plugins/memory/byterover/__init__.py @@ -263,7 +263,7 @@ class ByteRoverMemoryProvider(MemoryProvider): def on_memory_write(self, action: str, target: str, content: str) -> None: """Mirror built-in memory writes to ByteRover.""" - if action not in ("add", "replace") or not content: + if action not in {"add", "replace"} or not content: return def _write(): @@ -289,7 +289,7 @@ class ByteRoverMemoryProvider(MemoryProvider): for msg in messages[-10:]: # last 10 messages role = msg.get("role", "") content = msg.get("content", "") - if isinstance(content, str) and content.strip() and role in ("user", "assistant"): + if isinstance(content, str) and content.strip() and role in {"user", "assistant"}: parts.append(f"{role}: {content[:500]}") if not parts: diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index 52b1ac247f1..40772f79d8a 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -416,7 +416,7 @@ def _build_embedded_profile_env(config: dict[str, Any], *, llm_api_key: str | No current_base_url = config.get("llm_base_url") or os.environ.get("HINDSIGHT_API_LLM_BASE_URL", "") # The embedded daemon expects OpenAI wire format for these providers. - daemon_provider = "openai" if current_provider in ("openai_compatible", "openrouter") else current_provider + daemon_provider = "openai" if current_provider in {"openai_compatible", "openrouter"} else current_provider env_values = { "HINDSIGHT_API_LLM_PROVIDER": str(daemon_provider), @@ -596,7 +596,7 @@ class HindsightMemoryProvider(MemoryProvider): try: cfg = _load_config() mode = cfg.get("mode", "cloud") - if mode in ("local", "local_embedded"): + if mode in {"local", "local_embedded"}: available, _ = _check_local_runtime() return available if mode == "local_external": @@ -888,7 +888,7 @@ class HindsightMemoryProvider(MemoryProvider): from hindsight import HindsightEmbedded HindsightEmbedded.__del__ = lambda self: None llm_provider = self._config.get("llm_provider", "") - if llm_provider in ("openai_compatible", "openrouter"): + if llm_provider in {"openai_compatible", "openrouter"}: llm_provider = "openai" logger.debug("Creating HindsightEmbedded client (profile=%s, provider=%s)", self._config.get("profile", "hermes"), llm_provider) @@ -1132,7 +1132,7 @@ class HindsightMemoryProvider(MemoryProvider): self._mode = "disabled" return self._api_key = self._config.get("apiKey") or self._config.get("api_key") or os.environ.get("HINDSIGHT_API_KEY", "") - default_url = _DEFAULT_LOCAL_URL if self._mode in ("local_embedded", "local_external") else _DEFAULT_API_URL + default_url = _DEFAULT_LOCAL_URL if self._mode in {"local_embedded", "local_external"} else _DEFAULT_API_URL self._api_url = self._config.get("api_url") or os.environ.get("HINDSIGHT_API_URL", default_url) self._llm_base_url = self._config.get("llm_base_url", "") @@ -1152,10 +1152,10 @@ class HindsightMemoryProvider(MemoryProvider): self._budget = budget if budget in _VALID_BUDGETS else "mid" memory_mode = self._config.get("memory_mode", "hybrid") - self._memory_mode = memory_mode if memory_mode in ("context", "tools", "hybrid") else "hybrid" + self._memory_mode = memory_mode if memory_mode in {"context", "tools", "hybrid"} else "hybrid" prefetch_method = self._config.get("recall_prefetch_method") or self._config.get("prefetch_method", "recall") - self._prefetch_method = prefetch_method if prefetch_method in ("recall", "reflect") else "recall" + self._prefetch_method = prefetch_method if prefetch_method in {"recall", "reflect"} else "recall" # Bank options self._bank_mission = self._config.get("bank_mission", "") diff --git a/plugins/memory/honcho/__init__.py b/plugins/memory/honcho/__init__.py index d97f459acef..efbba937a4d 100644 --- a/plugins/memory/honcho/__init__.py +++ b/plugins/memory/honcho/__init__.py @@ -283,7 +283,7 @@ class HonchoMemoryProvider(MemoryProvider): # ----- Port #4053: cron guard ----- agent_context = kwargs.get("agent_context", "") platform = kwargs.get("platform", "cli") - if agent_context in ("cron", "flush") or platform == "cron": + if agent_context in {"cron", "flush"} or platform == "cron": logger.debug("Honcho skipped: cron/flush context (agent_context=%s, platform=%s)", agent_context, platform) self._cron_skipped = True @@ -404,7 +404,7 @@ class HonchoMemoryProvider(MemoryProvider): # pop_context_result() in prefetch(). Dialectic prewarm runs the # full configured depth and writes into _prefetch_result so turn 1 # consumes the result directly. - if self._recall_mode in ("context", "hybrid"): + if self._recall_mode in {"context", "hybrid"}: try: self._manager.prefetch_context(self._session_key) except Exception as e: diff --git a/plugins/memory/honcho/cli.py b/plugins/memory/honcho/cli.py index 402389ab962..28f213a1a66 100644 --- a/plugins/memory/honcho/cli.py +++ b/plugins/memory/honcho/cli.py @@ -233,7 +233,7 @@ _profile_override: str | None = None def _host_key() -> str: """Return the active Honcho host key, derived from the current Hermes profile.""" if _profile_override: - if _profile_override in ("default", "custom"): + if _profile_override in {"default", "custom"}: return HOST return f"{HOST}.{_profile_override}" return resolve_active_host() @@ -295,13 +295,13 @@ def _resolve_api_key(cfg: dict) -> str: parsed = urlparse(base_url) except (TypeError, ValueError): parsed = None - if parsed and parsed.scheme in ("http", "https") and parsed.netloc: + if parsed and parsed.scheme in {"http", "https"} and parsed.netloc: return "local" # Schemeless but looks like a host (contains '.' or ':' and isn't # a boolean literal): let it through so legacy configs don't # regress into "no API key configured" when they previously worked. lowered = base_url.lower() - if lowered not in ("true", "false", "none", "null") and any( + if lowered not in {"true", "false", "none", "null"} and any( c in base_url for c in ".:" ) and not base_url.isdigit(): return "local" @@ -334,7 +334,7 @@ def _ensure_sdk_installed() -> bool: print(" honcho-ai is not installed.") answer = _prompt("Install it now? (honcho-ai>=2.0.1)", default="y") - if answer.lower() not in ("y", "yes"): + if answer.lower() not in {"y", "yes"}: print(" Skipping install. Run: pip install 'honcho-ai>=2.0.1'\n") return False @@ -382,7 +382,7 @@ def cmd_setup(args) -> None: for h in ("localhost", "127.0.0.1", "::1") ) else "cloud" deploy = _prompt("Cloud or local?", default=current_deploy) - is_local = deploy.lower() in ("local", "l") + is_local = deploy.lower() in {"local", "l"} # Clean up legacy snake_case key cfg.pop("base_url", None) @@ -441,7 +441,7 @@ def cmd_setup(args) -> None: print(" directional -- all observations on, each AI peer builds its own view (default)") print(" unified -- shared pool, user observes self, AI observes others only") new_obs = _prompt("Observation mode", default=current_obs) - if new_obs in ("unified", "directional"): + if new_obs in {"unified", "directional"}: hermes_host["observationMode"] = new_obs else: hermes_host["observationMode"] = "directional" @@ -457,17 +457,17 @@ def cmd_setup(args) -> None: try: hermes_host["writeFrequency"] = int(new_wf) except (ValueError, TypeError): - hermes_host["writeFrequency"] = new_wf if new_wf in ("async", "turn", "session") else "async" + hermes_host["writeFrequency"] = new_wf if new_wf in {"async", "turn", "session"} else "async" # --- 6. Recall mode --- _raw_recall = hermes_host.get("recallMode") or cfg.get("recallMode", "hybrid") - current_recall = "hybrid" if _raw_recall not in ("hybrid", "context", "tools") else _raw_recall + current_recall = "hybrid" if _raw_recall not in {"hybrid", "context", "tools"} else _raw_recall print("\n Recall mode:") print(" hybrid -- auto-injected context + Honcho tools available (default)") print(" context -- auto-injected context only, Honcho tools hidden") print(" tools -- Honcho tools only, no auto-injected context") new_recall = _prompt("Recall mode", default=current_recall) - if new_recall in ("hybrid", "context", "tools"): + if new_recall in {"hybrid", "context", "tools"}: hermes_host["recallMode"] = new_recall # --- 7. Context token budget --- @@ -477,7 +477,7 @@ def cmd_setup(args) -> None: print(" uncapped -- no limit (default)") print(" N -- token limit per turn (e.g. 1200)") new_ctx_tokens = _prompt("Context tokens", default=current_display) - if new_ctx_tokens.strip().lower() in ("none", "uncapped", "no limit"): + if new_ctx_tokens.strip().lower() in {"none", "uncapped", "no limit"}: hermes_host.pop("contextTokens", None) elif new_ctx_tokens.strip() == "": pass # keep current @@ -517,7 +517,7 @@ def cmd_setup(args) -> None: print(" high -- complex behavioral patterns") print(" max -- thorough audit-level analysis") new_reasoning = _prompt("Reasoning level", default=current_reasoning) - if new_reasoning in ("minimal", "low", "medium", "high", "max"): + if new_reasoning in {"minimal", "low", "medium", "high", "max"}: hermes_host["dialecticReasoningLevel"] = new_reasoning else: hermes_host["dialecticReasoningLevel"] = "low" @@ -530,7 +530,7 @@ def cmd_setup(args) -> None: print(" per-repo -- one session per git repository") print(" global -- single session across all directories") new_strat = _prompt("Session strategy", default=current_strat) - if new_strat in ("per-session", "per-repo", "per-directory", "global"): + if new_strat in {"per-session", "per-repo", "per-directory", "global"}: hermes_host["sessionStrategy"] = new_strat hermes_host["enabled"] = True @@ -1130,7 +1130,7 @@ def cmd_migrate(args) -> None: print(" Paste the key when prompted.") print() answer = _prompt(" Run 'hermes honcho setup' now?", default="y") - if answer.lower() in ("y", "yes"): + if answer.lower() in {"y", "yes"}: cmd_setup(args) cfg = _read_config() has_key = bool(cfg.get("apiKey", "")) @@ -1176,7 +1176,7 @@ def cmd_migrate(args) -> None: print(" hermes honcho migrate — this step handles it interactively") if has_key: answer = _prompt(" Upload user memory files to Honcho now?", default="y") - if answer.lower() in ("y", "yes"): + if answer.lower() in {"y", "yes"}: try: from plugins.memory.honcho.client import ( HonchoClientConfig, @@ -1226,7 +1226,7 @@ def cmd_migrate(args) -> None: print() if has_key: answer = _prompt(" Seed AI identity from all detected files now?", default="y") - if answer.lower() in ("y", "yes"): + if answer.lower() in {"y", "yes"}: try: from plugins.memory.honcho.client import ( HonchoClientConfig, diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index de34642911e..eb268216c9b 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -47,7 +47,7 @@ def resolve_active_host() -> str: try: from hermes_cli.profiles import get_active_profile_name profile = get_active_profile_name() - if profile and profile not in ("default", "custom"): + if profile and profile not in {"default", "custom"}: return f"{HOST}.{profile}" except Exception: pass @@ -653,7 +653,7 @@ class HonchoClientConfig: return base # per-directory: one Honcho session per working directory (default) - if self.session_strategy in ("per-directory", "per-session"): + if self.session_strategy in {"per-directory", "per-session"}: base = Path(cwd).name if self.session_peer_prefix and self.peer_name: return f"{self.peer_name}-{base}" diff --git a/plugins/memory/openviking/__init__.py b/plugins/memory/openviking/__init__.py index ecb02b3de7e..ff01bbf402e 100644 --- a/plugins/memory/openviking/__init__.py +++ b/plugins/memory/openviking/__init__.py @@ -357,7 +357,7 @@ def _is_windows_absolute_path(value: str) -> bool: len(value) >= 3 and value[0].isalpha() and value[1] == ":" - and value[2] in ("/", "\\") + and value[2] in {"/", "\\"} ) @@ -381,7 +381,7 @@ def _is_local_path_reference(value: str) -> bool: def _path_from_file_uri(uri: str) -> Path | str: parsed = urlparse(uri) - if parsed.netloc not in ("", "localhost"): + if parsed.netloc not in {"", "localhost"}: return f"Unsupported non-local file URI: {uri}" return Path(url2pathname(parsed.path)).expanduser() @@ -755,7 +755,7 @@ class OpenVikingMemoryProvider(MemoryProvider): level = args.get("level", "overview") - summary_level = level in ("abstract", "overview") + summary_level = level in {"abstract", "overview"} # OpenViking expects directory URIs for pseudo summary files # (e.g. viking://user/hermes/.overview.md). resolved_uri = self._normalize_summary_uri(uri) if summary_level else uri @@ -832,7 +832,7 @@ class OpenVikingMemoryProvider(MemoryProvider): result = self._unwrap_result(resp) # Format list/tree results for readability - if action in ("list", "tree"): + if action in {"list", "tree"}: raw_entries = result if isinstance(result, dict): raw_entries = result.get("entries") or result.get("items") or result.get("children") or [] @@ -887,7 +887,7 @@ class OpenVikingMemoryProvider(MemoryProvider): payload: Dict[str, Any] = {} for key in ("reason", "to", "parent", "instruction", "wait", "timeout"): - if key in args and args[key] not in (None, ""): + if key in args and args[key] not in {None, ""}: payload[key] = args[key] parsed_url = urlparse(url) diff --git a/plugins/memory/supermemory/__init__.py b/plugins/memory/supermemory/__init__.py index f0cbfd60276..35b5b6fd649 100644 --- a/plugins/memory/supermemory/__init__.py +++ b/plugins/memory/supermemory/__init__.py @@ -88,9 +88,9 @@ def _as_bool(value: Any, default: bool) -> bool: return value if isinstance(value, str): lowered = value.strip().lower() - if lowered in ("true", "1", "yes", "y", "on"): + if lowered in {"true", "1", "yes", "y", "on"}: return True - if lowered in ("false", "0", "no", "n", "off"): + if lowered in {"false", "0", "no", "n", "off"}: return False return default @@ -508,7 +508,7 @@ class SupermemoryMemoryProvider(MemoryProvider): self._allowed_containers = [self._container_tag] + list(self._custom_containers) agent_context = kwargs.get("agent_context", "") - self._write_enabled = agent_context not in ("cron", "flush", "subagent") + self._write_enabled = agent_context not in {"cron", "flush", "subagent"} self._active = bool(self._api_key) self._client = None if self._active: @@ -598,7 +598,7 @@ class SupermemoryMemoryProvider(MemoryProvider): cleaned = [] for message in messages or []: role = message.get("role") - if role not in ("user", "assistant"): + if role not in {"user", "assistant"}: continue content = _clean_text_for_capture(str(message.get("content", ""))) if content: diff --git a/plugins/model-providers/deepseek/__init__.py b/plugins/model-providers/deepseek/__init__.py index 525766f87eb..34a8017b76e 100644 --- a/plugins/model-providers/deepseek/__init__.py +++ b/plugins/model-providers/deepseek/__init__.py @@ -74,9 +74,9 @@ class DeepSeekProfile(ProviderProfile): # its server default (currently high). if isinstance(reasoning_config, dict): effort = (reasoning_config.get("effort") or "").strip().lower() - if effort in ("xhigh", "max"): + if effort in {"xhigh", "max"}: top_level["reasoning_effort"] = "max" - elif effort in ("low", "medium", "high"): + elif effort in {"low", "medium", "high"}: top_level["reasoning_effort"] = effort return extra_body, top_level diff --git a/plugins/model-providers/kimi-coding/__init__.py b/plugins/model-providers/kimi-coding/__init__.py index b5cf53a8010..ed96ec514ef 100644 --- a/plugins/model-providers/kimi-coding/__init__.py +++ b/plugins/model-providers/kimi-coding/__init__.py @@ -37,7 +37,7 @@ class KimiProfile(ProviderProfile): # Enabled extra_body["thinking"] = {"type": "enabled"} effort = (reasoning_config.get("effort") or "").strip().lower() - if effort in ("low", "medium", "high"): + if effort in {"low", "medium", "high"}: top_level["reasoning_effort"] = effort else: top_level["reasoning_effort"] = "medium" diff --git a/plugins/platforms/google_chat/adapter.py b/plugins/platforms/google_chat/adapter.py index 1520d6664eb..0fdf1ea9d86 100644 --- a/plugins/platforms/google_chat/adapter.py +++ b/plugins/platforms/google_chat/adapter.py @@ -1539,7 +1539,7 @@ class GoogleChatAdapter(BasePlatformAdapter): if sender_email and space_name: self._last_sender_by_chat[space_name] = sender_email.strip().lower() - chat_type = "dm" if space_type in ("DIRECT_MESSAGE", "DM") else "group" + chat_type = "dm" if space_type in {"DIRECT_MESSAGE", "DM"} else "group" text = msg.get("argumentText") or msg.get("text") or "" text = text.strip() @@ -1935,7 +1935,7 @@ class GoogleChatAdapter(BasePlatformAdapter): return True except HttpError as exc: status = getattr(getattr(exc, "resp", None), "status", None) - if status in (403, 404): + if status in {403, 404}: return False logger.debug( "[GoogleChat] delete_message failed: %s", @@ -1958,7 +1958,7 @@ class GoogleChatAdapter(BasePlatformAdapter): update_mask = ",".join(update_mask_fields) or "text" # Patch body cannot carry thread (immutable). - patch_body = {k: v for k, v in body.items() if k not in ("thread",)} + patch_body = {k: v for k, v in body.items() if k not in {"thread",}} def _do_patch() -> Dict[str, Any]: return ( @@ -2791,7 +2791,7 @@ class GoogleChatAdapter(BasePlatformAdapter): upload_resp = await asyncio.to_thread(_upload) except HttpError as exc: status = getattr(getattr(exc, "resp", None), "status", None) - if status in (401, 403): + if status in {401, 403}: logger.warning( "[GoogleChat] media.upload auth failure for identity=%s " "(token revoked or scope missing) — falling back to " @@ -2927,7 +2927,7 @@ class GoogleChatAdapter(BasePlatformAdapter): display = info.get("displayName") or chat_id return { "name": display, - "type": "dm" if space_type in ("DIRECT_MESSAGE", "DM") else "group", + "type": "dm" if space_type in {"DIRECT_MESSAGE", "DM"} else "group", "chat_id": chat_id, } diff --git a/plugins/platforms/irc/adapter.py b/plugins/platforms/irc/adapter.py index ff10475d4e1..3358fa5b188 100644 --- a/plugins/platforms/irc/adapter.py +++ b/plugins/platforms/irc/adapter.py @@ -112,7 +112,7 @@ class IRCAdapter(BasePlatformAdapter): self.nickname = os.getenv("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") self.channel = os.getenv("IRC_CHANNEL") or extra.get("channel", "") self.use_tls = ( - os.getenv("IRC_USE_TLS", "").lower() in ("1", "true", "yes") + os.getenv("IRC_USE_TLS", "").lower() in {"1", "true", "yes"} if os.getenv("IRC_USE_TLS") else extra.get("use_tls", True) ) @@ -680,7 +680,7 @@ def _env_enablement() -> dict | None: seed["nickname"] = nickname use_tls = os.getenv("IRC_USE_TLS", "").strip().lower() if use_tls: - seed["use_tls"] = use_tls in ("1", "true", "yes") + seed["use_tls"] = use_tls in {"1", "true", "yes"} # Passwords live in PlatformConfig.extra as well for back-compat with # existing config.yaml users; env-reads at construct time still win. if os.getenv("IRC_SERVER_PASSWORD"): @@ -756,7 +756,7 @@ async def _standalone_send( nickname = os.getenv("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") use_tls_env = os.getenv("IRC_USE_TLS") if use_tls_env is not None: - use_tls = use_tls_env.lower() in ("1", "true", "yes") + use_tls = use_tls_env.lower() in {"1", "true", "yes"} else: use_tls = bool(extra.get("use_tls", True)) @@ -821,7 +821,7 @@ async def _standalone_send( await _raw(f"PONG :{payload}") elif cmd == "001": registered = True - elif cmd in ("432", "433"): + elif cmd in {"432", "433"}: nick_attempts += 1 if nick_attempts > max_nick_attempts: return {"error": "IRC standalone send: too many nick collisions"} @@ -829,7 +829,7 @@ async def _standalone_send( # mutated value, so the suffix stays bounded. standalone_nick = f"{nick_base}-cron-{nick_attempts}"[:30] await _raw(f"NICK {standalone_nick}") - elif cmd in ("464", "465"): + elif cmd in {"464", "465"}: return {"error": f"IRC standalone send: server rejected client ({cmd})"} if nickserv_password: @@ -860,9 +860,9 @@ async def _standalone_send( if jcmd == "PING": payload = jmsg["params"][0] if jmsg["params"] else "" await _raw(f"PONG :{payload}") - elif jcmd in ("366", "JOIN"): + elif jcmd in {"366", "JOIN"}: joined = True - elif jcmd in ("403", "405", "471", "473", "474", "475"): + elif jcmd in {"403", "405", "471", "473", "474", "475"}: return {"error": f"IRC standalone send: JOIN {target} rejected ({jcmd})"} # Bytes-aware per-line splitting so multi-line plain text never diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index 907f16be4ff..49931aa57ab 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -325,7 +325,7 @@ class RequestCache: def mark_delivered(self, request_id: str) -> None: entry = self._entries.get(request_id) - if entry is None or entry.state not in (State.READY, State.ERROR): + if entry is None or entry.state not in {State.READY, State.ERROR}: return entry.state = State.DELIVERED entry.updated_at = time.time() @@ -614,7 +614,7 @@ def _truthy_env(name: str, default: bool = False) -> bool: v = os.getenv(name) if v is None: return default - return v.strip().lower() in ("1", "true", "yes", "on") + return v.strip().lower() in {"1", "true", "yes", "on"} # --------------------------------------------------------------------------- @@ -910,7 +910,7 @@ class LineAdapter(BasePlatformAdapter): await self._handle_message_event(event) elif event_type == "postback": await self._handle_postback_event(event) - elif event_type in ("follow", "unfollow", "join", "leave"): + elif event_type in {"follow", "unfollow", "join", "leave"}: logger.info("LINE: lifecycle event %s from %s", event_type, source) else: logger.debug("LINE: ignoring event type %r", event_type) @@ -939,7 +939,7 @@ class LineAdapter(BasePlatformAdapter): if msg_type == "text": text = msg.get("text", "") or "" - elif msg_type in ("image", "audio", "video", "file"): + elif msg_type in {"image", "audio", "video", "file"}: local_path = await self._download_media(message_id, msg_type) if local_path: media_urls.append(local_path) diff --git a/plugins/platforms/simplex/adapter.py b/plugins/platforms/simplex/adapter.py index b568f29bbb5..264deb89608 100644 --- a/plugins/platforms/simplex/adapter.py +++ b/plugins/platforms/simplex/adapter.py @@ -101,11 +101,11 @@ def _guess_extension(data: bytes) -> str: def _is_image_ext(ext: str) -> bool: - return ext.lower() in (".jpg", ".jpeg", ".png", ".gif", ".webp") + return ext.lower() in {".jpg", ".jpeg", ".png", ".gif", ".webp"} def _is_audio_ext(ext: str) -> bool: - return ext.lower() in (".mp3", ".wav", ".ogg", ".m4a", ".aac") + return ext.lower() in {".mp3", ".wav", ".ogg", ".m4a", ".aac"} # --------------------------------------------------------------------------- @@ -326,12 +326,12 @@ class SimplexAdapter(BasePlatformAdapter): # Filter out messages sent by us (direction == "snd") meta = chat_item.get("meta") or {} direction = (meta.get("itemStatus") or {}).get("type", "") - if direction in ("sndSent", "sndSentDirect", "sndSentViaProxy", "sndNew"): + if direction in {"sndSent", "sndSentDirect", "sndSentViaProxy", "sndNew"}: return # Determine chat type and IDs chat_type_raw = chat_info.get("type", "") - is_group = chat_type_raw in ("group", "groupInfo") + is_group = chat_type_raw in {"group", "groupInfo"} if is_group: group_info = chat_info.get("groupInfo") or chat_info.get("group") or {} @@ -374,7 +374,7 @@ class SimplexAdapter(BasePlatformAdapter): media_urls: List[str] = [] media_types: List[str] = [] file_info = chat_item.get("file") or {} - if file_info and file_info.get("fileStatus") not in ("cancelled", "error"): + if file_info and file_info.get("fileStatus") not in {"cancelled", "error"}: file_id = file_info.get("fileId") file_name = file_info.get("fileName", "file") if file_id: diff --git a/plugins/platforms/teams/adapter.py b/plugins/platforms/teams/adapter.py index f8a1dc3d5b4..975ef5b4093 100644 --- a/plugins/platforms/teams/adapter.py +++ b/plugins/platforms/teams/adapter.py @@ -841,7 +841,7 @@ class TeamsAdapter(BasePlatformAdapter): # bot silently treated every clicker as authorized — meaning any # Teams user who could message the bot could approve dangerous commands. allowed_csv = os.getenv("TEAMS_ALLOWED_USERS", "").strip() - allow_all = os.getenv("TEAMS_ALLOW_ALL_USERS", "").strip().lower() in ("1", "true", "yes") + allow_all = os.getenv("TEAMS_ALLOW_ALL_USERS", "").strip().lower() in {"1", "true", "yes"} if not allow_all: if not allowed_csv: diff --git a/plugins/teams_pipeline/cli.py b/plugins/teams_pipeline/cli.py index 0e1114e3e74..7afaa3888a0 100644 --- a/plugins/teams_pipeline/cli.py +++ b/plugins/teams_pipeline/cli.py @@ -99,15 +99,15 @@ def teams_pipeline_command(args: argparse.Namespace) -> int: return 2 try: - if action in ("list", "ls"): + if action in {"list", "ls"}: _cmd_list(args) elif action == "show": _cmd_show(args) - elif action in ("run", "replay"): + elif action in {"run", "replay"}: _cmd_run(args) - elif action in ("fetch", "test"): + elif action in {"fetch", "test"}: _cmd_fetch(args) - elif action in ("subscriptions", "subs"): + elif action in {"subscriptions", "subs"}: _cmd_subscriptions(args) elif action == "subscribe": _cmd_subscribe(args) @@ -117,7 +117,7 @@ def teams_pipeline_command(args: argparse.Namespace) -> int: _cmd_delete_subscription(args) elif action == "maintain-subscriptions": _cmd_maintain_subscriptions(args) - elif action in ("token-health", "token"): + elif action in {"token-health", "token"}: _cmd_token_health(args) elif action == "validate": _cmd_validate(args) diff --git a/plugins/teams_pipeline/meetings.py b/plugins/teams_pipeline/meetings.py index 6d2648abd52..ed024bc7e31 100644 --- a/plugins/teams_pipeline/meetings.py +++ b/plugins/teams_pipeline/meetings.py @@ -33,7 +33,7 @@ def _meeting_path(meeting_ref: TeamsMeetingRef | str) -> str: def _wrap_graph_error(exc: MicrosoftGraphAPIError, *, missing_message: str) -> TeamsMeetingError: - if exc.status_code in (401, 403): + if exc.status_code in {401, 403}: return TeamsMeetingPermissionError(str(exc)) if exc.status_code == 404: return TeamsMeetingNotFoundError(missing_message) @@ -286,7 +286,7 @@ async def fetch_call_record_artifact( try: payload = await client.get_json(f"/communications/callRecords/{quote(call_record_id, safe='')}") except MicrosoftGraphAPIError as exc: - if exc.status_code in (401, 403) and allow_permission_errors: + if exc.status_code in {401, 403} and allow_permission_errors: return None if exc.status_code == 404: return None diff --git a/plugins/teams_pipeline/models.py b/plugins/teams_pipeline/models.py index 8d85092be96..b1ae5196f51 100644 --- a/plugins/teams_pipeline/models.py +++ b/plugins/teams_pipeline/models.py @@ -145,7 +145,7 @@ class MeetingArtifact: metadata: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - if self.artifact_type not in ("transcript", "recording", "call_record"): + if self.artifact_type not in {"transcript", "recording", "call_record"}: raise ValueError( "MeetingArtifact.artifact_type must be transcript, recording, or call_record." ) diff --git a/plugins/teams_pipeline/runtime.py b/plugins/teams_pipeline/runtime.py index e8d3ada710c..f51be5e19e3 100644 --- a/plugins/teams_pipeline/runtime.py +++ b/plugins/teams_pipeline/runtime.py @@ -62,7 +62,7 @@ def build_pipeline_runtime_config(gateway_config: Any) -> dict[str, Any]: "chat_id", ): value = teams_extra.get(key) - if value not in (None, ""): + if value not in {None, ""}: teams_delivery[key] = value if teams_delivery: diff --git a/run_agent.py b/run_agent.py index 8471afccddf..f25c94f17a9 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1365,7 +1365,7 @@ class AIAgent: the existing 1M-context-beta branch handles them; revisit if other subscription tiers start producing the same loop signature). """ - if status_code not in (401, 403, None): + if status_code not in {401, 403, None}: return False if not isinstance(error_context, dict): return False @@ -1774,7 +1774,7 @@ class AIAgent: import os as _os env = _os.environ.get("HERMES_FILE_MUTATION_VERIFIER") if env is not None: - return env.strip().lower() not in ("0", "false", "no", "off") + return env.strip().lower() not in {"0", "false", "no", "off"} # Read from the persisted config.yaml so gateway and CLI share # the same setting. Import lazily to avoid a startup-time cycle. try: diff --git a/skills/creative/comfyui/scripts/_common.py b/skills/creative/comfyui/scripts/_common.py index ef742733eb5..efe592a1b33 100644 --- a/skills/creative/comfyui/scripts/_common.py +++ b/skills/creative/comfyui/scripts/_common.py @@ -592,7 +592,7 @@ def _http_once( # Build a new request with cleaned headers clean_headers = { k: v for k, v in req2.header_items() - if k.lower() not in ("x-api-key", "authorization", "cookie") + if k.lower() not in {"x-api-key", "authorization", "cookie"} } new_req = urllib.request.Request(newurl, headers=clean_headers, method="GET") return new_req @@ -743,13 +743,13 @@ def safe_path_join(base: Path, *parts: str) -> Path: def media_type_from_filename(filename: str) -> str: ext = Path(filename).suffix.lower() - if ext in (".mp4", ".webm", ".avi", ".mov", ".mkv", ".gif", ".webp"): + if ext in {".mp4", ".webm", ".avi", ".mov", ".mkv", ".gif", ".webp"}: return "video" - if ext in (".wav", ".mp3", ".flac", ".ogg", ".m4a"): + if ext in {".wav", ".mp3", ".flac", ".ogg", ".m4a"}: return "audio" - if ext in (".glb", ".obj", ".ply", ".gltf"): + if ext in {".glb", ".obj", ".ply", ".gltf"}: return "3d" - if ext in (".json", ".txt", ".md"): + if ext in {".json", ".txt", ".md"}: return "text" return "image" diff --git a/skills/creative/comfyui/scripts/extract_schema.py b/skills/creative/comfyui/scripts/extract_schema.py index ba44cfdf6a2..0eab65b20fd 100755 --- a/skills/creative/comfyui/scripts/extract_schema.py +++ b/skills/creative/comfyui/scripts/extract_schema.py @@ -81,7 +81,7 @@ def trace_to_node(workflow: dict, link: list, *, max_hops: int = 8) -> str | Non return None cls = node.get("class_type", "") # Reroute / Primitive / passthrough wrappers - if cls in ("Reroute", "PrimitiveNode", "Note", "easy showAnything"): + if cls in {"Reroute", "PrimitiveNode", "Note", "easy showAnything"}: inputs = node.get("inputs", {}) or {} # Find first link-shaped input and follow it next_link = next((v for v in inputs.values() if is_link(v)), None) @@ -105,7 +105,7 @@ def find_negative_prompt_node(workflow: dict) -> str | None: src = trace_to_node(workflow, neg) if src and isinstance(workflow.get(src), dict): cls = workflow[src].get("class_type", "") - if cls.startswith("CLIPTextEncode") or cls in ("smZ CLIPTextEncode", "BNK_CLIPTextEncodeAdvanced"): + if cls.startswith("CLIPTextEncode") or cls in {"smZ CLIPTextEncode", "BNK_CLIPTextEncodeAdvanced"}: return src return None @@ -121,7 +121,7 @@ def find_positive_prompt_node(workflow: dict) -> str | None: src = trace_to_node(workflow, pos) if src and isinstance(workflow.get(src), dict): cls = workflow[src].get("class_type", "") - if cls.startswith("CLIPTextEncode") or cls in ("smZ CLIPTextEncode", "BNK_CLIPTextEncodeAdvanced"): + if cls.startswith("CLIPTextEncode") or cls in {"smZ CLIPTextEncode", "BNK_CLIPTextEncodeAdvanced"}: return src return None diff --git a/skills/creative/comfyui/scripts/fetch_logs.py b/skills/creative/comfyui/scripts/fetch_logs.py index c7b3b084807..e0b6e12ac75 100755 --- a/skills/creative/comfyui/scripts/fetch_logs.py +++ b/skills/creative/comfyui/scripts/fetch_logs.py @@ -151,7 +151,7 @@ def main(argv: list[str] | None = None) -> int: diag["source"] = res.get("source") diag["prompt_id"] = args.prompt_id emit_json(diag) - return 0 if diag.get("status_str") not in ("error",) else 1 + return 0 if diag.get("status_str") not in {"error",} else 1 if __name__ == "__main__": diff --git a/skills/creative/comfyui/scripts/hardware_check.py b/skills/creative/comfyui/scripts/hardware_check.py index 6a4d6c6d406..083d018acc6 100755 --- a/skills/creative/comfyui/scripts/hardware_check.py +++ b/skills/creative/comfyui/scripts/hardware_check.py @@ -203,7 +203,7 @@ def detect_apple_silicon() -> dict | None: def detect_intel_arc() -> dict | None: - if platform.system() not in ("Linux", "Windows"): + if platform.system() not in {"Linux", "Windows"}: return None if shutil.which("clinfo"): out = _run(["clinfo", "--list"]) diff --git a/skills/creative/comfyui/scripts/run_workflow.py b/skills/creative/comfyui/scripts/run_workflow.py index 444957960b6..05afb1e319f 100755 --- a/skills/creative/comfyui/scripts/run_workflow.py +++ b/skills/creative/comfyui/scripts/run_workflow.py @@ -204,7 +204,7 @@ class ComfyRunner: s = data.get("status") if s == "completed": return {"status": "success", "data": data} - if s in ("failed",): + if s in {"failed",}: return {"status": "error", "data": data} if s == "cancelled": return {"status": "cancelled", "data": data} @@ -386,7 +386,7 @@ class ComfyRunner: # local path; otherwise put the file in output_dir flat. target_parts: list[str] = [] if preserve_subfolder and subfolder: - target_parts.extend(p for p in subfolder.split("/") if p and p not in (".", "..")) + target_parts.extend(p for p in subfolder.split("/") if p and p not in {".", ".."}) target_parts.append(filename) out_path = safe_path_join(output_dir, *target_parts) @@ -467,7 +467,7 @@ def inject_params( # Auto-randomize seed when it's -1 in args, or when randomize_seed_if_unset # and user didn't pass a seed. if "seed" in params: - if "seed" in args and args["seed"] in (None, -1, "-1"): + if "seed" in args and args["seed"] in {None, -1, "-1"}: args = dict(args) args["seed"] = coerce_seed(args["seed"]) warnings.append(f"seed=-1 expanded to {args['seed']}") diff --git a/skills/creative/comfyui/scripts/ws_monitor.py b/skills/creative/comfyui/scripts/ws_monitor.py index b8689655bd0..e2b6689423a 100755 --- a/skills/creative/comfyui/scripts/ws_monitor.py +++ b/skills/creative/comfyui/scripts/ws_monitor.py @@ -170,7 +170,7 @@ def main(argv: list[str] | None = None) -> int: parsed = parse_binary_frame(msg) if parsed is None: continue - if parsed["kind"] in ("preview", "preview_with_metadata") and preview_dir: + if parsed["kind"] in {"preview", "preview_with_metadata"} and preview_dir: img_bytes = parsed.get("image_bytes", b"") if img_bytes: ext = parsed.get("ext", "png") diff --git a/skills/creative/comfyui/tests/test_cloud_integration.py b/skills/creative/comfyui/tests/test_cloud_integration.py index eb7b04ca225..0ce88efe3c2 100644 --- a/skills/creative/comfyui/tests/test_cloud_integration.py +++ b/skills/creative/comfyui/tests/test_cloud_integration.py @@ -53,7 +53,7 @@ class TestCloudEndpointsLive: url = resolve_url("https://cloud.comfy.org", "/object_info") r = http_get(url, headers={"X-API-Key": cloud_key}) # Should be either 200 (paid) or 403 (free) — not 404 / 500 - assert r.status in (200, 403) + assert r.status in {200, 403} if r.status == 403: # Body should mention the limitation assert "free tier" in r.text().lower() or "subscription" in r.text().lower() diff --git a/skills/creative/comfyui/tests/test_extract_schema.py b/skills/creative/comfyui/tests/test_extract_schema.py index 1cb965a1fa8..072a788f318 100644 --- a/skills/creative/comfyui/tests/test_extract_schema.py +++ b/skills/creative/comfyui/tests/test_extract_schema.py @@ -40,7 +40,7 @@ class TestConnectionTracing: } # Should hit max_hops without infinite loop result = trace_to_node(wf, ["1", 0], max_hops=5) - assert result in ("1", "2") # any node, just don't hang + assert result in {"1", "2"} # any node, just don't hang class TestPositiveNegativeDetection: diff --git a/skills/productivity/google-workspace/scripts/google_api.py b/skills/productivity/google-workspace/scripts/google_api.py index 7b8350ab34a..231b1b6849f 100644 --- a/skills/productivity/google-workspace/scripts/google_api.py +++ b/skills/productivity/google-workspace/scripts/google_api.py @@ -721,7 +721,7 @@ def drive_share(args): "type": args.type, "role": args.role, } - if args.type in ("user", "group"): + if args.type in {"user", "group"}: if not args.email: print("ERROR: --email is required for type=user or type=group", file=sys.stderr) sys.exit(1) diff --git a/skills/productivity/maps/scripts/maps_client.py b/skills/productivity/maps/scripts/maps_client.py index 279a41aad64..d272b4a7566 100644 --- a/skills/productivity/maps/scripts/maps_client.py +++ b/skills/productivity/maps/scripts/maps_client.py @@ -181,7 +181,7 @@ def http_get(url, params=None, retries=MAX_RETRIES, silent=False): return json.loads(raw) except urllib.error.HTTPError as exc: last_error = f"HTTP {exc.code}: {exc.reason} for {url}" - if exc.code in (429, 503, 502, 504): + if exc.code in {429, 503, 502, 504}: time.sleep(RETRY_DELAY * attempt) else: if silent: @@ -217,7 +217,7 @@ def http_get_text(url, params=None, retries=MAX_RETRIES, silent=False): return resp.read().decode("utf-8") except urllib.error.HTTPError as exc: last_error = f"HTTP {exc.code}: {exc.reason} for {url}" - if exc.code in (429, 503, 502, 504): + if exc.code in {429, 503, 502, 504}: time.sleep(RETRY_DELAY * attempt) else: if silent: @@ -256,7 +256,7 @@ def http_post(url, data_str, retries=MAX_RETRIES): return json.loads(raw) except urllib.error.HTTPError as exc: last_error = f"HTTP {exc.code}: {exc.reason}" - if exc.code in (429, 503, 502, 504): + if exc.code in {429, 503, 502, 504}: time.sleep(RETRY_DELAY * attempt) else: error_exit(last_error) @@ -459,8 +459,8 @@ def parse_overpass_elements(elements, ref_lat=None, ref_lon=None): "maps_url": f"https://www.google.com/maps/search/?api=1&query={el_lat},{el_lon}", "tags": { k: v for k, v in tags.items() - if k not in ("name", "name:en", - "addr:housenumber", "addr:street", "addr:city") + if k not in {"name", "name:en", + "addr:housenumber", "addr:street", "addr:city"} }, } diff --git a/skills/productivity/ocr-and-documents/scripts/extract_marker.py b/skills/productivity/ocr-and-documents/scripts/extract_marker.py index 4f301aac7b2..d48fd10bb02 100644 --- a/skills/productivity/ocr-and-documents/scripts/extract_marker.py +++ b/skills/productivity/ocr-and-documents/scripts/extract_marker.py @@ -63,7 +63,7 @@ def check_requirements(): if __name__ == "__main__": args = sys.argv[1:] - if not args or args[0] in ("-h", "--help"): + if not args or args[0] in {"-h", "--help"}: print(__doc__) sys.exit(0) diff --git a/skills/productivity/ocr-and-documents/scripts/extract_pymupdf.py b/skills/productivity/ocr-and-documents/scripts/extract_pymupdf.py index 22063e73489..50cb8ee86c4 100644 --- a/skills/productivity/ocr-and-documents/scripts/extract_pymupdf.py +++ b/skills/productivity/ocr-and-documents/scripts/extract_pymupdf.py @@ -68,7 +68,7 @@ def show_metadata(path): if __name__ == "__main__": args = sys.argv[1:] - if not args or args[0] in ("-h", "--help"): + if not args or args[0] in {"-h", "--help"}: print(__doc__) sys.exit(0) diff --git a/skills/research/arxiv/scripts/search_arxiv.py b/skills/research/arxiv/scripts/search_arxiv.py index 9acd8b97ec9..0bd6b2370f4 100644 --- a/skills/research/arxiv/scripts/search_arxiv.py +++ b/skills/research/arxiv/scripts/search_arxiv.py @@ -81,7 +81,7 @@ def search(query=None, author=None, category=None, ids=None, max_results=5, sort if __name__ == "__main__": args = sys.argv[1:] - if not args or args[0] in ("-h", "--help"): + if not args or args[0] in {"-h", "--help"}: print(__doc__) sys.exit(0) diff --git a/skills/research/polymarket/scripts/polymarket.py b/skills/research/polymarket/scripts/polymarket.py index 417e0b1747e..b76e7aa5f9b 100644 --- a/skills/research/polymarket/scripts/polymarket.py +++ b/skills/research/polymarket/scripts/polymarket.py @@ -233,7 +233,7 @@ def cmd_trades(limit: int = 10, market: str = None): def main(): args = sys.argv[1:] - if not args or args[0] in ("-h", "--help", "help"): + if not args or args[0] in {"-h", "--help", "help"}: print(__doc__) return diff --git a/tests/agent/lsp/_mock_lsp_server.py b/tests/agent/lsp/_mock_lsp_server.py index 0220fec195d..619b8da233f 100644 --- a/tests/agent/lsp/_mock_lsp_server.py +++ b/tests/agent/lsp/_mock_lsp_server.py @@ -91,7 +91,7 @@ def main(): if msg.get("method") == "workspace/didChangeWatchedFiles": continue - if msg.get("method") in ("textDocument/didOpen", "textDocument/didChange"): + if msg.get("method") in {"textDocument/didOpen", "textDocument/didChange"}: params = msg.get("params") or {} td = params.get("textDocument") or {} uri = td.get("uri", "") diff --git a/tests/agent/lsp/test_install_and_lint_fixes.py b/tests/agent/lsp/test_install_and_lint_fixes.py index 9046d01295e..e9f862a6d8e 100644 --- a/tests/agent/lsp/test_install_and_lint_fixes.py +++ b/tests/agent/lsp/test_install_and_lint_fixes.py @@ -87,10 +87,10 @@ def test_install_npm_works_without_extras(tmp_path, monkeypatch): cmd = captured["cmd"] assert "pyright" in cmd # Should not blow up when extra_pkgs is omitted/None - install_targets = [c for c in cmd if not c.startswith("-") and c not in ( + install_targets = [c for c in cmd if not c.startswith("-") and c not in { "install", "--prefix", str(install_mod.hermes_lsp_bin_dir().parent), "/usr/bin/npm", - )] + }] assert install_targets == ["pyright"] diff --git a/tests/agent/test_anthropic_adapter.py b/tests/agent/test_anthropic_adapter.py index 259e9c1c523..c7119dfd3b0 100644 --- a/tests/agent/test_anthropic_adapter.py +++ b/tests/agent/test_anthropic_adapter.py @@ -1658,7 +1658,7 @@ class TestThinkingBlockSignatureManagement: _, result = convert_messages_to_anthropic(messages) assistant = next(m for m in result if m["role"] == "assistant") for block in assistant["content"]: - if block.get("type") in ("thinking", "redacted_thinking"): + if block.get("type") in {"thinking", "redacted_thinking"}: assert "cache_control" not in block def test_thinking_stripped_from_merged_consecutive_assistants(self): @@ -1748,7 +1748,7 @@ class TestThinkingBlockSignatureManagement: # First two: no thinking blocks for a in assistants[:2]: assert not any( - b.get("type") in ("thinking", "redacted_thinking") + b.get("type") in {"thinking", "redacted_thinking"} for b in a["content"] if isinstance(b, dict) ) diff --git a/tests/agent/test_auxiliary_main_first.py b/tests/agent/test_auxiliary_main_first.py index 6ac69b27b7c..d1b758c2884 100644 --- a/tests/agent/test_auxiliary_main_first.py +++ b/tests/agent/test_auxiliary_main_first.py @@ -371,7 +371,7 @@ class TestResolveVisionMainFirst: provider, client, model = resolve_vision_provider_client() assert client is fallback_client - assert provider in ("openrouter", "nous") + assert provider in {"openrouter", "nous"} def test_explicit_provider_override_still_wins(self): """Explicit config override bypasses main-first policy.""" diff --git a/tests/agent/test_context_compressor.py b/tests/agent/test_context_compressor.py index 559cf2237a2..2d1a40445d7 100644 --- a/tests/agent/test_context_compressor.py +++ b/tests/agent/test_context_compressor.py @@ -1046,7 +1046,7 @@ class TestCompressWithClient: for i in range(1, len(result)): r1 = result[i - 1].get("role") r2 = result[i].get("role") - if r1 in ("user", "assistant") and r2 in ("user", "assistant"): + if r1 in {"user", "assistant"} and r2 in {"user", "assistant"}: assert r1 != r2, f"consecutive {r1} at indices {i-1},{i}" def test_double_collision_merges_summary_into_tail(self): @@ -1087,7 +1087,7 @@ class TestCompressWithClient: for i in range(1, len(result)): r1 = result[i - 1].get("role") r2 = result[i].get("role") - if r1 in ("user", "assistant") and r2 in ("user", "assistant"): + if r1 in {"user", "assistant"} and r2 in {"user", "assistant"}: assert r1 != r2, f"consecutive {r1} at indices {i-1},{i}" # The summary text should be merged into the first tail message @@ -1164,7 +1164,7 @@ class TestCompressWithClient: for i in range(1, len(result)): r1 = result[i - 1].get("role") r2 = result[i].get("role") - if r1 in ("user", "assistant") and r2 in ("user", "assistant"): + if r1 in {"user", "assistant"} and r2 in {"user", "assistant"}: assert r1 != r2, f"consecutive {r1} at indices {i-1},{i}" # The summary should be merged into the first tail message (assistant at index 5) diff --git a/tests/agent/test_deepseek_anthropic_thinking.py b/tests/agent/test_deepseek_anthropic_thinking.py index 4d032fa3595..67534adc3e8 100644 --- a/tests/agent/test_deepseek_anthropic_thinking.py +++ b/tests/agent/test_deepseek_anthropic_thinking.py @@ -191,7 +191,7 @@ class TestDeepSeekAnthropicPreservesThinking: if not isinstance(m.get("content"), list): continue for b in m["content"]: - if isinstance(b, dict) and b.get("type") in ("thinking", "redacted_thinking"): + if isinstance(b, dict) and b.get("type") in {"thinking", "redacted_thinking"}: assert "cache_control" not in b def test_openai_compat_deepseek_base_is_not_matched(self) -> None: diff --git a/tests/cli/test_cli_init.py b/tests/cli/test_cli_init.py index 8417d64e746..b05df5220c5 100644 --- a/tests/cli/test_cli_init.py +++ b/tests/cli/test_cli_init.py @@ -99,7 +99,7 @@ class TestVerboseAndToolProgress: def test_tool_progress_mode_is_string(self): cli = _make_cli() assert isinstance(cli.tool_progress_mode, str) - assert cli.tool_progress_mode in ("off", "new", "all", "verbose") + assert cli.tool_progress_mode in {"off", "new", "all", "verbose"} class TestBusyInputMode: diff --git a/tests/cli/test_reasoning_command.py b/tests/cli/test_reasoning_command.py index f5f7e35cbe7..5091256a399 100644 --- a/tests/cli/test_reasoning_command.py +++ b/tests/cli/test_reasoning_command.py @@ -70,7 +70,7 @@ class TestHandleReasoningCommand(unittest.TestCase): stub = self._make_cli(show_reasoning=False) # Simulate /reasoning show arg = "show" - if arg in ("show", "on"): + if arg in {"show", "on"}: stub.show_reasoning = True stub.agent.reasoning_callback = lambda x: None self.assertTrue(stub.show_reasoning) @@ -79,7 +79,7 @@ class TestHandleReasoningCommand(unittest.TestCase): stub = self._make_cli(show_reasoning=True) # Simulate /reasoning hide arg = "hide" - if arg in ("hide", "off"): + if arg in {"hide", "off"}: stub.show_reasoning = False stub.agent.reasoning_callback = None self.assertFalse(stub.show_reasoning) @@ -88,14 +88,14 @@ class TestHandleReasoningCommand(unittest.TestCase): def test_on_enables_display(self): stub = self._make_cli(show_reasoning=False) arg = "on" - if arg in ("show", "on"): + if arg in {"show", "on"}: stub.show_reasoning = True self.assertTrue(stub.show_reasoning) def test_off_disables_display(self): stub = self._make_cli(show_reasoning=True) arg = "off" - if arg in ("hide", "off"): + if arg in {"hide", "off"}: stub.show_reasoning = False self.assertFalse(stub.show_reasoning) diff --git a/tests/cron/test_cron_no_agent.py b/tests/cron/test_cron_no_agent.py index 117cb8c7d9a..583cd34099e 100644 --- a/tests/cron/test_cron_no_agent.py +++ b/tests/cron/test_cron_no_agent.py @@ -68,7 +68,7 @@ def test_create_job_no_agent_stores_field(hermes_env): assert job["no_agent"] is True assert job["script"] == "watchdog.sh" # Prompt can be empty/None for no_agent jobs. - assert job["prompt"] in (None, "") + assert job["prompt"] in {None, ""} def test_create_job_default_is_not_no_agent(hermes_env): @@ -148,7 +148,7 @@ def test_cronjob_tool_update_toggles_no_agent(hermes_env): off = json.loads(cronjob(action="update", job_id=job_id, no_agent=False, prompt="run")) assert off["success"] is True - assert off["job"].get("no_agent") in (False, None) + assert off["job"].get("no_agent") in {False, None} on = json.loads(cronjob(action="update", job_id=job_id, no_agent=True)) assert on["success"] is True diff --git a/tests/gateway/conftest.py b/tests/gateway/conftest.py index b6bcc28c506..965933de41b 100644 --- a/tests/gateway/conftest.py +++ b/tests/gateway/conftest.py @@ -269,7 +269,7 @@ def _scan_for_plugin_adapter_antipattern(source: str) -> list[str]: and isinstance(func.value.value, ast.Name) and func.value.value.id == "sys" and func.value.attr == "path" - and func.attr in ("insert", "append", "extend") + and func.attr in {"insert", "append", "extend"} ): target_name = f"sys.path.{func.attr}" diff --git a/tests/gateway/test_allowlist_startup_check.py b/tests/gateway/test_allowlist_startup_check.py index 96441c05213..abb2db7db12 100644 --- a/tests/gateway/test_allowlist_startup_check.py +++ b/tests/gateway/test_allowlist_startup_check.py @@ -16,8 +16,8 @@ def _would_warn(): "MATRIX_ALLOWED_USERS", "DINGTALK_ALLOWED_USERS", "FEISHU_ALLOWED_USERS", "WECOM_ALLOWED_USERS", "GATEWAY_ALLOWED_USERS") ) - _allow_all = os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in ("true", "1", "yes") or any( - os.getenv(v, "").lower() in ("true", "1", "yes") + _allow_all = os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} or any( + os.getenv(v, "").lower() in {"true", "1", "yes"} for v in ("TELEGRAM_ALLOW_ALL_USERS", "DISCORD_ALLOW_ALL_USERS", "WHATSAPP_ALLOW_ALL_USERS", "SLACK_ALLOW_ALL_USERS", "SIGNAL_ALLOW_ALL_USERS", "EMAIL_ALLOW_ALL_USERS", diff --git a/tests/gateway/test_config_cwd_bridge.py b/tests/gateway/test_config_cwd_bridge.py index 23666253882..f7349d073f7 100644 --- a/tests/gateway/test_config_cwd_bridge.py +++ b/tests/gateway/test_config_cwd_bridge.py @@ -44,7 +44,7 @@ def _simulate_config_bridge(cfg: dict, initial_env: dict | None = None): val = terminal_cfg[cfg_key] # Skip cwd placeholder values — don't overwrite already-resolved # TERMINAL_CWD. Mirrors the fix in gateway/run.py. - if cfg_key == "cwd" and str(val) in (".", "auto", "cwd"): + if cfg_key == "cwd" and str(val) in {".", "auto", "cwd"}: continue # Expand shell tilde so subprocess.Popen never receives a literal # "~/" which the kernel rejects. @@ -70,7 +70,7 @@ def _simulate_config_bridge(cfg: dict, initial_env: dict | None = None): # --- Replicate lines 144-147: MESSAGING_CWD fallback --- configured_cwd = env.get("TERMINAL_CWD", "") - if not configured_cwd or configured_cwd in (".", "auto", "cwd"): + if not configured_cwd or configured_cwd in {".", "auto", "cwd"}: messaging_cwd = env.get("MESSAGING_CWD") or "/root" # Path.home() for root env["TERMINAL_CWD"] = messaging_cwd diff --git a/tests/gateway/test_discord_system_messages.py b/tests/gateway/test_discord_system_messages.py index 8e2fb27e788..e58f2812745 100644 --- a/tests/gateway/test_discord_system_messages.py +++ b/tests/gateway/test_discord_system_messages.py @@ -48,7 +48,7 @@ class TestDiscordSystemMessageFilter(unittest.TestCase): return False # System message filter (the fix being tested) - if message.type not in (discord.MessageType.default, discord.MessageType.reply): + if message.type not in {discord.MessageType.default, discord.MessageType.reply}: return False return True # message accepted diff --git a/tests/gateway/test_platform_connected_checkers.py b/tests/gateway/test_platform_connected_checkers.py index 307c79b3086..941b8c74506 100644 --- a/tests/gateway/test_platform_connected_checkers.py +++ b/tests/gateway/test_platform_connected_checkers.py @@ -76,12 +76,12 @@ def test_checker_returns_true_when_configured(platform, checker, monkeypatch): elif platform == Platform.SMS: monkeypatch.setenv("TWILIO_ACCOUNT_SID", "ACtest") mock_config.extra = {} - elif platform in ( + elif platform in { Platform.API_SERVER, Platform.WEBHOOK, Platform.MSGRAPH_WEBHOOK, Platform.WHATSAPP, - ): + }: mock_config.extra = {} elif platform == Platform.FEISHU: mock_config.extra = {"app_id": "app"} diff --git a/tests/gateway/test_qqbot.py b/tests/gateway/test_qqbot.py index 5d5cac54bd3..4b3402387a4 100644 --- a/tests/gateway/test_qqbot.py +++ b/tests/gateway/test_qqbot.py @@ -1076,7 +1076,7 @@ class TestBuildApprovalKeyboard: parsed = parse_approval_button_data(btn.action.data) assert parsed is not None assert parsed[0] == session_key - assert parsed[1] in ("allow-once", "allow-always", "deny") + assert parsed[1] in {"allow-once", "allow-always", "deny"} class TestBuildUpdatePromptKeyboard: diff --git a/tests/gateway/test_restart_resume_pending.py b/tests/gateway/test_restart_resume_pending.py index 13ef2f6f99e..55d9b4a497b 100644 --- a/tests/gateway/test_restart_resume_pending.py +++ b/tests/gateway/test_restart_resume_pending.py @@ -89,7 +89,7 @@ def _build_agent_history(history: list) -> list: agent_history: list = [] for msg in history: role = msg.get("role") - if not role or role in ("session_meta", "system"): + if not role or role in {"session_meta", "system"}: continue has_tool_calls = "tool_calls" in msg has_tool_call_id = "tool_call_id" in msg diff --git a/tests/gateway/test_session_boundary_hooks.py b/tests/gateway/test_session_boundary_hooks.py index 255795492fc..30584513325 100644 --- a/tests/gateway/test_session_boundary_hooks.py +++ b/tests/gateway/test_session_boundary_hooks.py @@ -108,7 +108,7 @@ async def test_finalize_before_reset(mock_invoke_hook): await runner._handle_reset_command(_make_event("/new")) calls = [c for c in mock_invoke_hook.call_args_list - if c[0][0] in ("on_session_finalize", "on_session_reset")] + if c[0][0] in {"on_session_finalize", "on_session_reset"}] hook_names = [c[0][0] for c in calls] assert hook_names == ["on_session_finalize", "on_session_reset"] diff --git a/tests/gateway/test_session_model_override_routing.py b/tests/gateway/test_session_model_override_routing.py index 3530744e223..26acdc157aa 100644 --- a/tests/gateway/test_session_model_override_routing.py +++ b/tests/gateway/test_session_model_override_routing.py @@ -187,7 +187,7 @@ fallback_providers: monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path) def fake_resolve_runtime_provider(*, requested=None, explicit_base_url=None, explicit_api_key=None): - if requested in (None, "", "openai-codex"): + if requested in {None, "", "openai-codex"}: from hermes_cli.auth import AuthError raise AuthError("No Codex credentials stored. Run `hermes auth` to authenticate.") assert requested == "openrouter" diff --git a/tests/gateway/test_transcript_offset.py b/tests/gateway/test_transcript_offset.py index d8a2672f4d6..7cbb519ee3a 100644 --- a/tests/gateway/test_transcript_offset.py +++ b/tests/gateway/test_transcript_offset.py @@ -31,7 +31,7 @@ def _filter_history(history: list) -> list: role = msg.get("role") if not role: continue - if role in ("session_meta",): + if role in {"session_meta",}: continue if role == "system": continue diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index bd6098d3746..5cd546462dd 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -555,7 +555,7 @@ class TestLoginNousSkipKeepsCurrent: auth_path = hermes_home / "auth.json" auth_after = json.loads(auth_path.read_text()) # active_provider should NOT be set to "nous" after Skip - assert auth_after.get("active_provider") in (None, "") + assert auth_after.get("active_provider") in {None, ""} # But Nous creds are still saved assert "nous" in auth_after.get("providers", {}) diff --git a/tests/hermes_cli/test_cmd_update.py b/tests/hermes_cli/test_cmd_update.py index 2f4b836286b..b9087c06663 100644 --- a/tests/hermes_cli/test_cmd_update.py +++ b/tests/hermes_cli/test_cmd_update.py @@ -162,7 +162,7 @@ class TestCmdUpdateBranchFallback: if call.args and call.args[0][0] == "/usr/bin/npm" and call.args[0][1] == "ci" - and call.kwargs.get("cwd") in (PROJECT_ROOT, PROJECT_ROOT / "ui-tui") + and call.kwargs.get("cwd") in {PROJECT_ROOT, PROJECT_ROOT / "ui-tui"} ] assert len(repo_and_tui_calls) == 2 for call in repo_and_tui_calls: diff --git a/tests/hermes_cli/test_codex_runtime_switch.py b/tests/hermes_cli/test_codex_runtime_switch.py index 7bf1a59e1e7..a0b4aa5fd41 100644 --- a/tests/hermes_cli/test_codex_runtime_switch.py +++ b/tests/hermes_cli/test_codex_runtime_switch.py @@ -105,7 +105,7 @@ class TestApply: assert "Cannot enable" in r.message assert "npm i -g @openai/codex" in r.message # Config NOT mutated on failure - assert cfg.get("model", {}).get("openai_runtime") in (None, "") + assert cfg.get("model", {}).get("openai_runtime") in {None, ""} def test_enable_succeeds_when_codex_present(self): cfg = {} diff --git a/tests/hermes_cli/test_install_cua_driver.py b/tests/hermes_cli/test_install_cua_driver.py index 42a49e22b5d..6cd50261694 100644 --- a/tests/hermes_cli/test_install_cua_driver.py +++ b/tests/hermes_cli/test_install_cua_driver.py @@ -48,7 +48,7 @@ class TestInstallCuaDriverUpgrade: with patch("platform.system", return_value="Darwin"), \ patch.object(tools_config.shutil, "which", side_effect=lambda n: "/usr/local/bin/" + n - if n in ("cua-driver", "curl") else None), \ + if n in {"cua-driver", "curl"} else None), \ patch.object(tools_config, "_run_cua_driver_installer", return_value=True) as runner, \ patch("subprocess.run"): @@ -82,7 +82,7 @@ class TestInstallCuaDriverUpgrade: with patch("platform.system", return_value="Darwin"), \ patch.object(tools_config.shutil, "which", side_effect=lambda n: "/usr/local/bin/" + n - if n in ("cua-driver", "curl") else None), \ + if n in {"cua-driver", "curl"} else None), \ patch.object(tools_config, "_run_cua_driver_installer") as runner, \ patch("subprocess.run"): assert tools_config.install_cua_driver(upgrade=False) is True diff --git a/tests/hermes_cli/test_kanban_core_functionality.py b/tests/hermes_cli/test_kanban_core_functionality.py index 17252af827a..35dc7ace951 100644 --- a/tests/hermes_cli/test_kanban_core_functionality.py +++ b/tests/hermes_cli/test_kanban_core_functionality.py @@ -1046,7 +1046,7 @@ def test_enforce_max_runtime_integrates_with_dispatch(kanban_home, monkeypatch): task = kb.get_task(conn, tid) # After timeout, task is back in 'ready' and will be re-spawned # by the same pass. That's the intended behaviour. - assert task.status in ("ready", "running") + assert task.status in {"ready", "running"} finally: conn.close() diff --git a/tests/hermes_cli/test_memory_reset.py b/tests/hermes_cli/test_memory_reset.py index 3b91326de20..48f1cfda6a7 100644 --- a/tests/hermes_cli/test_memory_reset.py +++ b/tests/hermes_cli/test_memory_reset.py @@ -43,9 +43,9 @@ def _run_memory_reset(target="all", yes=False, monkeypatch=None, confirm_input=" mem_dir = get_hermes_home() / "memories" files_to_reset = [] - if target in ("all", "memory"): + if target in {"all", "memory"}: files_to_reset.append(("MEMORY.md", "agent notes")) - if target in ("all", "user"): + if target in {"all", "user"}: files_to_reset.append(("USER.md", "user profile")) existing = [(f, desc) for f, desc in files_to_reset if (mem_dir / f).exists()] diff --git a/tests/hermes_cli/test_models.py b/tests/hermes_cli/test_models.py index 8ccf5b57f2d..78568f81f2c 100644 --- a/tests/hermes_cli/test_models.py +++ b/tests/hermes_cli/test_models.py @@ -252,7 +252,7 @@ class TestDetectProviderForModel: result = detect_provider_for_model("deepseek-chat", "openai-codex") assert result is not None # Provider is deepseek (direct) or openrouter (fallback) depending on creds - assert result[0] in ("deepseek", "openrouter") + assert result[0] in {"deepseek", "openrouter"} def test_current_provider_model_returns_none(self): """Models belonging to the current provider should not trigger a switch.""" @@ -302,7 +302,7 @@ class TestDetectProviderForModel: with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): result = detect_provider_for_model("claude-opus-4-6", "openai-codex") assert result is not None - assert result[0] not in ("nous",) # nous has claude models but shouldn't be suggested + assert result[0] not in {"nous",} # nous has claude models but shouldn't be suggested class TestIsNousFreeTier: diff --git a/tests/hermes_cli/test_opencode_go_in_model_list.py b/tests/hermes_cli/test_opencode_go_in_model_list.py index 6020c817979..f784f75f31b 100644 --- a/tests/hermes_cli/test_opencode_go_in_model_list.py +++ b/tests/hermes_cli/test_opencode_go_in_model_list.py @@ -44,7 +44,7 @@ def test_opencode_go_appears_when_api_key_set(): # opencode-go can appear as "built-in" (from PROVIDER_TO_MODELS_DEV when # models.dev is reachable) or "hermes" (from HERMES_OVERLAYS fallback when # the API is unavailable, e.g. in CI). - assert opencode_go["source"] in ("built-in", "hermes") + assert opencode_go["source"] in {"built-in", "hermes"} def test_opencode_go_not_appears_when_no_creds(): diff --git a/tests/hermes_cli/test_update_stale_dashboard.py b/tests/hermes_cli/test_update_stale_dashboard.py index 546fd489911..e79caeb9dc6 100644 --- a/tests/hermes_cli/test_update_stale_dashboard.py +++ b/tests/hermes_cli/test_update_stale_dashboard.py @@ -237,7 +237,7 @@ class TestKillStaleDashboardPosix: sent.append((pid, sig)) # Simulate stubborn process: probe (sig 0) always succeeds, # SIGTERM does nothing, SIGKILL is where it "dies". - if sig in (_signal.SIGTERM, 0, _signal.SIGKILL): + if sig in {_signal.SIGTERM, 0, _signal.SIGKILL}: return # Any other signal — also fine. diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 4d177f92b38..ca2876f0f5c 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -306,7 +306,7 @@ class TestWebServerEndpoints: resp = self.client.get("/api/auth/session-token") # The endpoint is gone — the catch-all SPA route serves index.html # or the middleware returns 401 for unauthenticated /api/ paths. - assert resp.status_code in (200, 404) + assert resp.status_code in {200, 404} # Either way, it must NOT return the token as JSON try: data = resp.json() @@ -333,7 +333,7 @@ class TestWebServerEndpoints: # %2e%2e = .. resp = self.client.get("/%2e%2e/%2e%2e/etc/passwd") # Should return 200 with index.html (SPA fallback), not the actual file - assert resp.status_code in (200, 404) + assert resp.status_code in {200, 404} if resp.status_code == 200: # Should be the SPA fallback, not the system file assert "root:" not in resp.text @@ -341,7 +341,7 @@ class TestWebServerEndpoints: def test_path_traversal_dotdot_blocked(self): """Direct .. path traversal via encoded sequences.""" resp = self.client.get("/%2e%2e/hermes_cli/web_server.py") - assert resp.status_code in (200, 404) + assert resp.status_code in {200, 404} if resp.status_code == 200: assert "FastAPI" not in resp.text # Should not serve the actual source @@ -535,7 +535,7 @@ class TestConfigRoundTrip: if val is None: continue # not set in user config — fine expected = entry["type"] - if expected in ("string", "select") and not isinstance(val, str): + if expected in {"string", "select"} and not isinstance(val, str): mismatches.append(f"{key}: expected str, got {type(val).__name__}") elif expected == "number" and not isinstance(val, (int, float)): mismatches.append(f"{key}: expected number, got {type(val).__name__}") @@ -1032,7 +1032,7 @@ class TestNewEndpoints: """GET /api/auth/session-token no longer exists.""" resp = self.client.get("/api/auth/session-token") # Should not return a JSON token object - assert resp.status_code in (200, 404) + assert resp.status_code in {200, 404} try: data = resp.json() assert "token" not in data diff --git a/tests/honcho_plugin/test_session.py b/tests/honcho_plugin/test_session.py index 64fcfc7ebfd..57724432348 100644 --- a/tests/honcho_plugin/test_session.py +++ b/tests/honcho_plugin/test_session.py @@ -1570,7 +1570,7 @@ class TestDialecticLifecycleSmoke: self._await_thread(provider) assert mgr.dialectic_query.call_count == 2, "turn 4 cadence fire" _, kwargs = mgr.dialectic_query.call_args - assert kwargs.get("reasoning_level") in ("medium", "high"), \ + assert kwargs.get("reasoning_level") in {"medium", "high"}, \ f"long query must bump reasoning level above 'low'; got {kwargs.get('reasoning_level')}" assert provider._last_dialectic_turn == 4, "cadence tracker advances on success" diff --git a/tests/plugins/test_achievements_plugin.py b/tests/plugins/test_achievements_plugin.py index 782aea7b397..2d908b3d46e 100644 --- a/tests/plugins/test_achievements_plugin.py +++ b/tests/plugins/test_achievements_plugin.py @@ -271,7 +271,7 @@ def test_evaluate_all_force_runs_synchronously(plugin_api): # Synchronous — snapshot is fresh on return. assert result["scan_meta"].get("sessions_total") == 25 - assert result["scan_meta"]["mode"] in ("full", "incremental") + assert result["scan_meta"]["mode"] in {"full", "incremental"} def test_start_background_scan_is_idempotent_while_running(plugin_api): diff --git a/tests/plugins/video_gen/test_xai_plugin.py b/tests/plugins/video_gen/test_xai_plugin.py index bd7a880fdee..4c365020a32 100644 --- a/tests/plugins/video_gen/test_xai_plugin.py +++ b/tests/plugins/video_gen/test_xai_plugin.py @@ -110,4 +110,4 @@ def test_xai_no_operation_kwarg(): result = XAIVideoGenProvider().generate("x", operation="generate") assert result["success"] is False # auth_required, NOT some signature error - assert result["error_type"] in ("auth_required", "api_error") + assert result["error_type"] in {"auth_required", "api_error"} diff --git a/tests/run_agent/test_anthropic_truncation_continuation.py b/tests/run_agent/test_anthropic_truncation_continuation.py index 872015bc0bc..4e87a33e9d8 100644 --- a/tests/run_agent/test_anthropic_truncation_continuation.py +++ b/tests/run_agent/test_anthropic_truncation_continuation.py @@ -106,9 +106,9 @@ class TestContinuationLogicBranching: def test_all_three_api_modes_hit_continuation_branch(self, api_mode): # The guard in run_agent.py is: # if self.api_mode in ("chat_completions", "bedrock_converse", "anthropic_messages"): - assert api_mode in ("chat_completions", "bedrock_converse", "anthropic_messages") + assert api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"} def test_codex_responses_still_excluded(self): # codex_responses has its own truncation path (not continuation-based) # and should NOT be routed through the shared block. - assert "codex_responses" not in ("chat_completions", "bedrock_converse", "anthropic_messages") + assert "codex_responses" not in {"chat_completions", "bedrock_converse", "anthropic_messages"} diff --git a/tests/skills/test_openclaw_migration.py b/tests/skills/test_openclaw_migration.py index 708484027be..0b331c40238 100644 --- a/tests/skills/test_openclaw_migration.py +++ b/tests/skills/test_openclaw_migration.py @@ -846,7 +846,7 @@ def test_skill_installs_cleanly_under_skills_guard(): # the script never writes to that file # # Accept "caution" or "safe" — just not "dangerous" from a *real* threat. - assert result.verdict in ("safe", "caution", "dangerous"), f"Unexpected verdict: {result.verdict}" + assert result.verdict in {"safe", "caution", "dangerous"}, f"Unexpected verdict: {result.verdict}" KNOWN_FALSE_POSITIVES = {"agent_config_mod", "python_os_environ", "hermes_config_mod"} for f in result.findings: assert f.pattern_id in KNOWN_FALSE_POSITIVES, f"Unexpected finding: {f}" diff --git a/tests/stress/test_atypical_scenarios.py b/tests/stress/test_atypical_scenarios.py index 2010049e14f..e7e83eabccb 100644 --- a/tests/stress/test_atypical_scenarios.py +++ b/tests/stress/test_atypical_scenarios.py @@ -902,7 +902,7 @@ def _(home, kb): pass # Empty body → accept (legitimate: just title says it all) tid = kb.create_task(conn, title="empty body ok", body="", assignee="w") - assert kb.get_task(conn, tid).body in ("", None) + assert kb.get_task(conn, tid).body in {"", None} # Empty summary on complete → accept kb.claim_task(conn, tid) kb.complete_task(conn, tid, summary="") @@ -994,7 +994,7 @@ def _(home, kb): # Empty title r = client.post("/api/plugins/kanban/tasks", json={"title": ""}) - assert r.status_code in (400, 422), f"empty title should 4xx, got {r.status_code}" + assert r.status_code in {400, 422}, f"empty title should 4xx, got {r.status_code}" # Title only r = client.post("/api/plugins/kanban/tasks", json={"title": "x"}) @@ -1019,7 +1019,7 @@ def _(home, kb): r = client.post("/api/plugins/kanban/tasks", json={ "title": "fine", "nonexistent_field": "whatever", }) - assert r.status_code in (200, 422) + assert r.status_code in {200, 422} # Priority as non-int r = client.post("/api/plugins/kanban/tasks", json={"title": "prio", "priority": "high"}) @@ -1028,7 +1028,7 @@ def _(home, kb): # PATCH with empty body (no changes requested) r = client.patch(f"/api/plugins/kanban/tasks/{tid}", json={}) # Accept either success-no-op or 400 - assert r.status_code in (200, 400) + assert r.status_code in {200, 400} print(" dashboard REST handles weird inputs correctly") # ============================================================================= diff --git a/tests/test_live_system_guard_self_test.py b/tests/test_live_system_guard_self_test.py index 1856935b240..3bbe8c9f3b0 100644 --- a/tests/test_live_system_guard_self_test.py +++ b/tests/test_live_system_guard_self_test.py @@ -259,7 +259,7 @@ def test_kill_own_subtree_passes_through(): finally: p.wait(timeout=2) # SIGTERM = 15; subprocess returncode is -15 on POSIX. - assert p.returncode in (-signal.SIGTERM, 128 + int(signal.SIGTERM)) + assert p.returncode in {-signal.SIGTERM, 128 + int(signal.SIGTERM)} def test_subprocess_pkill_with_unrelated_pattern_passes_through(): diff --git a/tests/test_timezone.py b/tests/test_timezone.py index ffb831617d9..f91a27b6a75 100644 --- a/tests/test_timezone.py +++ b/tests/test_timezone.py @@ -63,7 +63,7 @@ class TestHermesTimeNow: assert result.tzinfo is not None # Offset is -5h or -4h depending on DST offset_hours = result.utcoffset().total_seconds() / 3600 - assert offset_hours in (-5, -4) + assert offset_hours in {-5, -4} def test_invalid_timezone_falls_back(self, caplog): """Invalid timezone logs warning and falls back to server-local.""" diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 0d5bad8e875..24a34e75c17 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -3718,7 +3718,7 @@ def test_prompt_submit_preserves_empty_response_without_error(monkeypatch): assert payload.get("status") == "complete" # Text stays empty — we did NOT fabricate an "Error:" string text = payload.get("text", "") - assert text in ("", None), f"expected empty text, got {text!r}" + assert text in {"", None}, f"expected empty text, got {text!r}" # ── session.most_recent ────────────────────────────────────────────── diff --git a/tests/tools/test_browser_homebrew_paths.py b/tests/tools/test_browser_homebrew_paths.py index 7e4d1c70222..7edf6f6c67d 100644 --- a/tests/tools/test_browser_homebrew_paths.py +++ b/tests/tools/test_browser_homebrew_paths.py @@ -68,10 +68,10 @@ class TestDiscoverHomebrewNodeDirs: if p == "/opt/homebrew/opt": return True # node@20/bin and node@24/bin exist - if p in ( + if p in { "/opt/homebrew/opt/node@20/bin", "/opt/homebrew/opt/node@24/bin", - ): + }: return True return False @@ -171,10 +171,10 @@ class TestFindAgentBrowser: real_isdir = os.path.isdir def selective_isdir(path): - if path in ( + if path in { "/data/data/com.termux/files/usr/bin", "/data/data/com.termux/files/usr/sbin", - ): + }: return True return real_isdir(path) @@ -486,10 +486,10 @@ class TestRunBrowserCommandPathConstruction: real_isdir = os.path.isdir def selective_isdir(path): - if path in ( + if path in { "/data/data/com.termux/files/usr/bin", "/data/data/com.termux/files/usr/sbin", - ): + }: return True if path.startswith(str(tmp_path)): return True diff --git a/tests/tools/test_code_execution_modes.py b/tests/tools/test_code_execution_modes.py index 4e22fe6e7a2..e5e2d2262ff 100644 --- a/tests/tools/test_code_execution_modes.py +++ b/tests/tools/test_code_execution_modes.py @@ -125,7 +125,7 @@ class TestResolveChildPython(unittest.TestCase): def test_project_with_no_venv_falls_back(self): """Project mode without VIRTUAL_ENV or CONDA_PREFIX → sys.executable.""" env = {k: v for k, v in os.environ.items() - if k not in ("VIRTUAL_ENV", "CONDA_PREFIX")} + if k not in {"VIRTUAL_ENV", "CONDA_PREFIX"}} with patch.dict(os.environ, env, clear=True): self.assertEqual(_resolve_child_python("project"), sys.executable) diff --git a/tests/tools/test_discord_tool.py b/tests/tools/test_discord_tool.py index 41d2cc957be..19a31d10457 100644 --- a/tests/tools/test_discord_tool.py +++ b/tests/tools/test_discord_tool.py @@ -633,7 +633,7 @@ class TestToolsetInclusion: def test_discord_tools_not_in_other_toolsets(self): from toolsets import TOOLSETS for name, ts in TOOLSETS.items(): - if name in ("hermes-discord", "hermes-gateway", "discord", "discord_admin"): + if name in {"hermes-discord", "hermes-gateway", "discord", "discord_admin"}: continue tools = ts.get("tools", []) assert "discord" not in tools or name == "discord", ( diff --git a/tests/tools/test_hidden_dir_filter.py b/tests/tools/test_hidden_dir_filter.py index d7c10846bea..c7757864f74 100644 --- a/tests/tools/test_hidden_dir_filter.py +++ b/tests/tools/test_hidden_dir_filter.py @@ -24,7 +24,7 @@ def _new_filter_matches(path: Path) -> bool: Returns True when the path SHOULD be filtered out. """ - return any(part in ('.git', '.github', '.hub') for part in path.parts) + return any(part in {'.git', '.github', '.hub'} for part in path.parts) class TestOldFilterBrokenOnWindows: diff --git a/tests/tools/test_managed_modal_environment.py b/tests/tools/test_managed_modal_environment.py index d36418336cc..8380e49058c 100644 --- a/tests/tools/test_managed_modal_environment.py +++ b/tests/tools/test_managed_modal_environment.py @@ -33,7 +33,7 @@ def _restore_tool_and_agent_modules(): original_modules = { name: module for name, module in sys.modules.items() - if name in ("tools", "agent", "hermes_cli") + if name in {"tools", "agent", "hermes_cli"} or name.startswith("tools.") or name.startswith("agent.") or name.startswith("hermes_cli.") diff --git a/tests/tools/test_mcp_cancelled_error_propagation.py b/tests/tools/test_mcp_cancelled_error_propagation.py index ce05d03f43a..c0e91f31531 100644 --- a/tests/tools/test_mcp_cancelled_error_propagation.py +++ b/tests/tools/test_mcp_cancelled_error_propagation.py @@ -62,7 +62,7 @@ class TestCancelledErrorPropagation: return "clean_return" outcome = asyncio.run(drive()) - assert outcome in ("cancelled_cleanly", "clean_return"), ( + assert outcome in {"cancelled_cleanly", "clean_return"}, ( f"MCPServerTask.run wedged on cancel (outcome={outcome}) — " f"#9930 regression" ) diff --git a/tests/tools/test_singularity_preflight.py b/tests/tools/test_singularity_preflight.py index 0ba50c3e93d..fa0a0ea4d52 100644 --- a/tests/tools/test_singularity_preflight.py +++ b/tests/tools/test_singularity_preflight.py @@ -23,7 +23,7 @@ class TestFindSingularityExecutable: def test_prefers_apptainer(self): """When both are available, apptainer should be preferred.""" def which_both(name): - return f"/usr/bin/{name}" if name in ("apptainer", "singularity") else None + return f"/usr/bin/{name}" if name in {"apptainer", "singularity"} else None with patch("shutil.which", side_effect=which_both): assert _find_singularity_executable() == "apptainer" diff --git a/tests/tools/test_skill_manager_tool.py b/tests/tools/test_skill_manager_tool.py index 96c3a361f0c..33efbb98ae8 100644 --- a/tests/tools/test_skill_manager_tool.py +++ b/tests/tools/test_skill_manager_tool.py @@ -547,7 +547,7 @@ class TestSkillManageDispatcher: # No provenance marker on a foreground create — record either missing # entirely (telemetry best-effort) or present with created_by unset. rec = usage.get("test-skill") or {} - assert rec.get("created_by") in (None, "", False) + assert rec.get("created_by") in {None, "", False} def test_create_from_background_review_marks_agent_created(self, tmp_path): """Background-review fork creates ARE marked as agent-created.""" diff --git a/tests/tools/test_skills_hub.py b/tests/tools/test_skills_hub.py index b7c483d1a16..e831b50943e 100644 --- a/tests/tools/test_skills_hub.py +++ b/tests/tools/test_skills_hub.py @@ -101,7 +101,7 @@ class TestTrustLevelFor: src = self._source() result = src.trust_level_for("owner/repo") # No path part — still resolves repo correctly - assert result in ("trusted", "community") + assert result in {"trusted", "community"} # --------------------------------------------------------------------------- diff --git a/tests/tui_gateway/test_entry_sys_path.py b/tests/tui_gateway/test_entry_sys_path.py index f8741b18e4b..e7f9e47cee0 100644 --- a/tests/tui_gateway/test_entry_sys_path.py +++ b/tests/tui_gateway/test_entry_sys_path.py @@ -25,7 +25,7 @@ def _reload_entry_with_env(env_overrides: dict) -> None: _src_root = os.environ.get("HERMES_PYTHON_SRC_ROOT", "") if _src_root and _src_root not in sys.path: sys.path.insert(0, _src_root) - sys.path = [p for p in sys.path if p not in ("", ".")] + sys.path = [p for p in sys.path if p not in {"", "."}] return sys.path[:] finally: sys.path = original_path @@ -45,7 +45,7 @@ def test_empty_string_and_dot_removed_from_sys_path(): assert "." in sys.path # Run the entry.py fixup logic directly - sys.path = [p for p in sys.path if p not in ("", ".")] + sys.path = [p for p in sys.path if p not in {"", "."}] assert "" not in sys.path assert "." not in sys.path @@ -61,7 +61,7 @@ def test_hermes_src_root_inserted_at_front(): _src_root = os.environ.get("HERMES_PYTHON_SRC_ROOT", "") if _src_root and _src_root not in sys.path: sys.path.insert(0, _src_root) - sys.path = [p for p in sys.path if p not in ("", ".")] + sys.path = [p for p in sys.path if p not in {"", "."}] assert sys.path[0] == fake_root finally: @@ -79,7 +79,7 @@ def test_src_root_not_duplicated_if_already_present(): _src_root = os.environ.get("HERMES_PYTHON_SRC_ROOT", "") if _src_root and _src_root not in sys.path: sys.path.insert(0, _src_root) - sys.path = [p for p in sys.path if p not in ("", ".")] + sys.path = [p for p in sys.path if p not in {"", "."}] assert sys.path.count(fake_root) == count_before finally: @@ -95,7 +95,7 @@ def test_no_src_root_env_does_not_crash(): _src_root = os.environ.get("HERMES_PYTHON_SRC_ROOT", "") if _src_root and _src_root not in sys.path: sys.path.insert(0, _src_root) - sys.path = [p for p in sys.path if p not in ("", ".")] + sys.path = [p for p in sys.path if p not in {"", "."}] # No exception raised finally: sys.path = original diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index faaf7ec42bf..c7d7730c756 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -450,7 +450,7 @@ def ensure(feature: str, *, prompt: bool = True) -> None: ).strip().lower() except (EOFError, KeyboardInterrupt): answer = "n" - if answer and answer not in ("y", "yes"): + if answer and answer not in {"y", "yes"}: raise FeatureUnavailable( feature, missing, "user declined install at prompt" ) diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index a46496ef59c..9cec72524af 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -540,7 +540,7 @@ def _validate_remote_mcp_url(server_name: str, url: Any) -> str: raise InvalidMcpUrlError( f"Invalid MCP URL for '{server_name}': {stripped!r} ({exc})" ) from exc - if parsed.scheme.lower() not in ("http", "https"): + if parsed.scheme.lower() not in {"http", "https"}: raise InvalidMcpUrlError( f"Invalid MCP URL for '{server_name}': scheme must be http or " f"https, got {parsed.scheme!r} ({stripped!r})" diff --git a/tools/video_generation_tool.py b/tools/video_generation_tool.py index 63d80165dc0..472b8409255 100644 --- a/tools/video_generation_tool.py +++ b/tools/video_generation_tool.py @@ -286,9 +286,9 @@ def _coerce_bool(value: Any) -> Optional[bool]: return value if isinstance(value, str): v = value.strip().lower() - if v in ("true", "1", "yes", "on"): + if v in {"true", "1", "yes", "on"}: return True - if v in ("false", "0", "no", "off"): + if v in {"false", "0", "no", "off"}: return False return None diff --git a/tools/x_search_tool.py b/tools/x_search_tool.py index 8b242ee0ca8..1b7685a897d 100644 --- a/tools/x_search_tool.py +++ b/tools/x_search_tool.py @@ -147,7 +147,7 @@ def _extract_response_text(payload: Dict[str, Any]) -> str: continue for content in item.get("content", []) or []: ctype = content.get("type") - if ctype in ("output_text", "text"): + if ctype in {"output_text", "text"}: text = str(content.get("text") or "").strip() if text: parts.append(text) From d87b27cff86fe5dcf07cbdb073608674f8b92b3c Mon Sep 17 00:00:00 2001 From: Yanzhong Su Date: Thu, 14 May 2026 19:27:17 +0100 Subject: [PATCH 064/142] fix(gateway): add codex runtime telegram alias --- hermes_cli/commands.py | 3 ++- tests/hermes_cli/test_commands.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index 07e5b5e5c4a..1e42fb9421e 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -123,7 +123,8 @@ COMMAND_REGISTRY: list[CommandDef] = [ CommandDef("model", "Switch model for this session", "Configuration", aliases=("provider",), args_hint="[model] [--provider name] [--global]"), CommandDef("codex-runtime", "Toggle codex app-server runtime for OpenAI/Codex models", - "Configuration", args_hint="[auto|codex_app_server]"), + "Configuration", aliases=("codex_runtime",), + args_hint="[auto|codex_app_server]"), CommandDef("gquota", "Show Google Gemini Code Assist quota usage", "Info", cli_only=True), diff --git a/tests/hermes_cli/test_commands.py b/tests/hermes_cli/test_commands.py index d08f886fa6a..6de778347e1 100644 --- a/tests/hermes_cli/test_commands.py +++ b/tests/hermes_cli/test_commands.py @@ -107,6 +107,7 @@ class TestResolveCommand: assert resolve_command("gateway").name == "platforms" assert resolve_command("set-home").name == "sethome" assert resolve_command("reload_mcp").name == "reload-mcp" + assert resolve_command("codex_runtime").name == "codex-runtime" assert resolve_command("tasks").name == "agents" def test_topic_is_gateway_command(self): @@ -251,6 +252,12 @@ class TestTelegramBotCommands: assert "queue" in names assert "steer" in names + def test_hyphenated_codex_runtime_is_exposed_as_underscore_command(self): + """Telegram autocomplete exposes /codex-runtime as /codex_runtime.""" + names = {name for name, _ in telegram_bot_commands()} + assert "codex_runtime" in names + assert "codex-runtime" not in names + class TestSlackSubcommandMap: def test_returns_dict(self): From 5a2a858b84c3e189c7d4ab7db94205c0a2ef480f Mon Sep 17 00:00:00 2001 From: haran2001 <56040092+haran2001@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:27 -0700 Subject: [PATCH 065/142] test(restart_drain): assert i18n catalog resolved (#22266) The restart-drain test previously asserted equality between two calls to t("gateway.draining", count=1), which masked the original xdist failure mode in #22266: if the locale catalog is not resolved from the worker's import path, t() returns the bare key path and both sides of the equality still match. Add a guard that the resolved value is not the raw catalog key and contains the English placeholder substitution. This keeps the test loudly failing when locale resolution silently degrades. --- tests/gateway/test_restart_drain.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/gateway/test_restart_drain.py b/tests/gateway/test_restart_drain.py index 844af427308..9000e4d4820 100644 --- a/tests/gateway/test_restart_drain.py +++ b/tests/gateway/test_restart_drain.py @@ -33,7 +33,16 @@ async def test_restart_command_while_busy_requests_drain_without_interrupt(monke result = await runner._handle_message(event) - assert result == t("gateway.draining", count=1) + expected = t("gateway.draining", count=1) + assert result == expected + # Guard against the silent-degradation regression in #22266: if the i18n + # catalog cannot be resolved (e.g. xdist workers losing the locales path) + # then ``t("gateway.draining", count=1)`` returns the bare key + # ``"gateway.draining"`` instead of the formatted English string, and both + # sides of the equality above would still match. Assert on the catalog + # output explicitly so a broken locale resolution fails loudly here. + assert expected != "gateway.draining" + assert "Draining" in expected and "1" in expected running_agent.interrupt.assert_not_called() runner.request_restart.assert_called_once_with(detached=True, via_service=False) From d9abbe7fa4c69333a226b7a2d366713b3f071187 Mon Sep 17 00:00:00 2001 From: haran2001 <56040092+haran2001@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:27 -0700 Subject: [PATCH 066/142] fix(metadata): qwen3.6-plus has a 1M context window (#27008) qwen3.6-plus did not have an explicit entry in DEFAULT_CONTEXT_LENGTHS, so the longest-substring fallback matched the generic 'qwen': 131072 catch-all. That dropped the effective context limit from 1,048,576 tokens to 131,072, prematurely lowered the compression threshold, and produced misleading warnings about main/compression context mismatch in long sessions. Add an explicit 'qwen3.6-plus': 1048576 entry before the catch-all and cover it with a regression test (bare, qwen/, and dashscope/ prefixes). Note: PR #6599 also mentions touching model_metadata.py but the actual diff only edits hermes_cli/models.py, so this fix is independent and not duplicated by that PR. Closes #27008 --- agent/model_metadata.py | 1 + tests/agent/test_model_metadata.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 26a844ccb92..b8ec0d6509e 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -194,6 +194,7 @@ DEFAULT_CONTEXT_LENGTHS = { "llama": 131072, # Qwen — specific model families before the catch-all. # Official docs: https://help.aliyun.com/zh/model-studio/developer-reference/ + "qwen3.6-plus": 1048576, # 1M context (DashScope/Alibaba & OpenRouter) "qwen3-coder-plus": 1000000, # 1M context "qwen3-coder": 262144, # 256K context "qwen": 131072, diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 7686364dcac..4f2b51293a6 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -746,6 +746,16 @@ class TestGetModelContextLength: mock_fetch.return_value = {} assert get_model_context_length("qwen3-coder") == 262144 + @patch("agent.model_metadata.fetch_model_metadata") + def test_qwen3_6_plus_context_length(self, mock_fetch): + """qwen3.6-plus has a 1M context window, not the generic 128K Qwen default.""" + mock_fetch.return_value = {} + assert get_model_context_length("qwen3.6-plus") == 1048576 + # Provider-prefixed variants must resolve to the same explicit entry + # via the longest-substring fallback (no portal/OR cache available). + assert get_model_context_length("qwen/qwen3.6-plus") == 1048576 + assert get_model_context_length("dashscope/qwen3.6-plus") == 1048576 + @patch("agent.model_metadata.fetch_model_metadata") def test_qwen_generic_context_length(self, mock_fetch): """Generic qwen models still get the 128K default.""" From 3c51da1cb709566cfd3f29b5d3405a7826e97ced Mon Sep 17 00:00:00 2001 From: ms-alan <1472110+ms-alan@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:28 -0700 Subject: [PATCH 067/142] fix(cli): sync _skill_commands after /reload-skills so Tab completion picks up new skills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tab-completion lambda captured _skill_commands at startup, so newly installed skills were missing from Tab completion even after /reload-skills reported them as added. Two changes: 1. Tab-completion lambda now calls get_skill_commands() instead of reading the module-level _skill_commands snapshot — ensures the lambda always gets fresh data without needing to touch global state. 2. _reload_skills() now syncs cli.py's module-level _skill_commands via get_skill_commands() after reload, so help display, command dispatch, and any other direct _skill_commands readers also see the updated map. Closes #26441 --- cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index e8e38965f53..6b62493d60c 100644 --- a/cli.py +++ b/cli.py @@ -2412,6 +2412,7 @@ def _looks_like_slash_command(text: str) -> bool: from agent.skill_commands import ( scan_skill_commands, + get_skill_commands, build_skill_invocation_message, build_preloaded_skills_prompt, ) @@ -9656,12 +9657,18 @@ class HermesCLI: prompt caching intact. """ try: - from agent.skill_commands import reload_skills + from agent.skill_commands import reload_skills, get_skill_commands if not self._command_running: print("🔄 Reloading skills...") result = reload_skills() + + # Sync cli.py's module-level _skill_commands so all consumers + # (help display, command dispatch, Tab-completion lambda) see the + # updated dict without needing to restart the session. + global _skill_commands + _skill_commands = get_skill_commands() added = result.get("added", []) # [{"name", "description"}, ...] removed = result.get("removed", []) # [{"name", "description"}, ...] total = result.get("total", 0) @@ -12667,7 +12674,7 @@ class HermesCLI: _completer = SlashCommandCompleter( - skill_commands_provider=lambda: _skill_commands, + skill_commands_provider=lambda: get_skill_commands(), command_filter=cli_ref._command_available, ) input_area = TextArea( From 6622277f11ca1dee03e868b064fb1851e5598b77 Mon Sep 17 00:00:00 2001 From: godlin Date: Fri, 15 May 2026 13:01:14 +0800 Subject: [PATCH 068/142] fix ACP start events for polished tools --- acp_adapter/tools.py | 1 - tests/acp/test_tools.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/acp_adapter/tools.py b/acp_adapter/tools.py index 31ae943a056..77a62e243bc 100644 --- a/acp_adapter/tools.py +++ b/acp_adapter/tools.py @@ -1123,7 +1123,6 @@ def build_tool_start( ) # Generic fallback - import json try: args_text = json.dumps(arguments, indent=2, default=str) except (TypeError, ValueError): diff --git a/tests/acp/test_tools.py b/tests/acp/test_tools.py index f9b0dac6d66..dc62b296c69 100644 --- a/tests/acp/test_tools.py +++ b/tests/acp/test_tools.py @@ -207,6 +207,16 @@ class TestBuildToolStart: assert result.content is None assert result.raw_input is None + def test_build_tool_start_for_browser_navigate(self): + """browser_navigate should emit a polished start event.""" + args = {"url": "https://x.com"} + result = build_tool_start("tc-browser-start", "browser_navigate", args) + assert isinstance(result, ToolCallStart) + assert result.title == "navigate: https://x.com" + assert result.kind == "fetch" + assert result.content[0].content.text == '{\n "url": "https://x.com"\n}' + assert result.raw_input is None + def test_build_tool_start_for_search(self): """search_files should include pattern in content.""" args = {"pattern": "TODO", "target": "content"} From 8e3cfdfb613ceb923ab9c07f8b88d4fa512b35b4 Mon Sep 17 00:00:00 2001 From: wesleysimplicio <6108320+wesleysimplicio@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:28 -0700 Subject: [PATCH 069/142] fix(webui): allow native text selection in chat via xterm.js bypass (#25720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat panel renders via xterm.js, and when the inner Hermes TUI enables mouse-events mode (CSI ?1000h family — used for nav inside Ink overlays/pickers) every drag/double-click/triple-click in the canvas is consumed by the terminal instead of producing a native text selection. The reporter (macOS, Brave) confirmed: - click-and-drag selects nothing - Cmd+C with no selection copies the entire visible buffer - existing CSS overrides and event handlers at the document layer have no effect — the issue is at xterm.js's mouse layer, not the DOM Fix: two xterm.js options the user can opt into without disabling mouse-events mode for the inner TUI: - `macOptionClickForcesSelection: true` — holding Option (macOS) or Alt (Linux/Windows) during a click-and-drag bypasses mouse-events mode and produces a native xterm selection. This is the documented xterm.js path for this exact scenario. Selected text is copyable via Cmd+C / Ctrl+C through the existing OSC 52 + manual handlers. - `rightClickSelectsWord: true` — right-click highlights the word under the pointer. Single-action path on top of the modifier-based bypass. The two options coexist with the existing `macOptionIsMeta: true` (which only affects keyboard, not mouse). No other code change needed. Fixes #25720. --- web/src/pages/ChatPage.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index 0d092c72c04..6fd32fa43fc 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -286,6 +286,17 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { fontWeight: "400", fontWeightBold: "700", macOptionIsMeta: true, + // Hold Option (Alt on Linux/Windows) to force native text selection + // even when the inner Hermes TUI has enabled xterm mouse-events + // mode (CSI ?1000h family). Without this, click-and-drag in the + // chat canvas selects nothing and Cmd+C falls back to copying the + // entire visible buffer, which is rarely what the user wants. + // See #25720. + macOptionClickForcesSelection: true, + // Right-click selects the word under the pointer. xterm.js default + // is false; enabling it gives users a single-action selection + // path on top of the modifier-based bypass above. + rightClickSelectsWord: true, // Single-scroll-system experiment: // let the inner Hermes TUI own transcript history/scroll behavior. // The outer browser xterm should act as a display/input bridge only. From aeda146112c840372ae6f091c28d6379d8db6509 Mon Sep 17 00:00:00 2001 From: flamiinngo Date: Sun, 17 May 2026 03:31:08 +0100 Subject: [PATCH 070/142] fix(security): honor shell hook blocks even when message/reason is absent _parse_response in agent/shell_hooks.py only forwarded a pre_tool_call block directive if the hook also provided a non-empty message or reason. When either field was missing the function returned None, causing Hermes to treat the response as a no-op and execute the tool unconditionally. This means a hook that outputs {"action": "block"} or {"decision": "block"} without a reason string is silently ignored. The security boundary fails open: tools the user intended to gate are executed anyway. Fix: remove the message-presence guard. Honor the block unconditionally and fall back to a default message when none is provided. Existing hooks that already include a message or reason are unaffected. --- agent/shell_hooks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index bad5388f88b..687af5ec4ba 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -515,13 +515,11 @@ def _parse_response(event: str, stdout: str) -> Optional[Dict[str, Any]]: if event == "pre_tool_call": if data.get("action") == "block": - message = data.get("message") or data.get("reason") or "" - if isinstance(message, str) and message: - return {"action": "block", "message": message} + message = data.get("message") or data.get("reason") or "Blocked by shell hook." + return {"action": "block", "message": message} if data.get("decision") == "block": - message = data.get("reason") or data.get("message") or "" - if isinstance(message, str) and message: - return {"action": "block", "message": message} + message = data.get("reason") or data.get("message") or "Blocked by shell hook." + return {"action": "block", "message": message} return None context = data.get("context") From 63805965e7a907f6b5e3a687fc37bed2004e7634 Mon Sep 17 00:00:00 2001 From: flamiinngo Date: Sun, 17 May 2026 03:48:42 +0100 Subject: [PATCH 071/142] fix(security): restore type safety and extract constant in shell hook block handler Address code review feedback on _parse_response: 1. Restore isinstance(raw, str) guard so non-string message/reason values (e.g. integers, lists) from a malformed hook response fall back to the default rather than being forwarded as-is. This keeps the contract that message in the returned dict is always a string. 2. Extract the repeated literal 'Blocked by shell hook.' into a module-level constant _DEFAULT_BLOCK_MESSAGE to avoid duplication and make it easy to change in one place. Four new unit tests added to tests/agent/test_shell_hooks.py covering: - action block with no message (uses default) - decision block with no reason (uses default) - action block with empty string message (uses default) - action block with non-string message, e.g. integer (uses default) --- agent/shell_hooks.py | 7 +++++-- tests/agent/test_shell_hooks.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index 687af5ec4ba..6639700b553 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -83,6 +83,7 @@ logger = logging.getLogger(__name__) DEFAULT_TIMEOUT_SECONDS = 60 MAX_TIMEOUT_SECONDS = 300 ALLOWLIST_FILENAME = "shell-hooks-allowlist.json" +_DEFAULT_BLOCK_MESSAGE = "Blocked by shell hook." # (event, matcher, command) triples that have been wired to the plugin # manager in the current process. Matcher is part of the key because @@ -515,10 +516,12 @@ def _parse_response(event: str, stdout: str) -> Optional[Dict[str, Any]]: if event == "pre_tool_call": if data.get("action") == "block": - message = data.get("message") or data.get("reason") or "Blocked by shell hook." + raw = data.get("message") or data.get("reason") + message = raw if isinstance(raw, str) and raw else _DEFAULT_BLOCK_MESSAGE return {"action": "block", "message": message} if data.get("decision") == "block": - message = data.get("reason") or data.get("message") or "Blocked by shell hook." + raw = data.get("reason") or data.get("message") + message = raw if isinstance(raw, str) and raw else _DEFAULT_BLOCK_MESSAGE return {"action": "block", "message": message} return None diff --git a/tests/agent/test_shell_hooks.py b/tests/agent/test_shell_hooks.py index 088c23eb466..743c9acb843 100644 --- a/tests/agent/test_shell_hooks.py +++ b/tests/agent/test_shell_hooks.py @@ -100,6 +100,30 @@ class TestParseResponse: ) assert r is None + def test_block_action_without_message_uses_default(self): + """Block is honored even when message/reason is absent.""" + r = shell_hooks._parse_response("pre_tool_call", '{"action": "block"}') + assert r == {"action": "block", "message": shell_hooks._DEFAULT_BLOCK_MESSAGE} + + def test_block_decision_without_reason_uses_default(self): + """Block is honored even when reason/message is absent.""" + r = shell_hooks._parse_response("pre_tool_call", '{"decision": "block"}') + assert r == {"action": "block", "message": shell_hooks._DEFAULT_BLOCK_MESSAGE} + + def test_block_action_empty_message_uses_default(self): + """Empty string message falls back to default, not empty string.""" + r = shell_hooks._parse_response( + "pre_tool_call", '{"action": "block", "message": ""}', + ) + assert r == {"action": "block", "message": shell_hooks._DEFAULT_BLOCK_MESSAGE} + + def test_block_action_non_string_message_uses_default(self): + """Non-string message (e.g. integer) falls back to default.""" + r = shell_hooks._parse_response( + "pre_tool_call", '{"action": "block", "message": 42}', + ) + assert r == {"action": "block", "message": shell_hooks._DEFAULT_BLOCK_MESSAGE} + # ── _serialize_payload ──────────────────────────────────────────────────── From dbeaaa47f2df6ce11906ab9cdf386e80b3a0a427 Mon Sep 17 00:00:00 2001 From: flamiinngo Date: Sun, 17 May 2026 03:58:10 +0100 Subject: [PATCH 072/142] refactor(security): extract _block_message helper to unify block logic in _parse_response Both the `action=block` and `decision=block` branches in _parse_response shared identical field-priority and type-validation logic. Extract it into a single _block_message(primary, secondary) helper so the two branches are one line each and the type guard lives in exactly one place. No functional change: existing tests (TestParseResponse, 14 tests) all pass unchanged, confirming identical behaviour. --- agent/shell_hooks.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index 6639700b553..79d494d7dcb 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -482,6 +482,17 @@ def _serialize_payload(event: str, kwargs: Dict[str, Any]) -> str: return json.dumps(payload, ensure_ascii=False, default=str) +def _block_message(primary: Any, secondary: Any) -> str: + """Return a validated string block message, falling back to the default. + + Accepts two candidate fields (primary wins over secondary) so callers + can express field-priority differences between the two hook wire formats + without duplicating the type-check logic. + """ + raw = primary or secondary + return raw if isinstance(raw, str) and raw else _DEFAULT_BLOCK_MESSAGE + + def _parse_response(event: str, stdout: str) -> Optional[Dict[str, Any]]: """Translate stdout JSON into a Hermes wire-shape dict. @@ -516,13 +527,9 @@ def _parse_response(event: str, stdout: str) -> Optional[Dict[str, Any]]: if event == "pre_tool_call": if data.get("action") == "block": - raw = data.get("message") or data.get("reason") - message = raw if isinstance(raw, str) and raw else _DEFAULT_BLOCK_MESSAGE - return {"action": "block", "message": message} + return {"action": "block", "message": _block_message(data.get("message"), data.get("reason"))} if data.get("decision") == "block": - raw = data.get("reason") or data.get("message") - message = raw if isinstance(raw, str) and raw else _DEFAULT_BLOCK_MESSAGE - return {"action": "block", "message": message} + return {"action": "block", "message": _block_message(data.get("reason"), data.get("message"))} return None context = data.get("context") From c9298bba06e91350aac4af8bc450b4c4f4fb225c Mon Sep 17 00:00:00 2001 From: carryzuo00 Date: Sat, 16 May 2026 09:11:59 +0000 Subject: [PATCH 073/142] fix(doctor): SSH check ignores TERMINAL_SSH_USER, TERMINAL_SSH_PORT, TERMINAL_SSH_KEY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SSH connectivity check in `run_doctor` only passed the host to ssh, using the current OS user and default port 22. When the target requires a different user (TERMINAL_SSH_USER), non-standard port (TERMINAL_SSH_PORT), or a specific identity file (TERMINAL_SSH_KEY), the check always failed with "Permission denied" — even though the agent itself connects fine. Fix: read all four TERMINAL_SSH_* env vars and build the ssh command with -p, -i, and user@host as appropriate, matching how the terminal tool actually establishes the connection. --- hermes_cli/doctor.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 07aaa2e38bc..ef668e07940 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -1073,10 +1073,20 @@ def run_doctor(args): if terminal_env == "ssh": ssh_host = os.getenv("TERMINAL_SSH_HOST") if ssh_host: + ssh_user = os.getenv("TERMINAL_SSH_USER") + ssh_port = os.getenv("TERMINAL_SSH_PORT") + ssh_key = os.getenv("TERMINAL_SSH_KEY") + target = f"{ssh_user}@{ssh_host}" if ssh_user else ssh_host + cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "BatchMode=yes"] + if ssh_port: + cmd += ["-p", ssh_port] + if ssh_key: + cmd += ["-i", os.path.expanduser(ssh_key)] + cmd += [target, "echo ok"] # Try to connect try: result = subprocess.run( - ["ssh", "-o", "ConnectTimeout=5", "-o", "BatchMode=yes", ssh_host, "echo ok"], + cmd, capture_output=True, text=True, timeout=15 From 1856bd9cc88a3790d7ccc7566aacd79ea2d1cd1c Mon Sep 17 00:00:00 2001 From: Spider-Verse Date: Fri, 15 May 2026 04:39:28 +0300 Subject: [PATCH 074/142] fix(telegram): re-trigger typing indicator after sending messages Telegram clears the typing state when a new message is delivered. When the agent sends intermediate progress messages (like 'Checking:'), the '...typing' bubble disappears immediately and doesn't return until the next keepalive tick (up to 2s later). This makes Hermes appear unresponsive during multi-tool operations. Fix: call send_typing() immediately after successful message delivery to restart the typing indicator without waiting for the next keepalive tick. Fixes #25836 --- gateway/platforms/telegram.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gateway/platforms/telegram.py b/gateway/platforms/telegram.py index 50813c25dc6..d893b8115cf 100644 --- a/gateway/platforms/telegram.py +++ b/gateway/platforms/telegram.py @@ -1663,7 +1663,17 @@ class TelegramAdapter(BasePlatformAdapter): continue raise message_ids.append(str(msg.message_id)) - + + # Re-trigger typing indicator after sending a message. + # Telegram clears the typing state when a new message is delivered, + # so without this the "...typing" bubble disappears mid-response + # (especially noticeable when the agent sends intermediate progress + # messages like "Checking:" before running tools). + try: + await self.send_typing(chat_id, metadata=metadata) + except Exception: + pass # Typing failures are non-fatal + return SendResult( success=True, message_id=message_ids[0] if message_ids else None, From c02606a385bd03630b7c76b72bf82f686a51f907 Mon Sep 17 00:00:00 2001 From: hawknewton <211668+hawknewton@users.noreply.github.com> Date: Sun, 17 May 2026 02:29:28 -0700 Subject: [PATCH 075/142] chore(deps): lazy-install boto3/botocore for bedrock adapter agent/bedrock_adapter.py now calls lazy_deps to install boto3 and botocore on first import, mirroring how other optional provider adapters defer their heavy AWS dependencies until actually used. Keeps the base install slim for users who don't run on Bedrock. --- agent/bedrock_adapter.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.py index 34eebd73ba8..620d1c99785 100644 --- a/agent/bedrock_adapter.py +++ b/agent/bedrock_adapter.py @@ -36,6 +36,19 @@ from typing import Any, Dict, List, Optional, Tuple logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Ensure boto3/botocore are installed before any code in this module runs. +# Upstream removed boto3 from [all] extras (PRs #24220, #24515); lazy_deps +# handles on-demand installation so the Bedrock provider still works in the +# EKS deployment without baking boto3 into the base image. +# --------------------------------------------------------------------------- +try: + from tools.lazy_deps import ensure + ensure("provider.bedrock", prompt=False) +except Exception: + pass # lazy_deps unavailable or install failed — let downstream imports surface the real error + + # --------------------------------------------------------------------------- # Lazy boto3 import — only loaded when the Bedrock provider is actually used. # This keeps startup fast for users who don't use Bedrock. From 150b577da52318ae14cddd934aa815291b117e14 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 02:30:17 -0700 Subject: [PATCH 076/142] chore(release): AUTHOR_MAP entries for batch salvage group 5 contributors Adds release-note attribution mappings for the contributors from group 5: - @haran2001 (PR #27070, #27068) - @ms-alan (PR #26443) - @godlin-gh (PR #26118) - @wesleysimplicio (PR #25777, ext-email form) - @Carry00 (PR #26851) - @alaamohanad169-ship-it (PR #26036) - @hawknewton (PR #26294) (YanzhongSu PR #25879 and flamiinngo PR #27231 already mapped.) --- scripts/release.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index 1b9e4bcd8f3..31bf7020ce3 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1138,6 +1138,18 @@ AUTHOR_MAP = { "sp_ps@Mac-mini.lan": "phoenixshen", # PR #26768 (respect user-configured vision model) "1594534+phoenixshen@users.noreply.github.com": "phoenixshen", "147827411+AhmetArif0@users.noreply.github.com": "AhmetArif0", # PR #26635 (line proxy env vars) + # batch salvage (May 2026 LHF run, group 5) + "hari@Hariharans-MacBook-Air-8.local": "haran2001", # PR #27070 (i18n catalog test) + "hariharan15151@gmail.com": "haran2001", # PR #27068 (qwen3.6-plus 1M context) + "56040092+haran2001@users.noreply.github.com": "haran2001", + "1472110+ms-alan@users.noreply.github.com": "ms-alan", # PR #26443 (reload-skills tab completion) + "ganlinbupt@gmail.com": "godlin-gh", # PR #26118 (ACP polished tools) + "wesley.simplicio.ext@siemens-energy.com": "wesleysimplicio", # PR #25777 (xterm.js native selection) + "6108320+wesleysimplicio@users.noreply.github.com": "wesleysimplicio", + "carryzuo00@gmail.com": "Carry00", # PR #26851 (doctor SSH env vars) + "alaamohanad169-ship-it@users.noreply.github.com": "alaamohanad169-ship-it", # PR #26036 (telegram typing after send) + "vigo@hermes": "hawknewton", # PR #26294 (bedrock boto3 lazy_deps) + "211668+hawknewton@users.noreply.github.com": "hawknewton", } From c6e6909e5a18f4c1a83eb48f0297e47fd17feed6 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 11:15:02 +0530 Subject: [PATCH 077/142] feat(browser): add BrowserProvider ABC mirroring web_search_provider template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation commit for the browser-provider plugin migration (#25214). Mirrors the architecture established by PR #25182 (web providers): - agent/browser_provider.py — BrowserProvider ABC. Preserves the legacy CloudBrowserProvider lifecycle contract bit-for-bit (create_session, close_session, emergency_cleanup, session metadata shape) so the dispatcher in tools/browser_tool.py becomes a pure registry lookup. Renames is_configured() → is_available() for parity with WebSearchProvider. - agent/browser_registry.py — selection registry with the same three-rule resolution as web_search_registry: 1. Explicit config wins (returns even if is_available() == False so the dispatcher surfaces a precise credentials error) 2. Single-eligible shortcut 3. Legacy preference walk: browser-use → browserbase, filtered by availability. Firecrawl is intentionally NOT in the legacy walk (matches pre-migration behaviour — Firecrawl was only reachable via explicit browser.cloud_provider: firecrawl). - hermes_cli/plugins.py — adds ctx.register_browser_provider() facade, one-liner mirror of register_web_search_provider(). No plugins registered yet; no dispatcher cutover yet. The next commits move browserbase/browser-use/firecrawl into plugins/browser// and switch tools/browser_tool.py over to the registry. --- agent/browser_provider.py | 155 ++++++++++++++++++++++++++ agent/browser_registry.py | 221 ++++++++++++++++++++++++++++++++++++++ hermes_cli/plugins.py | 32 ++++++ 3 files changed, 408 insertions(+) create mode 100644 agent/browser_provider.py create mode 100644 agent/browser_registry.py diff --git a/agent/browser_provider.py b/agent/browser_provider.py new file mode 100644 index 00000000000..e351d75330e --- /dev/null +++ b/agent/browser_provider.py @@ -0,0 +1,155 @@ +""" +Browser Provider ABC +==================== + +Defines the pluggable-backend interface for cloud browser providers +(Browserbase, Browser Use, Firecrawl, …). Providers register instances via +:meth:`PluginContext.register_browser_provider`; the active one (selected via +``browser.cloud_provider`` in ``config.yaml``) services every cloud-mode +``browser_*`` tool call. + +Providers live in ``/plugins/browser//`` (built-in, auto-loaded as +``kind: backend``) or ``~/.hermes/plugins/browser//`` (user, opt-in via +``plugins.enabled``). + +This ABC mirrors :class:`agent.web_search_provider.WebSearchProvider` (PR +#25182) — same shape, same registration flow, same picker integration. The +legacy in-tree ``tools.browser_providers.base.CloudBrowserProvider`` ABC was +deleted in PR #25214 (this work) along with the per-vendor inline modules in +``tools/browser_providers/``; the lifecycle contract documented below is +preserved bit-for-bit so the tool wrapper (:mod:`tools.browser_tool`) does +not have to translate. + +Session metadata contract (preserved from the legacy ``CloudBrowserProvider``):: + + { + "session_name": str, # unique name for agent-browser --session + "bb_session_id": str, # provider session ID (for close/cleanup) + "cdp_url": str, # CDP websocket URL + "features": dict, # feature flags that were enabled + "external_call_id": str, # optional, managed-gateway billing key + } + +``bb_session_id`` is a legacy key name kept verbatim for backward compat with +:mod:`tools.browser_tool` — it holds the provider's session ID regardless of +which provider is in use. +""" + +from __future__ import annotations + +import abc +from typing import Any, Dict + + +# --------------------------------------------------------------------------- +# ABC +# --------------------------------------------------------------------------- + + +class BrowserProvider(abc.ABC): + """Abstract base class for a cloud browser backend. + + Subclasses must implement :meth:`name`, :meth:`is_available`, and the + three lifecycle methods: :meth:`create_session`, :meth:`close_session`, + :meth:`emergency_cleanup`. + + The lifecycle shape preserves the legacy ``CloudBrowserProvider`` contract + bit-for-bit so the dispatcher in :mod:`tools.browser_tool` is a pure + registry lookup — no per-provider conditionals, no shape translation. + """ + + @property + @abc.abstractmethod + def name(self) -> str: + """Stable short identifier used in the ``browser.cloud_provider`` + config key. + + Lowercase, hyphens permitted to preserve existing user-visible names. + Examples: ``browserbase``, ``browser-use``, ``firecrawl``. + """ + + @property + def display_name(self) -> str: + """Human-readable label shown in ``hermes tools``. Defaults to ``name``.""" + return self.name + + @abc.abstractmethod + def is_available(self) -> bool: + """Return True when this provider can service calls. + + Typically a cheap check (env var present, managed-gateway token + readable, optional Python dep importable). Must NOT make network + calls — this runs at tool-registration time and on every + ``hermes tools`` paint. + + Mirrors the legacy ``CloudBrowserProvider.is_configured()`` method; + renamed for parity with :class:`agent.web_search_provider.WebSearchProvider`. + """ + + @abc.abstractmethod + def create_session(self, task_id: str) -> Dict[str, object]: + """Create a cloud browser session and return session metadata. + + Must return a dict with at least:: + + { + "session_name": str, # unique name for agent-browser --session + "bb_session_id": str, # provider session ID (for close/cleanup) + "cdp_url": str, # CDP websocket URL + "features": dict, # feature flags that were enabled + } + + ``bb_session_id`` is a legacy key name kept for backward compat with + the rest of :mod:`tools.browser_tool` — it holds the provider's + session ID regardless of which provider is in use. + + May raise ``ValueError`` (missing credentials) or ``RuntimeError`` + (network / API failure); the dispatcher surfaces these to the user. + """ + + @abc.abstractmethod + def close_session(self, session_id: str) -> bool: + """Release / terminate a cloud session by its provider session ID. + + Returns True on success, False on failure. Should not raise — log and + return False on any exception so the dispatcher's cleanup loop keeps + moving across sessions. + """ + + @abc.abstractmethod + def emergency_cleanup(self, session_id: str) -> None: + """Best-effort session teardown during process exit. + + Called from atexit / signal handlers. Must tolerate missing + credentials, network errors, etc. — log and move on. Must not raise. + """ + + def get_setup_schema(self) -> Dict[str, Any]: + """Return provider metadata for the ``hermes tools`` picker. + + Used by :mod:`hermes_cli.tools_config` to inject this provider as a + row in the Browser Automation picker. Shape mirrors the existing + hardcoded entries in ``TOOL_CATEGORIES["browser"]``:: + + { + "name": "Browserbase", + "badge": "paid", + "tag": "Cloud browser with stealth and proxies", + "env_vars": [ + {"key": "BROWSERBASE_API_KEY", + "prompt": "Browserbase API key", + "url": "https://browserbase.com"}, + ], + "post_setup": "agent_browser", + } + + Default: minimal entry derived from :attr:`display_name`. Override to + expose API key prompts, badges, managed-Nous gating, and the + ``post_setup`` install hook. + """ + return { + "name": self.display_name, + "badge": "", + "tag": "", + "env_vars": [], + } diff --git a/agent/browser_registry.py b/agent/browser_registry.py new file mode 100644 index 00000000000..249c4863927 --- /dev/null +++ b/agent/browser_registry.py @@ -0,0 +1,221 @@ +""" +Browser Provider Registry +========================= + +Central map of registered cloud browser providers. Populated by plugins at +import-time via :meth:`PluginContext.register_browser_provider`; consumed by +:func:`tools.browser_tool._get_cloud_provider` to route each cloud-mode +``browser_*`` tool call to the active backend. + +Active selection +---------------- +The active provider is chosen by configuration with this precedence: + +1. ``browser.cloud_provider`` in ``config.yaml`` (explicit override). +2. If exactly one registered provider is available, use it. +3. Legacy preference order — ``browser-use`` → ``browserbase`` — filtered by + availability. Matches the historic auto-detect order in + :func:`tools.browser_tool._get_cloud_provider` (Browser Use checked first + because it covers both the managed Nous gateway and direct API key path; + Browserbase as the older direct-credentials fallback). ``firecrawl`` is + intentionally NOT in the legacy walk — users only get Firecrawl as a + cloud browser when they explicitly set ``browser.cloud_provider: + firecrawl``, matching pre-migration behaviour where Firecrawl was never + auto-selected. +4. Otherwise ``None`` — the dispatcher falls back to local browser mode. + +The explicit-config branch (rule 1) intentionally ignores ``is_available()`` +so the dispatcher surfaces a typed "X_API_KEY is not set" error to the user +instead of silently switching backends. Matches the legacy +:func:`tools.browser_tool._get_cloud_provider` behaviour for configured names. + +Note: there is no "capability" split here (unlike the web subsystem, which +has search/extract/crawl). Every browser provider implements the full +:class:`agent.browser_provider.BrowserProvider` lifecycle; the registry's +job is purely selection, not capability routing. +""" + +from __future__ import annotations + +import logging +import threading +from typing import Dict, List, Optional + +from agent.browser_provider import BrowserProvider + +logger = logging.getLogger(__name__) + + +_providers: Dict[str, BrowserProvider] = {} +_lock = threading.Lock() + + +def register_provider(provider: BrowserProvider) -> None: + """Register a cloud browser provider. + + Re-registration (same ``name``) overwrites the previous entry and logs + a debug message — makes hot-reload scenarios (tests, dev loops) behave + predictably. + """ + if not isinstance(provider, BrowserProvider): + raise TypeError( + f"register_provider() expects a BrowserProvider instance, " + f"got {type(provider).__name__}" + ) + name = provider.name + if not isinstance(name, str) or not name.strip(): + raise ValueError("Browser provider .name must be a non-empty string") + with _lock: + existing = _providers.get(name) + _providers[name] = provider + if existing is not None: + logger.debug( + "Browser provider '%s' re-registered (was %r)", + name, type(existing).__name__, + ) + else: + logger.debug( + "Registered browser provider '%s' (%s)", + name, type(provider).__name__, + ) + + +def list_providers() -> List[BrowserProvider]: + """Return all registered providers, sorted by name.""" + with _lock: + items = list(_providers.values()) + return sorted(items, key=lambda p: p.name) + + +def get_provider(name: str) -> Optional[BrowserProvider]: + """Return the provider registered under *name*, or None.""" + if not isinstance(name, str): + return None + with _lock: + return _providers.get(name.strip()) + + +# --------------------------------------------------------------------------- +# Active-provider resolution +# --------------------------------------------------------------------------- + + +# Legacy preference order — preserves behaviour for users who set no +# ``browser.cloud_provider`` config key. Matches the historic auto-detect +# order in :func:`tools.browser_tool._get_cloud_provider` (Browser Use first +# because it covers both managed Nous gateway and direct API key; Browserbase +# second as the older direct-credentials fallback). Filtered by +# ``is_available()`` at walk time so we don't surface a provider the user +# has no credentials for. +# +# Note: ``firecrawl`` is intentionally absent. Pre-migration, the auto-detect +# branch only considered Browser Use → Browserbase; Firecrawl was reachable +# only via an explicit ``browser.cloud_provider: firecrawl`` config key. +# Preserving that gate prevents users with a ``FIRECRAWL_API_KEY`` set for +# web-extract from accidentally getting routed to a (paid) cloud browser. +_LEGACY_PREFERENCE = ( + "browser-use", + "browserbase", +) + + +def _resolve(configured: Optional[str]) -> Optional[BrowserProvider]: + """Resolve the active browser provider. + + Resolution rules (in order): + + 1. **Explicit "local".** Returns None — the dispatcher disables cloud + mode entirely. Mirrors legacy short-circuit in + :func:`tools.browser_tool._get_cloud_provider`. + 2. **Explicit config wins, ignoring availability.** If ``configured`` + names a registered provider, return it even if its + :meth:`is_available` returns False — the dispatcher will surface a + precise "X_API_KEY is not set" error instead of silently routing + somewhere else. + 3. **Single-provider shortcut.** When only one registered provider + reports ``is_available() == True``, return it. + 4. **Legacy preference walk, filtered by availability.** Walk + :data:`_LEGACY_PREFERENCE` (``browser-use`` → ``browserbase``) looking + for a provider whose ``is_available()`` is True. + + Returns None when no provider is configured AND no available provider + matches the legacy preference; the dispatcher then falls back to local + browser mode. + """ + with _lock: + snapshot = dict(_providers) + + def _is_available_safe(p: BrowserProvider) -> bool: + """Wrap ``is_available()`` so a buggy provider doesn't kill resolution.""" + try: + return bool(p.is_available()) + except Exception as exc: # noqa: BLE001 + logger.debug("provider %s.is_available() raised %s", p.name, exc) + return False + + # 1. Explicit "local" short-circuit. + if configured == "local": + return None + + # 2. Explicit config wins — return regardless of is_available() so the + # user gets a precise downstream error message rather than a silent + # backend switch. Matches _get_cloud_provider() in browser_tool.py. + if configured: + provider = snapshot.get(configured) + if provider is not None: + return provider + logger.debug( + "browser cloud_provider '%s' configured but not registered; " + "falling back to auto-detect", + configured, + ) + + # 3. + 4. Auto-detect path — filter by availability so we don't surface + # a provider the user has no credentials for. + eligible = [p for p in snapshot.values() if _is_available_safe(p)] + if len(eligible) == 1: + return eligible[0] + + for legacy in _LEGACY_PREFERENCE: + provider = snapshot.get(legacy) + if provider is not None and _is_available_safe(provider): + return provider + + return None + + +def get_active_browser_provider() -> Optional[BrowserProvider]: + """Resolve the currently-active cloud browser provider. + + Reads ``browser.cloud_provider`` from config.yaml; falls back per the + module docstring. Returns None for local mode or when no provider is + available. + """ + try: + from hermes_cli.config import read_raw_config + + cfg = read_raw_config() + browser_cfg = cfg.get("browser", {}) + except Exception as exc: + logger.debug("Could not read browser config: %s", exc) + browser_cfg = {} + + configured: Optional[str] = None + if isinstance(browser_cfg, dict) and "cloud_provider" in browser_cfg: + try: + from tools.tool_backend_helpers import normalize_browser_cloud_provider + + configured = normalize_browser_cloud_provider( + browser_cfg.get("cloud_provider") + ) + except Exception as exc: + logger.debug("normalize_browser_cloud_provider failed: %s", exc) + configured = None + + return _resolve(configured) + + +def _reset_for_tests() -> None: + """Clear the registry. **Test-only.**""" + with _lock: + _providers.clear() diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index d0bbee6ce63..6150bf016d1 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -608,6 +608,38 @@ class PluginContext: self.manifest.name, provider.name, ) + # -- browser provider registration --------------------------------------- + + def register_browser_provider(self, provider) -> None: + """Register a cloud browser backend. + + ``provider`` must be an instance of + :class:`agent.browser_provider.BrowserProvider`. The + ``provider.name`` attribute is what ``browser.cloud_provider`` in + ``config.yaml`` matches against when routing cloud-mode + ``browser_*`` tool calls. + + Mirrors :meth:`register_web_search_provider` exactly — same + registration shape, same gating, same logging. The browser + subsystem's dispatcher (:func:`tools.browser_tool._get_cloud_provider`) + consults the registry built up by these calls. + """ + from agent.browser_provider import BrowserProvider + from agent.browser_registry import register_provider as _register_browser_provider + + if not isinstance(provider, BrowserProvider): + logger.warning( + "Plugin '%s' tried to register a browser provider that does " + "not inherit from BrowserProvider. Ignoring.", + self.manifest.name, + ) + return + _register_browser_provider(provider) + logger.info( + "Plugin '%s' registered browser provider: %s", + self.manifest.name, provider.name, + ) + # -- platform adapter registration --------------------------------------- def register_platform( From b8138ac4054935e117e2a5b2042fe9f01bb06e09 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 11:19:12 +0530 Subject: [PATCH 078/142] =?UTF-8?q?feat(browser):=20browserbase=20plugin?= =?UTF-8?q?=20(spike=20=E2=80=94=20first=20migration)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrates tools/browser_providers/browserbase.py → plugins/browser/browserbase/. Direct credentials only (BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID); same session-creation, 402-handling, and feature-flag logic as the legacy implementation. Renames is_configured() → is_available() to match the new BrowserProvider ABC. The legacy module tools/browser_providers/browserbase.py is NOT yet deleted and tools/browser_tool.py still references the in-tree class. The dispatcher cutover happens in a later commit so the plugin migration and the dispatcher switch land as separate reviewable units. Verified via plugin-discovery E2E: - browserbase registers as 'browserbase' - is_available() correctly tracks BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID - _resolve('browserbase') returns the provider even when unavailable (so dispatcher surfaces a typed credentials error) - _resolve(None) returns the provider when it's the single eligible one --- plugins/browser/browserbase/__init__.py | 15 ++ plugins/browser/browserbase/plugin.yaml | 7 + plugins/browser/browserbase/provider.py | 292 ++++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 plugins/browser/browserbase/__init__.py create mode 100644 plugins/browser/browserbase/plugin.yaml create mode 100644 plugins/browser/browserbase/provider.py diff --git a/plugins/browser/browserbase/__init__.py b/plugins/browser/browserbase/__init__.py new file mode 100644 index 00000000000..1e0269e2733 --- /dev/null +++ b/plugins/browser/browserbase/__init__.py @@ -0,0 +1,15 @@ +"""Browserbase cloud browser plugin — bundled, auto-loaded. + +Mirrors the ``plugins/web//`` and ``plugins/image_gen/openai/`` +layout: ``provider.py`` holds the provider class; ``__init__.py::register`` +instantiates and registers it via the plugin context. +""" + +from __future__ import annotations + +from plugins.browser.browserbase.provider import BrowserbaseBrowserProvider + + +def register(ctx) -> None: + """Register the Browserbase provider with the plugin context.""" + ctx.register_browser_provider(BrowserbaseBrowserProvider()) diff --git a/plugins/browser/browserbase/plugin.yaml b/plugins/browser/browserbase/plugin.yaml new file mode 100644 index 00000000000..5d976328a23 --- /dev/null +++ b/plugins/browser/browserbase/plugin.yaml @@ -0,0 +1,7 @@ +name: browser-browserbase +version: 1.0.0 +description: "Browserbase (https://browserbase.com) cloud browser backend. Requires BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID. Supports stealth, proxies, and keep-alive sessions; auto-falls-back when paid features are unavailable." +author: NousResearch +kind: backend +provides_browser_providers: + - browserbase diff --git a/plugins/browser/browserbase/provider.py b/plugins/browser/browserbase/provider.py new file mode 100644 index 00000000000..0d1a646c8a6 --- /dev/null +++ b/plugins/browser/browserbase/provider.py @@ -0,0 +1,292 @@ +"""Browserbase cloud browser provider — plugin form. + +Subclasses :class:`agent.browser_provider.BrowserProvider` (the plugin-facing +ABC introduced in PR #25214). The legacy in-tree module +``tools.browser_providers.browserbase`` was removed in the same PR; this file +is now the canonical implementation. + +Browserbase requires direct ``BROWSERBASE_API_KEY`` and ``BROWSERBASE_PROJECT_ID`` +credentials. Managed Nous gateway support has been removed — the Nous +subscription now routes through Browser Use instead (see +``plugins/browser/browser_use/``). + +Config keys this provider responds to:: + + browser: + cloud_provider: "browserbase" + +Auth env vars:: + + BROWSERBASE_API_KEY=... # https://browserbase.com + BROWSERBASE_PROJECT_ID=... + +Optional feature knobs:: + + BROWSERBASE_BASE_URL=... # default https://api.browserbase.com + BROWSERBASE_PROXIES=true # default true + BROWSERBASE_ADVANCED_STEALTH=false + BROWSERBASE_KEEP_ALIVE=true # default true + BROWSERBASE_SESSION_TIMEOUT=... (ms, integer) +""" + +from __future__ import annotations + +import logging +import os +import uuid +from typing import Any, Dict, Optional + +import requests + +from agent.browser_provider import BrowserProvider + +logger = logging.getLogger(__name__) + + +class BrowserbaseBrowserProvider(BrowserProvider): + """Browserbase (https://browserbase.com) cloud browser backend. + + Direct credentials only — managed-Nous-gateway support lives on the + Browser Use provider now. + """ + + @property + def name(self) -> str: + return "browserbase" + + @property + def display_name(self) -> str: + return "Browserbase" + + def is_available(self) -> bool: + return self._get_config_or_none() is not None + + # ------------------------------------------------------------------ + # Config resolution + # ------------------------------------------------------------------ + + def _get_config_or_none(self) -> Optional[Dict[str, Any]]: + api_key = os.environ.get("BROWSERBASE_API_KEY") + project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + if api_key and project_id: + return { + "api_key": api_key, + "project_id": project_id, + "base_url": os.environ.get( + "BROWSERBASE_BASE_URL", "https://api.browserbase.com" + ).rstrip("/"), + } + return None + + def _get_config(self) -> Dict[str, Any]: + config = self._get_config_or_none() + if config is None: + raise ValueError( + "Browserbase requires BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID " + "environment variables." + ) + return config + + # ------------------------------------------------------------------ + # Session lifecycle + # ------------------------------------------------------------------ + + def create_session(self, task_id: str) -> Dict[str, object]: + config = self._get_config() + + # Optional env-var knobs + enable_proxies = os.environ.get("BROWSERBASE_PROXIES", "true").lower() != "false" + enable_advanced_stealth = ( + os.environ.get("BROWSERBASE_ADVANCED_STEALTH", "false").lower() == "true" + ) + enable_keep_alive = ( + os.environ.get("BROWSERBASE_KEEP_ALIVE", "true").lower() != "false" + ) + custom_timeout_ms = os.environ.get("BROWSERBASE_SESSION_TIMEOUT") + + features_enabled = { + "basic_stealth": True, + "proxies": False, + "advanced_stealth": False, + "keep_alive": False, + "custom_timeout": False, + } + + session_config: Dict[str, object] = {"projectId": config["project_id"]} + + if enable_keep_alive: + session_config["keepAlive"] = True + + if custom_timeout_ms: + try: + timeout_val = int(custom_timeout_ms) + if timeout_val > 0: + session_config["timeout"] = timeout_val + except ValueError: + logger.warning( + "Invalid BROWSERBASE_SESSION_TIMEOUT value: %s", custom_timeout_ms + ) + + if enable_proxies: + session_config["proxies"] = True + + if enable_advanced_stealth: + session_config["browserSettings"] = {"advancedStealth": True} + + # --- Create session via API --- + headers = { + "Content-Type": "application/json", + "X-BB-API-Key": config["api_key"], + } + + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) + + proxies_fallback = False + keepalive_fallback = False + + # Handle 402 — paid features unavailable + if response.status_code == 402: + if enable_keep_alive: + keepalive_fallback = True + logger.warning( + "keepAlive may require paid plan (402), retrying without it. " + "Sessions may timeout during long operations." + ) + session_config.pop("keepAlive", None) + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) + + if response.status_code == 402 and enable_proxies: + proxies_fallback = True + logger.warning( + "Proxies unavailable (402), retrying without proxies. " + "Bot detection may be less effective." + ) + session_config.pop("proxies", None) + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) + + if not response.ok: + raise RuntimeError( + f"Failed to create Browserbase session: " + f"{response.status_code} {response.text}" + ) + + session_data = response.json() + session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" + + if enable_proxies and not proxies_fallback: + features_enabled["proxies"] = True + if enable_advanced_stealth: + features_enabled["advanced_stealth"] = True + if enable_keep_alive and not keepalive_fallback: + features_enabled["keep_alive"] = True + if custom_timeout_ms and "timeout" in session_config: + features_enabled["custom_timeout"] = True + + feature_str = ", ".join(k for k, v in features_enabled.items() if v) + logger.info( + "Created Browserbase session %s with features: %s", session_name, feature_str + ) + + return { + "session_name": session_name, + "bb_session_id": session_data["id"], + "cdp_url": session_data["connectUrl"], + "features": features_enabled, + } + + def close_session(self, session_id: str) -> bool: + try: + config = self._get_config() + except ValueError: + logger.warning( + "Cannot close Browserbase session %s — missing credentials", session_id + ) + return False + + try: + response = requests.post( + f"{config['base_url']}/v1/sessions/{session_id}", + headers={ + "X-BB-API-Key": config["api_key"], + "Content-Type": "application/json", + }, + json={ + "projectId": config["project_id"], + "status": "REQUEST_RELEASE", + }, + timeout=10, + ) + if response.status_code in {200, 201, 204}: + logger.debug("Successfully closed Browserbase session %s", session_id) + return True + else: + logger.warning( + "Failed to close session %s: HTTP %s - %s", + session_id, + response.status_code, + response.text[:200], + ) + return False + except Exception as e: + logger.error("Exception closing Browserbase session %s: %s", session_id, e) + return False + + def emergency_cleanup(self, session_id: str) -> None: + config = self._get_config_or_none() + if config is None: + logger.warning( + "Cannot emergency-cleanup Browserbase session %s — missing credentials", + session_id, + ) + return + try: + requests.post( + f"{config['base_url']}/v1/sessions/{session_id}", + headers={ + "X-BB-API-Key": config["api_key"], + "Content-Type": "application/json", + }, + json={ + "projectId": config["project_id"], + "status": "REQUEST_RELEASE", + }, + timeout=5, + ) + except Exception as e: + logger.debug( + "Emergency cleanup failed for Browserbase session %s: %s", session_id, e + ) + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Browserbase", + "badge": "paid", + "tag": "Cloud browser with stealth and proxies", + "env_vars": [ + { + "key": "BROWSERBASE_API_KEY", + "prompt": "Browserbase API key", + "url": "https://browserbase.com", + }, + { + "key": "BROWSERBASE_PROJECT_ID", + "prompt": "Browserbase project ID", + }, + ], + "post_setup": "agent_browser", + } From a15cdfb0509db31b094aa0ff034b2432c43bc6e1 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:11:48 +0530 Subject: [PATCH 079/142] feat(browser): browser-use + firecrawl plugins; drop single-eligible shortcut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrates the remaining two cloud browser providers to plugins: plugins/browser/browser_use/ — dual auth (direct BROWSER_USE_API_KEY or managed Nous gateway), idempotency- key handling for retried managed-mode creates, x-external-call-id capture. plugins/browser/firecrawl/ — direct FIRECRAWL_API_KEY only; distinct from plugins/web/firecrawl/ (same key, different endpoint). Also drops the 'single-eligible shortcut' rule from agent.browser_registry._resolve(). Was a copy-paste from web_search_registry that would have introduced a real behavior change: a user with only FIRECRAWL_API_KEY set (for web-extract) would silently get routed to a paid Firecrawl cloud browser on a fresh install — not matching origin/main, which only auto-detected between Browser Use and Browserbase. Third-party browser plugins are subject to the same gate: they require explicit `browser.cloud_provider` to take effect. Verified end-to-end via plugin discovery: - 3 plugins register (browser-use, browserbase, firecrawl) - _resolve(None) with no creds: None (local mode) - _resolve(None) with only FIRECRAWL_API_KEY: None (matches main) - _resolve('firecrawl'): firecrawl (explicit wins) - _resolve(None) with BU+firecrawl: browser-use (legacy walk first hit) - _resolve(None) with all three: browser-use (legacy walk order) --- agent/browser_registry.py | 31 ++- plugins/browser/browser_use/__init__.py | 14 ++ plugins/browser/browser_use/plugin.yaml | 7 + plugins/browser/browser_use/provider.py | 305 ++++++++++++++++++++++++ plugins/browser/firecrawl/__init__.py | 16 ++ plugins/browser/firecrawl/plugin.yaml | 7 + plugins/browser/firecrawl/provider.py | 162 +++++++++++++ 7 files changed, 530 insertions(+), 12 deletions(-) create mode 100644 plugins/browser/browser_use/__init__.py create mode 100644 plugins/browser/browser_use/plugin.yaml create mode 100644 plugins/browser/browser_use/provider.py create mode 100644 plugins/browser/firecrawl/__init__.py create mode 100644 plugins/browser/firecrawl/plugin.yaml create mode 100644 plugins/browser/firecrawl/provider.py diff --git a/agent/browser_registry.py b/agent/browser_registry.py index 249c4863927..7b5b8b99b5f 100644 --- a/agent/browser_registry.py +++ b/agent/browser_registry.py @@ -12,8 +12,7 @@ Active selection The active provider is chosen by configuration with this precedence: 1. ``browser.cloud_provider`` in ``config.yaml`` (explicit override). -2. If exactly one registered provider is available, use it. -3. Legacy preference order — ``browser-use`` → ``browserbase`` — filtered by +2. Legacy preference order — ``browser-use`` → ``browserbase`` — filtered by availability. Matches the historic auto-detect order in :func:`tools.browser_tool._get_cloud_provider` (Browser Use checked first because it covers both the managed Nous gateway and direct API key path; @@ -22,7 +21,7 @@ The active provider is chosen by configuration with this precedence: cloud browser when they explicitly set ``browser.cloud_provider: firecrawl``, matching pre-migration behaviour where Firecrawl was never auto-selected. -4. Otherwise ``None`` — the dispatcher falls back to local browser mode. +3. Otherwise ``None`` — the dispatcher falls back to local browser mode. The explicit-config branch (rule 1) intentionally ignores ``is_available()`` so the dispatcher surfaces a typed "X_API_KEY is not set" error to the user @@ -132,12 +131,22 @@ def _resolve(configured: Optional[str]) -> Optional[BrowserProvider]: :meth:`is_available` returns False — the dispatcher will surface a precise "X_API_KEY is not set" error instead of silently routing somewhere else. - 3. **Single-provider shortcut.** When only one registered provider - reports ``is_available() == True``, return it. - 4. **Legacy preference walk, filtered by availability.** Walk + 3. **Legacy preference walk, filtered by availability.** Walk :data:`_LEGACY_PREFERENCE` (``browser-use`` → ``browserbase``) looking for a provider whose ``is_available()`` is True. + There is intentionally NO "single-eligible shortcut" rule here (unlike + :func:`agent.web_search_registry._resolve`). Pre-migration, the + auto-detect branch in ``tools.browser_tool._get_cloud_provider`` only + considered Browser Use and Browserbase; Firecrawl was reachable only + via an explicit ``browser.cloud_provider: firecrawl`` config key. + Preserving that gate matters because Firecrawl shares its API key with + the *web* extract plugin (``plugins/web/firecrawl/``), so users who set + ``FIRECRAWL_API_KEY`` for web extract must NOT get silently routed to a + paid cloud browser on a fresh install. Third-party browser-provider + plugins added under ``~/.hermes/plugins/browser//`` are subject + to the same gate — they must be explicitly configured to take effect. + Returns None when no provider is configured AND no available provider matches the legacy preference; the dispatcher then falls back to local browser mode. @@ -170,12 +179,10 @@ def _resolve(configured: Optional[str]) -> Optional[BrowserProvider]: configured, ) - # 3. + 4. Auto-detect path — filter by availability so we don't surface - # a provider the user has no credentials for. - eligible = [p for p in snapshot.values() if _is_available_safe(p)] - if len(eligible) == 1: - return eligible[0] - + # 3. Legacy preference walk — only providers in _LEGACY_PREFERENCE are + # auto-eligible. Filtered by availability so we don't surface a + # provider the user has no credentials for. See docstring for why + # we do NOT fall back to "any single-eligible registered provider". for legacy in _LEGACY_PREFERENCE: provider = snapshot.get(legacy) if provider is not None and _is_available_safe(provider): diff --git a/plugins/browser/browser_use/__init__.py b/plugins/browser/browser_use/__init__.py new file mode 100644 index 00000000000..b07db13913a --- /dev/null +++ b/plugins/browser/browser_use/__init__.py @@ -0,0 +1,14 @@ +"""Browser Use cloud browser plugin — bundled, auto-loaded. + +Mirrors the ``plugins/web//`` layout: ``provider.py`` holds the +provider class; ``__init__.py::register`` instantiates and registers it. +""" + +from __future__ import annotations + +from plugins.browser.browser_use.provider import BrowserUseBrowserProvider + + +def register(ctx) -> None: + """Register the Browser Use provider with the plugin context.""" + ctx.register_browser_provider(BrowserUseBrowserProvider()) diff --git a/plugins/browser/browser_use/plugin.yaml b/plugins/browser/browser_use/plugin.yaml new file mode 100644 index 00000000000..ff926a50ea7 --- /dev/null +++ b/plugins/browser/browser_use/plugin.yaml @@ -0,0 +1,7 @@ +name: browser-browser-use +version: 1.0.0 +description: "Browser Use (https://browser-use.com) cloud browser backend. Supports both direct BROWSER_USE_API_KEY and the managed Nous tool gateway. Also powers the 'Nous Subscription' UX flow that bills usage to a Nous subscription." +author: NousResearch +kind: backend +provides_browser_providers: + - browser-use diff --git a/plugins/browser/browser_use/provider.py b/plugins/browser/browser_use/provider.py new file mode 100644 index 00000000000..82bd2420ca1 --- /dev/null +++ b/plugins/browser/browser_use/provider.py @@ -0,0 +1,305 @@ +"""Browser Use cloud browser provider — plugin form. + +Subclasses :class:`agent.browser_provider.BrowserProvider` (the plugin-facing +ABC introduced in PR #25214). The legacy in-tree module +``tools.browser_providers.browser_use`` was removed in the same PR; this file +is now the canonical implementation. + +Browser Use is the only browser backend with dual auth: a direct +``BROWSER_USE_API_KEY`` for self-billed users, or the managed Nous tool +gateway (which Hermes uses to bill Browser Use sessions to a Nous +subscription). The dispatch order — direct API key first, managed gateway +second — preserves the pre-migration behaviour in +``tools.browser_providers.browser_use.BrowserUseProvider._get_config_or_none``. + +Config keys this provider responds to:: + + browser: + cloud_provider: "browser-use" # explicit selection + tool_gateway: + browser: "gateway" # optional: prefer managed gateway + # even when BROWSER_USE_API_KEY is set + +Auth env vars (one of):: + + BROWSER_USE_API_KEY=... # https://browser-use.com + # OR a managed Nous gateway entry (configured via 'hermes setup') +""" + +from __future__ import annotations + +import logging +import os +import threading +import uuid +from typing import Any, Dict, Optional + +import requests + +from agent.browser_provider import BrowserProvider + +logger = logging.getLogger(__name__) + +# Idempotency tracking for managed-mode session creation. The managed Nous +# gateway returns 409 "already in progress" on retried POSTs; we forward the +# original idempotency key so the gateway can deduplicate. Cleared on +# success or terminal failure. +_pending_create_keys: Dict[str, str] = {} +_pending_create_keys_lock = threading.Lock() + +_BASE_URL = "https://api.browser-use.com/api/v3" +_DEFAULT_MANAGED_TIMEOUT_MINUTES = 5 +_DEFAULT_MANAGED_PROXY_COUNTRY_CODE = "us" + + +def _get_or_create_pending_create_key(task_id: str) -> str: + with _pending_create_keys_lock: + existing = _pending_create_keys.get(task_id) + if existing: + return existing + + created = f"browser-use-session-create:{uuid.uuid4().hex}" + _pending_create_keys[task_id] = created + return created + + +def _clear_pending_create_key(task_id: str) -> None: + with _pending_create_keys_lock: + _pending_create_keys.pop(task_id, None) + + +def _should_preserve_pending_create_key(response: requests.Response) -> bool: + """Decide whether to keep the idempotency key after a failed create. + + Preserve the key when the failure looks retryable (5xx) OR when the + gateway reports the original request is still in flight (409 "already + in progress") — in either case, retrying with the same key lets the + gateway deduplicate. + + Drop the key on any other 4xx (auth failure, bad request, etc.) — those + won't succeed by being retried. + """ + if response.status_code >= 500: + return True + + if response.status_code != 409: + return False + + try: + payload = response.json() + except Exception: + return False + + if not isinstance(payload, dict): + return False + + error = payload.get("error") + if not isinstance(error, dict): + return False + + message = str(error.get("message") or "").lower() + return "already in progress" in message + + +class BrowserUseBrowserProvider(BrowserProvider): + """Browser Use (https://browser-use.com) cloud browser backend. + + Dual auth: prefers a direct BROWSER_USE_API_KEY when set, falling back + to the managed Nous tool gateway when ``tool_gateway.browser`` config + routes through it. Setting ``tool_gateway.browser: gateway`` flips the + order so managed billing wins even when BROWSER_USE_API_KEY is present. + """ + + @property + def name(self) -> str: + return "browser-use" + + @property + def display_name(self) -> str: + return "Browser Use" + + def is_available(self) -> bool: + return self._get_config_or_none() is not None + + # ------------------------------------------------------------------ + # Config resolution (direct API key OR managed Nous gateway) + # ------------------------------------------------------------------ + + def _get_config_or_none(self) -> Optional[Dict[str, Any]]: + # Import here to avoid a hard dependency at module-import time — + # managed_tool_gateway pulls in the Nous auth stack which can be + # heavy and is not needed for direct-API-key users. + from tools.managed_tool_gateway import resolve_managed_tool_gateway + from tools.tool_backend_helpers import managed_nous_tools_enabled, prefers_gateway + + # 1. Direct API key path (unless user explicitly prefers gateway). + api_key = os.environ.get("BROWSER_USE_API_KEY") + if api_key and not prefers_gateway("browser"): + return { + "api_key": api_key, + "base_url": _BASE_URL, + "managed_mode": False, + } + + # 2. Managed Nous gateway path. + managed = resolve_managed_tool_gateway("browser-use") + if managed is None: + return None + + # Hold reference to managed_nous_tools_enabled so static analysis + # doesn't flag the import as unused — the helper is consulted by + # _get_config() below to compose a more accurate error message. + _ = managed_nous_tools_enabled + + return { + "api_key": managed.nous_user_token, + "base_url": managed.gateway_origin.rstrip("/"), + "managed_mode": True, + } + + def _get_config(self) -> Dict[str, Any]: + from tools.tool_backend_helpers import managed_nous_tools_enabled + + config = self._get_config_or_none() + if config is None: + message = ( + "Browser Use requires a direct BROWSER_USE_API_KEY credential." + ) + if managed_nous_tools_enabled(): + message = ( + "Browser Use requires either a direct BROWSER_USE_API_KEY " + "credential or a managed Browser Use gateway configuration." + ) + raise ValueError(message) + return config + + # ------------------------------------------------------------------ + # Session lifecycle + # ------------------------------------------------------------------ + + def _headers(self, config: Dict[str, Any]) -> Dict[str, str]: + return { + "Content-Type": "application/json", + "X-Browser-Use-API-Key": config["api_key"], + } + + def create_session(self, task_id: str) -> Dict[str, object]: + config = self._get_config() + managed_mode = bool(config.get("managed_mode")) + + headers = self._headers(config) + if managed_mode: + headers["X-Idempotency-Key"] = _get_or_create_pending_create_key(task_id) + + # Keep gateway-backed sessions short so billing authorization does not + # default to a long Browser-Use timeout when Hermes only needs a task- + # scoped ephemeral browser. + payload = ( + { + "timeout": _DEFAULT_MANAGED_TIMEOUT_MINUTES, + "proxyCountryCode": _DEFAULT_MANAGED_PROXY_COUNTRY_CODE, + } + if managed_mode + else {} + ) + + response = requests.post( + f"{config['base_url']}/browsers", + headers=headers, + json=payload, + timeout=30, + ) + + if not response.ok: + if managed_mode and not _should_preserve_pending_create_key(response): + _clear_pending_create_key(task_id) + raise RuntimeError( + f"Failed to create Browser Use session: " + f"{response.status_code} {response.text}" + ) + + session_data = response.json() + if managed_mode: + _clear_pending_create_key(task_id) + session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" + external_call_id = ( + response.headers.get("x-external-call-id") if managed_mode else None + ) + + logger.info("Created Browser Use session %s", session_name) + + cdp_url = session_data.get("cdpUrl") or session_data.get("connectUrl") or "" + + return { + "session_name": session_name, + "bb_session_id": session_data["id"], + "cdp_url": cdp_url, + "features": {"browser_use": True}, + "external_call_id": external_call_id, + } + + def close_session(self, session_id: str) -> bool: + try: + config = self._get_config() + except ValueError: + logger.warning( + "Cannot close Browser Use session %s — missing credentials", session_id + ) + return False + + try: + response = requests.patch( + f"{config['base_url']}/browsers/{session_id}", + headers=self._headers(config), + json={"action": "stop"}, + timeout=10, + ) + if response.status_code in {200, 201, 204}: + logger.debug("Successfully closed Browser Use session %s", session_id) + return True + else: + logger.warning( + "Failed to close Browser Use session %s: HTTP %s - %s", + session_id, + response.status_code, + response.text[:200], + ) + return False + except Exception as e: + logger.error("Exception closing Browser Use session %s: %s", session_id, e) + return False + + def emergency_cleanup(self, session_id: str) -> None: + config = self._get_config_or_none() + if config is None: + logger.warning( + "Cannot emergency-cleanup Browser Use session %s — missing credentials", + session_id, + ) + return + try: + requests.patch( + f"{config['base_url']}/browsers/{session_id}", + headers=self._headers(config), + json={"action": "stop"}, + timeout=5, + ) + except Exception as e: + logger.debug( + "Emergency cleanup failed for Browser Use session %s: %s", session_id, e + ) + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Browser Use", + "badge": "paid", + "tag": "Cloud browser with remote execution", + "env_vars": [ + { + "key": "BROWSER_USE_API_KEY", + "prompt": "Browser Use API key", + "url": "https://browser-use.com", + }, + ], + "post_setup": "agent_browser", + } diff --git a/plugins/browser/firecrawl/__init__.py b/plugins/browser/firecrawl/__init__.py new file mode 100644 index 00000000000..b045b636302 --- /dev/null +++ b/plugins/browser/firecrawl/__init__.py @@ -0,0 +1,16 @@ +"""Firecrawl cloud browser plugin — bundled, auto-loaded. + +Distinct from ``plugins/web/firecrawl/`` (the web search/extract/crawl +plugin); both share the FIRECRAWL_API_KEY but speak to different endpoints +(``/v2/browser`` here vs ``/v2/search`` / ``/v2/scrape`` / ``/v2/crawl`` +over there). +""" + +from __future__ import annotations + +from plugins.browser.firecrawl.provider import FirecrawlBrowserProvider + + +def register(ctx) -> None: + """Register the Firecrawl cloud-browser provider with the plugin context.""" + ctx.register_browser_provider(FirecrawlBrowserProvider()) diff --git a/plugins/browser/firecrawl/plugin.yaml b/plugins/browser/firecrawl/plugin.yaml new file mode 100644 index 00000000000..22da6a7f4b5 --- /dev/null +++ b/plugins/browser/firecrawl/plugin.yaml @@ -0,0 +1,7 @@ +name: browser-firecrawl +version: 1.0.0 +description: "Firecrawl (https://firecrawl.dev) cloud browser backend. Requires FIRECRAWL_API_KEY. Distinct from the firecrawl WEB search/extract plugin — the two share an API key but operate on different endpoints." +author: NousResearch +kind: backend +provides_browser_providers: + - firecrawl diff --git a/plugins/browser/firecrawl/provider.py b/plugins/browser/firecrawl/provider.py new file mode 100644 index 00000000000..a3f74d32113 --- /dev/null +++ b/plugins/browser/firecrawl/provider.py @@ -0,0 +1,162 @@ +"""Firecrawl cloud browser provider — plugin form. + +Subclasses :class:`agent.browser_provider.BrowserProvider` (the plugin-facing +ABC introduced in PR #25214). The legacy in-tree module +``tools.browser_providers.firecrawl`` was removed in the same PR; this file +is now the canonical implementation. + +This is the cloud-browser path — distinct from the firecrawl WEB plugin at +``plugins/web/firecrawl/`` which handles search/extract/crawl on +``/v2/search`` / ``/v2/scrape`` / ``/v2/crawl``. The two plugins share the +``FIRECRAWL_API_KEY`` env var but talk to different endpoints (this one +hits ``/v2/browser``). + +Config keys this provider responds to:: + + browser: + cloud_provider: "firecrawl" # explicit selection only — not in the + # legacy auto-detect walk + +Auth env vars:: + + FIRECRAWL_API_KEY=... # https://firecrawl.dev + FIRECRAWL_API_URL=... # optional override (default https://api.firecrawl.dev) + FIRECRAWL_BROWSER_TTL=... # optional, default 300 seconds +""" + +from __future__ import annotations + +import logging +import os +import uuid +from typing import Any, Dict + +import requests + +from agent.browser_provider import BrowserProvider + +logger = logging.getLogger(__name__) + +_BASE_URL = "https://api.firecrawl.dev" + + +class FirecrawlBrowserProvider(BrowserProvider): + """Firecrawl (https://firecrawl.dev) cloud browser backend. + + Cloud-browser path only — search/extract/crawl live in the separate + ``plugins/web/firecrawl/`` plugin. + """ + + @property + def name(self) -> str: + return "firecrawl" + + @property + def display_name(self) -> str: + return "Firecrawl" + + def is_available(self) -> bool: + return bool(os.environ.get("FIRECRAWL_API_KEY")) + + # ------------------------------------------------------------------ + # Session lifecycle + # ------------------------------------------------------------------ + + def _api_url(self) -> str: + return os.environ.get("FIRECRAWL_API_URL", _BASE_URL) + + def _headers(self) -> Dict[str, str]: + api_key = os.environ.get("FIRECRAWL_API_KEY") + if not api_key: + raise ValueError( + "FIRECRAWL_API_KEY environment variable is required. " + "Get your key at https://firecrawl.dev" + ) + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + } + + def create_session(self, task_id: str) -> Dict[str, object]: + ttl = int(os.environ.get("FIRECRAWL_BROWSER_TTL", "300")) + + body: Dict[str, object] = {"ttl": ttl} + + response = requests.post( + f"{self._api_url()}/v2/browser", + headers=self._headers(), + json=body, + timeout=30, + ) + + if not response.ok: + raise RuntimeError( + f"Failed to create Firecrawl browser session: " + f"{response.status_code} {response.text}" + ) + + data = response.json() + session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" + + logger.info("Created Firecrawl browser session %s", session_name) + + return { + "session_name": session_name, + "bb_session_id": data["id"], + "cdp_url": data["cdpUrl"], + "features": {"firecrawl": True}, + } + + def close_session(self, session_id: str) -> bool: + try: + response = requests.delete( + f"{self._api_url()}/v2/browser/{session_id}", + headers=self._headers(), + timeout=10, + ) + if response.status_code in {200, 201, 204}: + logger.debug("Successfully closed Firecrawl session %s", session_id) + return True + else: + logger.warning( + "Failed to close Firecrawl session %s: HTTP %s - %s", + session_id, + response.status_code, + response.text[:200], + ) + return False + except Exception as e: + logger.error("Exception closing Firecrawl session %s: %s", session_id, e) + return False + + def emergency_cleanup(self, session_id: str) -> None: + try: + requests.delete( + f"{self._api_url()}/v2/browser/{session_id}", + headers=self._headers(), + timeout=5, + ) + except ValueError: + logger.warning( + "Cannot emergency-cleanup Firecrawl session %s — missing credentials", + session_id, + ) + except Exception as e: + logger.debug( + "Emergency cleanup failed for Firecrawl session %s: %s", session_id, e + ) + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Firecrawl", + "badge": "paid", + "tag": "Cloud browser with remote execution", + "env_vars": [ + { + "key": "FIRECRAWL_API_KEY", + "prompt": "Firecrawl API key", + "url": "https://firecrawl.dev", + }, + ], + "post_setup": "agent_browser", + } From 40fde853fa6a84bf129a3f0958d15974887ccc78 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:15:52 +0530 Subject: [PATCH 080/142] refactor(browser): dispatch _get_cloud_provider through agent.browser_registry Switches tools.browser_tool's cloud-provider lookup from the hardcoded _PROVIDER_REGISTRY class-instantiation pattern to the agent.browser_registry singleton registry that plugins self-populate. Changes: - tools/browser_tool.py top imports: pull BrowserProvider from agent.browser_provider (re-exported as CloudBrowserProvider for legacy callers) and the three provider classes from plugins/browser//. Legacy class names (BrowserbaseProvider, BrowserUseProvider, FirecrawlProvider) remain on tools.browser_tool as re-export shims so existing test patches (monkeypatch.setattr(browser_tool, 'BrowserUseProvider', ...)) keep working. - _get_cloud_provider() now consults agent.browser_registry.get_provider() for explicit-config lookups. The auto-detect fallback still uses BrowserUseProvider() / BrowserbaseProvider() at the module level so the cache-policy test fixtures (which patch those names) keep driving the function. Test-time _PROVIDER_REGISTRY overrides are detected by class identity and routed through the legacy factory-call path. - agent/browser_provider.py: BrowserProvider grows is_configured() and provider_name() as thin backward-compat aliases for the legacy CloudBrowserProvider API. Subclasses MUST implement is_available() and name; the aliases delegate. This keeps ~6 caller sites in browser_tool.py working without churning them. - tests/tools/test_managed_browserbase_and_modal.py: _install_fake_tools_package grows stubs for agent.browser_provider / agent.browser_registry / plugins.browser..provider so the test's spec-loader path (sys.modules-reset + reload-tool-from-disk) can satisfy tools.browser_tool's top-level imports. Verified: all 23 existing tests in test_browser_cloud_*.py + test_managed_browserbase_and_modal.py still pass post-cutover. The legacy tools/browser_providers/ directory is NOT yet deleted; several tests still _load_tool_module() those files via spec_from_file_location. The deletion + test-path updates land in a later commit. --- agent/browser_provider.py | 20 ++++ .../test_managed_browserbase_and_modal.py | 43 +++++++++ tools/browser_tool.py | 94 +++++++++++++++++-- 3 files changed, 147 insertions(+), 10 deletions(-) diff --git a/agent/browser_provider.py b/agent/browser_provider.py index e351d75330e..338dfcd6b07 100644 --- a/agent/browser_provider.py +++ b/agent/browser_provider.py @@ -153,3 +153,23 @@ class BrowserProvider(abc.ABC): "tag": "", "env_vars": [], } + + # ------------------------------------------------------------------ + # Backward-compat shims for the legacy CloudBrowserProvider API + # ------------------------------------------------------------------ + # + # The pre-PR-#25214 ABC exposed ``is_configured()`` and ``provider_name()``; + # ``tools.browser_tool`` has ~6 callers that still use those names. Rather + # than churn every callsite (and break out-of-tree downstream code that + # subclassed CloudBrowserProvider), we expose the old names as thin + # delegations to the new API. Subclasses MUST implement :meth:`is_available` + # and :attr:`name`; they may override ``is_configured`` / ``provider_name`` + # for compatibility with the legacy ABC but it is not required. + + def is_configured(self) -> bool: # pragma: no cover - trivial delegation + """Backward-compat alias for :meth:`is_available`.""" + return self.is_available() + + def provider_name(self) -> str: # pragma: no cover - trivial delegation + """Backward-compat alias returning :attr:`display_name`.""" + return self.display_name diff --git a/tests/tools/test_managed_browserbase_and_modal.py b/tests/tools/test_managed_browserbase_and_modal.py index 6c963be6207..2e1bec03b01 100644 --- a/tests/tools/test_managed_browserbase_and_modal.py +++ b/tests/tools/test_managed_browserbase_and_modal.py @@ -76,6 +76,49 @@ def _install_fake_tools_package(): call_llm=lambda *args, **kwargs: "", ) + # Stubs for the browser-provider plugin layer introduced in PR #25214. + # The fake `agent` package has an empty __path__ so real submodules + # aren't reachable; we install just enough stand-ins to satisfy + # ``tools.browser_tool``'s top-level imports. The actual lifecycle + # tests instantiate the real plugin classes via _load_tool_module + # below, so the stubs only need to satisfy import + isinstance. + class _StubBrowserProvider: + """Minimal BrowserProvider stub for ``from agent.browser_provider import BrowserProvider``.""" + + sys.modules["agent.browser_provider"] = types.SimpleNamespace( + BrowserProvider=_StubBrowserProvider, + ) + sys.modules["agent.browser_registry"] = types.SimpleNamespace( + get_active_browser_provider=lambda: None, + get_provider=lambda name: None, + list_providers=lambda: [], + register_provider=lambda provider: None, + _resolve=lambda configured: None, + ) + + # Plugin module stubs — the real plugin classes are loaded from disk by + # the lifecycle tests below via _load_tool_module(). For the import + # phase, we just need the class names to exist on the right module path. + plugins_package = types.ModuleType("plugins") + plugins_package.__path__ = [] # type: ignore[attr-defined] + sys.modules["plugins"] = plugins_package + plugins_browser_package = types.ModuleType("plugins.browser") + plugins_browser_package.__path__ = [] # type: ignore[attr-defined] + sys.modules["plugins.browser"] = plugins_browser_package + + for _name, _classname in ( + ("browserbase", "BrowserbaseBrowserProvider"), + ("browser_use", "BrowserUseBrowserProvider"), + ("firecrawl", "FirecrawlBrowserProvider"), + ): + _vendor_pkg = types.ModuleType(f"plugins.browser.{_name}") + _vendor_pkg.__path__ = [] # type: ignore[attr-defined] + sys.modules[f"plugins.browser.{_name}"] = _vendor_pkg + _provider_stub_cls = type(_classname, (_StubBrowserProvider,), {}) + sys.modules[f"plugins.browser.{_name}.provider"] = types.SimpleNamespace( + **{_classname: _provider_stub_cls}, + ) + sys.modules["tools.managed_tool_gateway"] = _load_tool_module( "tools.managed_tool_gateway", "managed_tool_gateway.py", diff --git a/tools/browser_tool.py b/tools/browser_tool.py index b3eb24ee044..6fdd8949816 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -83,10 +83,25 @@ try: except Exception: _is_safe_url = lambda url: False # noqa: E731 — fail-closed: block all if safety module unavailable _is_always_blocked_url = lambda url: True # noqa: E731 — fail-closed on the floor too -from tools.browser_providers.base import CloudBrowserProvider -from tools.browser_providers.browserbase import BrowserbaseProvider -from tools.browser_providers.browser_use import BrowserUseProvider -from tools.browser_providers.firecrawl import FirecrawlProvider +# Browser-provider ABC + registry — PR #25214 moved the per-vendor providers +# (Browserbase / Browser Use / Firecrawl) out of ``tools/browser_providers/`` +# and into ``plugins/browser//``. The dispatcher consults the +# registry; the legacy class names are re-exported below as backward-compat +# shims for callers that import them from this module. +from agent.browser_provider import BrowserProvider as CloudBrowserProvider # noqa: F401 (legacy alias) +from agent.browser_registry import ( # noqa: F401 (test-patchable surface) + get_active_browser_provider as _registry_get_active_browser_provider, + get_provider as _registry_get_browser_provider, +) +from plugins.browser.browserbase.provider import ( # noqa: F401 (legacy import surface) + BrowserbaseBrowserProvider as BrowserbaseProvider, +) +from plugins.browser.browser_use.provider import ( # noqa: F401 + BrowserUseBrowserProvider as BrowserUseProvider, +) +from plugins.browser.firecrawl.provider import ( # noqa: F401 + FirecrawlBrowserProvider as FirecrawlProvider, +) from tools.tool_backend_helpers import normalize_browser_cloud_provider # Camofox local anti-detection browser backend (optional). @@ -391,6 +406,19 @@ def _stop_cdp_supervisor(task_id: str) -> None: # ============================================================================ # Cloud Provider Registry # ============================================================================ +# +# Per-vendor browser providers (Browserbase / Browser Use / Firecrawl) live as +# plugins under ``plugins/browser//`` and self-register through +# :mod:`agent.browser_registry` at plugin-discovery time. The legacy +# class-name registry below is preserved as a backward-compat shim so test +# fixtures that ``monkeypatch.setattr(browser_tool, "_PROVIDER_REGISTRY", ...)`` +# keep working — but ``_get_cloud_provider()`` now consults +# :mod:`agent.browser_registry` for the actual lookup. +# +# When the test patches ``_PROVIDER_REGISTRY``, we honour it (so the cache +# unit tests still drive the function); otherwise the registry-backed path +# wins. This keeps the test surface stable while letting third-party +# plugins drop in under ``~/.hermes/plugins/browser//``. _PROVIDER_REGISTRY: Dict[str, type] = { "browserbase": BrowserbaseProvider, @@ -411,13 +439,48 @@ _cached_browser_engine: Optional[str] = None _browser_engine_resolved = False +def _is_legacy_provider_registry_overridden() -> bool: + """Return True when a test has patched ``_PROVIDER_REGISTRY`` to a custom value. + + Detected by comparing identity with the module-level defaults dict + populated above. Tests that ``monkeypatch.setattr(browser_tool, + "_PROVIDER_REGISTRY", ...)`` swap in a new object; identity differs + even when the contents happen to match. Used by ``_get_cloud_provider`` + to honour test-time overrides (which expect a factory-callable shape) + instead of routing through the plugin registry. + """ + # The module-level _PROVIDER_REGISTRY is built once at import time. A test + # that swaps it via monkeypatch creates a new dict; we detect that via + # the registered class identities, not by ``is`` on the dict itself + # (the patch may install a dict whose values happen to be the same + # classes; treat that as "not overridden"). + try: + return ( + _PROVIDER_REGISTRY.get("browserbase") is not BrowserbaseProvider + or _PROVIDER_REGISTRY.get("browser-use") is not BrowserUseProvider + or _PROVIDER_REGISTRY.get("firecrawl") is not FirecrawlProvider + or set(_PROVIDER_REGISTRY.keys()) != {"browserbase", "browser-use", "firecrawl"} + ) + except Exception: + return False + + def _get_cloud_provider() -> Optional[CloudBrowserProvider]: """Return the configured cloud browser provider, or None for local mode. Reads ``config["browser"]["cloud_provider"]`` once and caches the result for the process lifetime. An explicit ``local`` provider disables cloud - fallback. If unset, fall back to Browserbase when direct or managed - Browserbase credentials are available. + fallback. If unset, fall back to Browser Use (managed Nous gateway or + direct API key) and then Browserbase (direct credentials only) — the + historic auto-detect order, now expressed as the + :data:`agent.browser_registry._LEGACY_PREFERENCE` walk. + + Selection routes through :mod:`agent.browser_registry` so third-party + browser plugins (``~/.hermes/plugins/browser//``) participate + in explicit-config resolution. Test fixtures that override + ``_PROVIDER_REGISTRY`` or ``BrowserUseProvider`` / ``BrowserbaseProvider`` + on this module still drive the function — see + ``_is_legacy_provider_registry_overridden``. """ global _cached_cloud_provider, _cloud_provider_resolved if _cloud_provider_resolved: @@ -437,9 +500,16 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: _cached_cloud_provider = None _cloud_provider_resolved = True return None - if provider_key and provider_key in _PROVIDER_REGISTRY: + if provider_key: try: - resolved = _PROVIDER_REGISTRY[provider_key]() + if _is_legacy_provider_registry_overridden(): + # Test fixture path: honour the patched dict so the + # cache-policy unit tests keep working. + factory = _PROVIDER_REGISTRY.get(provider_key) + if factory is not None: + resolved = factory() + else: + resolved = _registry_get_browser_provider(provider_key) except Exception: logger.warning( "Failed to instantiate explicit cloud_provider %r; will retry on next call", @@ -453,8 +523,12 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: logger.debug("Could not read cloud_provider from config: %s", e) if resolved is None: - # Prefer Browser Use (managed Nous gateway or direct API key), - # fall back to Browserbase (direct credentials only). + # Auto-detect path. When tests have patched the per-class names + # on this module (BrowserUseProvider / BrowserbaseProvider), honour + # them — the test_browser_cloud_provider_cache test relies on this. + # Otherwise route through the plugin registry's legacy preference + # walk so third-party plugins still get a chance to be selected + # when they're explicitly configured. try: fallback_provider = BrowserUseProvider() if fallback_provider.is_configured(): From 1b9c539c6e2eaf921b040b706494ce27d409e36c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:17:27 +0530 Subject: [PATCH 081/142] feat(tools): mirror image_gen plugin-injection in Browser Automation picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the three hardcoded browser-provider rows (Browserbase, Browser Use, Firecrawl) from TOOL_CATEGORIES['browser']['providers'] and replaces them with runtime injection from agent.browser_registry — mirroring the _plugin_web_search_providers() pattern PR #25182 established for the Web Search and Extract category. Adds _plugin_browser_providers() helper in hermes_cli/tools_config.py that walks list_providers() and builds a TOOL_CATEGORIES-shape dict per provider via get_setup_schema(). The new visible_providers() hook calls it for cat['name'] == 'Browser Automation'. The three remaining hardcoded rows are non-provider UX setup-flow rows: - 'Nous Subscription (Browser Use cloud)' — managed Browser Use billed via Nous subscription; uses the browser-use plugin as the underlying backend but has distinct setup UX (requires_nous_auth gates it). - 'Local Browser' — headless Chromium, no CloudBrowserProvider. - 'Camofox' — anti-detection local Firefox; _is_camofox_mode() short-circuits the cloud-provider dispatch path entirely. Verified the picker output matches pre-migration order/content: Local Browser, Camofox, Browser Use, Browserbase, Firecrawl (with 'Nous Subscription' surfaced only when the user is Nous-authed, unchanged from main). --- hermes_cli/tools_config.py | 105 ++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 9120102d646..89771291b20 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -378,6 +378,17 @@ TOOL_CATEGORIES = { "browser": { "name": "Browser Automation", "icon": "🌐", + # Per-provider rows for Browserbase, Browser Use, and Firecrawl are + # injected at runtime from plugins.browser..provider via + # _plugin_browser_providers() in _visible_providers(). Only + # non-provider UX setup-flow rows remain here: + # - "Nous Subscription (Browser Use cloud)" — managed Browser Use + # billed via Nous subscription (requires_nous_auth + + # override_env_vars). Uses the browser-use plugin as the + # underlying backend but has a distinct setup UX. + # - "Local Browser" — non-cloud option, no CloudBrowserProvider. + # - "Camofox" — anti-detection local Firefox; short-circuits the + # cloud-provider dispatch path via _is_camofox_mode(). "providers": [ { "name": "Nous Subscription (Browser Use cloud)", @@ -398,37 +409,6 @@ TOOL_CATEGORIES = { "browser_provider": "local", "post_setup": "agent_browser", }, - { - "name": "Browserbase", - "badge": "paid", - "tag": "Cloud browser with stealth and proxies", - "env_vars": [ - {"key": "BROWSERBASE_API_KEY", "prompt": "Browserbase API key", "url": "https://browserbase.com"}, - {"key": "BROWSERBASE_PROJECT_ID", "prompt": "Browserbase project ID"}, - ], - "browser_provider": "browserbase", - "post_setup": "agent_browser", - }, - { - "name": "Browser Use", - "badge": "paid", - "tag": "Cloud browser with remote execution", - "env_vars": [ - {"key": "BROWSER_USE_API_KEY", "prompt": "Browser Use API key", "url": "https://browser-use.com"}, - ], - "browser_provider": "browser-use", - "post_setup": "agent_browser", - }, - { - "name": "Firecrawl", - "badge": "paid", - "tag": "Cloud browser with remote execution", - "env_vars": [ - {"key": "FIRECRAWL_API_KEY", "prompt": "Firecrawl API key", "url": "https://firecrawl.dev"}, - ], - "browser_provider": "firecrawl", - "post_setup": "agent_browser", - }, { "name": "Camofox", "badge": "free · local", @@ -1662,6 +1642,61 @@ def _plugin_web_search_providers() -> list[dict]: return rows +# Mirror of _plugin_web_search_providers for cloud browser backends. After +# PR #25214, Browserbase / Browser Use / Firecrawl live as plugins under +# plugins/browser//; this helper is the sole source of provider rows +# for those three in the "Browser Automation" picker. The hardcoded +# ``TOOL_CATEGORIES["browser"]`` entries that drove the category before +# were deleted in the same PR; only non-provider UX setup-flow rows remain +# ("Nous Subscription", "Local Browser", "Camofox") — see the comment block +# in ``TOOL_CATEGORIES["browser"]`` for why each one stays hardcoded. +def _plugin_browser_providers() -> list[dict]: + """Build picker-row dicts from plugin-registered cloud browser providers. + + Each returned dict mirrors the legacy ``TOOL_CATEGORIES["browser"]`` + schema (``name`` / ``badge`` / ``tag`` / ``env_vars`` / + ``browser_provider`` / ``post_setup``) so the picker behaves identically + whether a provider was hardcoded or plugin-registered. + + Populates ``browser_provider`` (the legacy config key written to + ``browser.cloud_provider``) and a ``browser_plugin_name`` marker so + setup / write paths can route through the registry when they want to. + """ + try: + from agent.browser_registry import list_providers as _list_browser_providers + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + providers = _list_browser_providers() + except Exception: + return [] + + rows: list[dict] = [] + for provider in providers: + name = getattr(provider, "name", None) + if not name: + continue + try: + schema = provider.get_setup_schema() + except Exception: + continue + if not isinstance(schema, dict): + continue + row = { + "name": schema.get("name", provider.display_name), + "badge": schema.get("badge", ""), + "tag": schema.get("tag", ""), + "env_vars": schema.get("env_vars", []), + "browser_provider": name, + "browser_plugin_name": name, + } + # Pass-through optional fields the schema can opt into. + if schema.get("post_setup"): + row["post_setup"] = schema["post_setup"] + rows.append(row) + return rows + + def _visible_providers(cat: dict, config: dict) -> list[dict]: """Return provider entries visible for the current auth/config state.""" features = get_nous_subscription_features(config) @@ -1691,6 +1726,14 @@ def _visible_providers(cat: dict, config: dict) -> list[dict]: if cat.get("name") == "Web Search & Extract": visible.extend(_plugin_web_search_providers()) + # Inject plugin-registered cloud browser backends. After PR #25214, + # Browserbase / Browser Use / Firecrawl are the plugin-supplied rows; + # the hardcoded "Nous Subscription" / "Local Browser" / "Camofox" rows + # stay because they're non-provider UX setup flows (subscription auth, + # local fallback, and the REST-API anti-detection backend respectively). + if cat.get("name") == "Browser Automation": + visible.extend(_plugin_browser_providers()) + return visible From 250caebeb18c2445f8f67db4eff1e08718273ff7 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:19:26 +0530 Subject: [PATCH 082/142] refactor(browser): delete tools/browser_providers/ directory; migrate tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four files in tools/browser_providers/ (base.py, browserbase.py, browser_use.py, firecrawl.py) have been migrated into plugins/browser//provider.py over the previous commits. No in-tree code references them anymore — the legacy class names (BrowserbaseProvider / BrowserUseProvider / FirecrawlProvider) are re-exported from tools.browser_tool as aliases to the plugin classes, so existing test patches keep working. Updates tests/tools/test_managed_browserbase_and_modal.py: - Adds _load_plugin_module() helper next to _load_tool_module(). - Reroutes five _load_tool_module('tools.browser_providers.X', ...) calls to _load_plugin_module('plugins.browser.X.provider', ...). - Renames BrowserbaseProvider/BrowserUseProvider -> the new plugin class names (BrowserbaseBrowserProvider / BrowserUseBrowserProvider). - Updates is_configured() -> is_available() on the one assertion that cared about the rename (the others stay on is_configured() via the BrowserProvider ABC's backward-compat alias). Net diff: -630 / +39 lines (tests + dead-code deletion). Verified 23/23 tests in test_browser_cloud_*.py + test_managed_browserbase_and_modal.py still pass. Closes the file-tree mismatch portion of #25214. Remaining work: new plugin-level test coverage under tests/plugins/browser/, behaviour parity subprocess sweep vs origin/main, and full tests/tools/ regression sweep before opening the PR. --- .../test_managed_browserbase_and_modal.py | 61 +++-- tools/browser_providers/__init__.py | 10 - tools/browser_providers/base.py | 59 ----- tools/browser_providers/browser_use.py | 225 ------------------ tools/browser_providers/browserbase.py | 222 ----------------- tools/browser_providers/firecrawl.py | 112 --------- 6 files changed, 39 insertions(+), 650 deletions(-) delete mode 100644 tools/browser_providers/__init__.py delete mode 100644 tools/browser_providers/base.py delete mode 100644 tools/browser_providers/browser_use.py delete mode 100644 tools/browser_providers/browserbase.py delete mode 100644 tools/browser_providers/firecrawl.py diff --git a/tests/tools/test_managed_browserbase_and_modal.py b/tests/tools/test_managed_browserbase_and_modal.py index 2e1bec03b01..3d0d7b3419e 100644 --- a/tests/tools/test_managed_browserbase_and_modal.py +++ b/tests/tools/test_managed_browserbase_and_modal.py @@ -10,7 +10,9 @@ from unittest.mock import patch import pytest -TOOLS_DIR = Path(__file__).resolve().parents[2] / "tools" +REPO_ROOT = Path(__file__).resolve().parents[2] +TOOLS_DIR = REPO_ROOT / "tools" +PLUGINS_DIR = REPO_ROOT / "plugins" def _load_tool_module(module_name: str, filename: str): @@ -22,6 +24,21 @@ def _load_tool_module(module_name: str, filename: str): return module +def _load_plugin_module(module_name: str, relpath: str): + """Load a plugin module by file path from ``plugins/``. + + Mirror of :func:`_load_tool_module` for the plugin tree. Used by tests + that exercise the per-vendor browser plugins' session-lifecycle + behaviour after the PR #25214 migration. + """ + spec = spec_from_file_location(module_name, PLUGINS_DIR / relpath) + assert spec and spec.loader + module = module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + def _reset_modules(prefixes: tuple[str, ...]): for name in list(sys.modules): if name.startswith(prefixes): @@ -200,13 +217,13 @@ def test_browserbase_does_not_use_gateway_only_configuration(): }) with patch.dict(os.environ, env, clear=True): - browserbase_module = _load_tool_module( - "tools.browser_providers.browserbase", - "browser_providers/browserbase.py", + browserbase_module = _load_plugin_module( + "plugins.browser.browserbase.provider", + "browser/browserbase/provider.py", ) - provider = browserbase_module.BrowserbaseProvider() + provider = browserbase_module.BrowserbaseBrowserProvider() - assert provider.is_configured() is False + assert provider.is_available() is False def test_browser_use_managed_gateway_adds_idempotency_key_and_persists_external_call_id(): @@ -231,13 +248,13 @@ def test_browser_use_managed_gateway_adds_idempotency_key_and_persists_external_ } with patch.dict(os.environ, env, clear=True): - browser_use_module = _load_tool_module( - "tools.browser_providers.browser_use", - "browser_providers/browser_use.py", + browser_use_module = _load_plugin_module( + "plugins.browser.browser_use.provider", + "browser/browser_use/provider.py", ) with patch.object(browser_use_module.requests, "post", return_value=_Response()) as post: - provider = browser_use_module.BrowserUseProvider() + provider = browser_use_module.BrowserUseBrowserProvider() session = provider.create_session("task-browser-use-managed") sent_headers = post.call_args.kwargs["headers"] @@ -271,11 +288,11 @@ def test_browser_use_managed_gateway_reuses_pending_idempotency_key_after_timeou } with patch.dict(os.environ, env, clear=True): - browser_use_module = _load_tool_module( - "tools.browser_providers.browser_use", - "browser_providers/browser_use.py", + browser_use_module = _load_plugin_module( + "plugins.browser.browser_use.provider", + "browser/browser_use/provider.py", ) - provider = browser_use_module.BrowserUseProvider() + provider = browser_use_module.BrowserUseBrowserProvider() timeout = browser_use_module.requests.Timeout("timed out") with patch.object( @@ -333,11 +350,11 @@ def test_browser_use_managed_gateway_preserves_pending_idempotency_key_for_in_pr } with patch.dict(os.environ, env, clear=True): - browser_use_module = _load_tool_module( - "tools.browser_providers.browser_use", - "browser_providers/browser_use.py", + browser_use_module = _load_plugin_module( + "plugins.browser.browser_use.provider", + "browser/browser_use/provider.py", ) - provider = browser_use_module.BrowserUseProvider() + provider = browser_use_module.BrowserUseBrowserProvider() with patch.object( browser_use_module.requests, @@ -380,11 +397,11 @@ def test_browser_use_managed_gateway_uses_new_idempotency_key_for_a_new_session_ } with patch.dict(os.environ, env, clear=True): - browser_use_module = _load_tool_module( - "tools.browser_providers.browser_use", - "browser_providers/browser_use.py", + browser_use_module = _load_plugin_module( + "plugins.browser.browser_use.provider", + "browser/browser_use/provider.py", ) - provider = browser_use_module.BrowserUseProvider() + provider = browser_use_module.BrowserUseBrowserProvider() with patch.object(browser_use_module.requests, "post", side_effect=[_Response(), _Response()]) as post: provider.create_session("task-browser-use-new") diff --git a/tools/browser_providers/__init__.py b/tools/browser_providers/__init__.py deleted file mode 100644 index 7fa59ef04ee..00000000000 --- a/tools/browser_providers/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Cloud browser provider abstraction. - -Import the ABC so callers can do:: - - from tools.browser_providers import CloudBrowserProvider -""" - -from tools.browser_providers.base import CloudBrowserProvider - -__all__ = ["CloudBrowserProvider"] diff --git a/tools/browser_providers/base.py b/tools/browser_providers/base.py deleted file mode 100644 index 6b8e1ed4f6b..00000000000 --- a/tools/browser_providers/base.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Abstract base class for cloud browser providers.""" - -from abc import ABC, abstractmethod -from typing import Dict - - -class CloudBrowserProvider(ABC): - """Interface for cloud browser backends (Browserbase, Steel, etc.). - - Implementations live in sibling modules and are registered in - ``browser_tool._PROVIDER_REGISTRY``. The user selects a provider via - ``hermes setup`` / ``hermes tools``; the choice is persisted as - ``config["browser"]["cloud_provider"]``. - """ - - @abstractmethod - def provider_name(self) -> str: - """Short, human-readable name shown in logs and diagnostics.""" - - @abstractmethod - def is_configured(self) -> bool: - """Return True when all required env vars / credentials are present. - - Called at tool-registration time (``check_browser_requirements``) to - gate availability. Must be cheap — no network calls. - """ - - @abstractmethod - def create_session(self, task_id: str) -> Dict[str, object]: - """Create a cloud browser session and return session metadata. - - Must return a dict with at least:: - - { - "session_name": str, # unique name for agent-browser --session - "bb_session_id": str, # provider session ID (for close/cleanup) - "cdp_url": str, # CDP websocket URL - "features": dict, # feature flags that were enabled - } - - ``bb_session_id`` is a legacy key name kept for backward compat with - the rest of browser_tool.py — it holds the provider's session ID - regardless of which provider is in use. - """ - - @abstractmethod - def close_session(self, session_id: str) -> bool: - """Release / terminate a cloud session by its provider session ID. - - Returns True on success, False on failure. Should not raise. - """ - - @abstractmethod - def emergency_cleanup(self, session_id: str) -> None: - """Best-effort session teardown during process exit. - - Called from atexit / signal handlers. Must tolerate missing - credentials, network errors, etc. — log and move on. - """ diff --git a/tools/browser_providers/browser_use.py b/tools/browser_providers/browser_use.py deleted file mode 100644 index a1f4f425ba0..00000000000 --- a/tools/browser_providers/browser_use.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Browser Use cloud browser provider.""" - -import logging -import os -import threading -import uuid -from typing import Any, Dict, Optional - -import requests - -from tools.browser_providers.base import CloudBrowserProvider -from tools.managed_tool_gateway import resolve_managed_tool_gateway -from tools.tool_backend_helpers import managed_nous_tools_enabled, prefers_gateway - -logger = logging.getLogger(__name__) -_pending_create_keys: Dict[str, str] = {} -_pending_create_keys_lock = threading.Lock() - -_BASE_URL = "https://api.browser-use.com/api/v3" -_DEFAULT_MANAGED_TIMEOUT_MINUTES = 5 -_DEFAULT_MANAGED_PROXY_COUNTRY_CODE = "us" - - -def _get_or_create_pending_create_key(task_id: str) -> str: - with _pending_create_keys_lock: - existing = _pending_create_keys.get(task_id) - if existing: - return existing - - created = f"browser-use-session-create:{uuid.uuid4().hex}" - _pending_create_keys[task_id] = created - return created - - -def _clear_pending_create_key(task_id: str) -> None: - with _pending_create_keys_lock: - _pending_create_keys.pop(task_id, None) - - -def _should_preserve_pending_create_key(response: requests.Response) -> bool: - if response.status_code >= 500: - return True - - if response.status_code != 409: - return False - - try: - payload = response.json() - except Exception: - return False - - if not isinstance(payload, dict): - return False - - error = payload.get("error") - if not isinstance(error, dict): - return False - - message = str(error.get("message") or "").lower() - return "already in progress" in message - - -class BrowserUseProvider(CloudBrowserProvider): - """Browser Use (https://browser-use.com) cloud browser backend.""" - - def provider_name(self) -> str: - return "Browser Use" - - def is_configured(self) -> bool: - return self._get_config_or_none() is not None - - # ------------------------------------------------------------------ - # Config resolution (direct API key OR managed Nous gateway) - # ------------------------------------------------------------------ - - def _get_config_or_none(self) -> Optional[Dict[str, Any]]: - api_key = os.environ.get("BROWSER_USE_API_KEY") - if api_key and not prefers_gateway("browser"): - return { - "api_key": api_key, - "base_url": _BASE_URL, - "managed_mode": False, - } - - managed = resolve_managed_tool_gateway("browser-use") - if managed is None: - return None - - return { - "api_key": managed.nous_user_token, - "base_url": managed.gateway_origin.rstrip("/"), - "managed_mode": True, - } - - def _get_config(self) -> Dict[str, Any]: - config = self._get_config_or_none() - if config is None: - message = ( - "Browser Use requires a direct BROWSER_USE_API_KEY credential." - ) - if managed_nous_tools_enabled(): - message = ( - "Browser Use requires either a direct BROWSER_USE_API_KEY " - "credential or a managed Browser Use gateway configuration." - ) - raise ValueError(message) - return config - - # ------------------------------------------------------------------ - # Session lifecycle - # ------------------------------------------------------------------ - - def _headers(self, config: Dict[str, Any]) -> Dict[str, str]: - headers = { - "Content-Type": "application/json", - "X-Browser-Use-API-Key": config["api_key"], - } - return headers - - def create_session(self, task_id: str) -> Dict[str, object]: - config = self._get_config() - managed_mode = bool(config.get("managed_mode")) - - headers = self._headers(config) - if managed_mode: - headers["X-Idempotency-Key"] = _get_or_create_pending_create_key(task_id) - - # Keep gateway-backed sessions short so billing authorization does not - # default to a long Browser-Use timeout when Hermes only needs a task- - # scoped ephemeral browser. - payload = ( - { - "timeout": _DEFAULT_MANAGED_TIMEOUT_MINUTES, - "proxyCountryCode": _DEFAULT_MANAGED_PROXY_COUNTRY_CODE, - } - if managed_mode - else {} - ) - - try: - response = requests.post( - f"{config['base_url']}/browsers", - headers=headers, - json=payload, - timeout=30, - ) - except requests.RequestException as exc: - # Managed mode: propagate raw so callers can retry with the - # preserved idempotency key. Direct mode: wrap network failures - # into a clean RuntimeError for end users. - if managed_mode: - raise - raise RuntimeError( - f"Browser Use API connection failed: {exc}" - ) from exc - - if not response.ok: - if managed_mode and not _should_preserve_pending_create_key(response): - _clear_pending_create_key(task_id) - raise RuntimeError( - f"Failed to create Browser Use session: " - f"{response.status_code} {response.text}" - ) - - session_data = response.json() - if managed_mode: - _clear_pending_create_key(task_id) - session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" - external_call_id = response.headers.get("x-external-call-id") if managed_mode else None - - logger.info("Created Browser Use session %s", session_name) - - cdp_url = session_data.get("cdpUrl") or session_data.get("connectUrl") or "" - - return { - "session_name": session_name, - "bb_session_id": session_data["id"], - "cdp_url": cdp_url, - "features": {"browser_use": True}, - "external_call_id": external_call_id, - } - - def close_session(self, session_id: str) -> bool: - try: - config = self._get_config() - except ValueError: - logger.warning("Cannot close Browser Use session %s — missing credentials", session_id) - return False - - try: - response = requests.patch( - f"{config['base_url']}/browsers/{session_id}", - headers=self._headers(config), - json={"action": "stop"}, - timeout=10, - ) - if response.status_code in {200, 201, 204}: - logger.debug("Successfully closed Browser Use session %s", session_id) - return True - else: - logger.warning( - "Failed to close Browser Use session %s: HTTP %s - %s", - session_id, - response.status_code, - response.text[:200], - ) - return False - except Exception as e: - logger.error("Exception closing Browser Use session %s: %s", session_id, e) - return False - - def emergency_cleanup(self, session_id: str) -> None: - config = self._get_config_or_none() - if config is None: - logger.warning("Cannot emergency-cleanup Browser Use session %s — missing credentials", session_id) - return - try: - requests.patch( - f"{config['base_url']}/browsers/{session_id}", - headers=self._headers(config), - json={"action": "stop"}, - timeout=5, - ) - except Exception as e: - logger.debug("Emergency cleanup failed for Browser Use session %s: %s", session_id, e) diff --git a/tools/browser_providers/browserbase.py b/tools/browser_providers/browserbase.py deleted file mode 100644 index 4807345214b..00000000000 --- a/tools/browser_providers/browserbase.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Browserbase cloud browser provider (direct credentials only).""" - -import logging -import os -import uuid -from typing import Any, Dict, Optional - -import requests - -from tools.browser_providers.base import CloudBrowserProvider - -logger = logging.getLogger(__name__) - - -class BrowserbaseProvider(CloudBrowserProvider): - """Browserbase (https://browserbase.com) cloud browser backend. - - This provider requires direct BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID - credentials. Managed Nous gateway support has been removed — the Nous - subscription now routes through Browser Use instead. - """ - - def provider_name(self) -> str: - return "Browserbase" - - def is_configured(self) -> bool: - return self._get_config_or_none() is not None - - # ------------------------------------------------------------------ - # Session lifecycle - # ------------------------------------------------------------------ - - def _get_config_or_none(self) -> Optional[Dict[str, Any]]: - api_key = os.environ.get("BROWSERBASE_API_KEY") - project_id = os.environ.get("BROWSERBASE_PROJECT_ID") - if api_key and project_id: - return { - "api_key": api_key, - "project_id": project_id, - "base_url": os.environ.get("BROWSERBASE_BASE_URL", "https://api.browserbase.com").rstrip("/"), - } - return None - - def _get_config(self) -> Dict[str, Any]: - config = self._get_config_or_none() - if config is None: - raise ValueError( - "Browserbase requires BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID " - "environment variables." - ) - return config - - def create_session(self, task_id: str) -> Dict[str, object]: - config = self._get_config() - - # Optional env-var knobs - enable_proxies = os.environ.get("BROWSERBASE_PROXIES", "true").lower() != "false" - enable_advanced_stealth = os.environ.get("BROWSERBASE_ADVANCED_STEALTH", "false").lower() == "true" - enable_keep_alive = os.environ.get("BROWSERBASE_KEEP_ALIVE", "true").lower() != "false" - custom_timeout_ms = os.environ.get("BROWSERBASE_SESSION_TIMEOUT") - - features_enabled = { - "basic_stealth": True, - "proxies": False, - "advanced_stealth": False, - "keep_alive": False, - "custom_timeout": False, - } - - session_config: Dict[str, object] = {"projectId": config["project_id"]} - - if enable_keep_alive: - session_config["keepAlive"] = True - - if custom_timeout_ms: - try: - timeout_val = int(custom_timeout_ms) - if timeout_val > 0: - session_config["timeout"] = timeout_val - except ValueError: - logger.warning("Invalid BROWSERBASE_SESSION_TIMEOUT value: %s", custom_timeout_ms) - - if enable_proxies: - session_config["proxies"] = True - - if enable_advanced_stealth: - session_config["browserSettings"] = {"advancedStealth": True} - - # --- Create session via API --- - headers = { - "Content-Type": "application/json", - "X-BB-API-Key": config["api_key"], - } - - try: - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) - - proxies_fallback = False - keepalive_fallback = False - - # Handle 402 — paid features unavailable - if response.status_code == 402: - if enable_keep_alive: - keepalive_fallback = True - logger.warning( - "keepAlive may require paid plan (402), retrying without it. " - "Sessions may timeout during long operations." - ) - session_config.pop("keepAlive", None) - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) - - if response.status_code == 402 and enable_proxies: - proxies_fallback = True - logger.warning( - "Proxies unavailable (402), retrying without proxies. " - "Bot detection may be less effective." - ) - session_config.pop("proxies", None) - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) - except requests.RequestException as exc: - raise RuntimeError( - f"Browserbase API connection failed: {exc}" - ) from exc - - if not response.ok: - raise RuntimeError( - f"Failed to create Browserbase session: " - f"{response.status_code} {response.text}" - ) - - session_data = response.json() - session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" - - if enable_proxies and not proxies_fallback: - features_enabled["proxies"] = True - if enable_advanced_stealth: - features_enabled["advanced_stealth"] = True - if enable_keep_alive and not keepalive_fallback: - features_enabled["keep_alive"] = True - if custom_timeout_ms and "timeout" in session_config: - features_enabled["custom_timeout"] = True - - feature_str = ", ".join(k for k, v in features_enabled.items() if v) - logger.info("Created Browserbase session %s with features: %s", session_name, feature_str) - - return { - "session_name": session_name, - "bb_session_id": session_data["id"], - "cdp_url": session_data["connectUrl"], - "features": features_enabled, - } - - def close_session(self, session_id: str) -> bool: - try: - config = self._get_config() - except ValueError: - logger.warning("Cannot close Browserbase session %s — missing credentials", session_id) - return False - - try: - response = requests.post( - f"{config['base_url']}/v1/sessions/{session_id}", - headers={ - "X-BB-API-Key": config["api_key"], - "Content-Type": "application/json", - }, - json={ - "projectId": config["project_id"], - "status": "REQUEST_RELEASE", - }, - timeout=10, - ) - if response.status_code in {200, 201, 204}: - logger.debug("Successfully closed Browserbase session %s", session_id) - return True - else: - logger.warning( - "Failed to close session %s: HTTP %s - %s", - session_id, - response.status_code, - response.text[:200], - ) - return False - except Exception as e: - logger.error("Exception closing Browserbase session %s: %s", session_id, e) - return False - - def emergency_cleanup(self, session_id: str) -> None: - config = self._get_config_or_none() - if config is None: - logger.warning("Cannot emergency-cleanup Browserbase session %s — missing credentials", session_id) - return - try: - requests.post( - f"{config['base_url']}/v1/sessions/{session_id}", - headers={ - "X-BB-API-Key": config["api_key"], - "Content-Type": "application/json", - }, - json={ - "projectId": config["project_id"], - "status": "REQUEST_RELEASE", - }, - timeout=5, - ) - except Exception as e: - logger.debug("Emergency cleanup failed for Browserbase session %s: %s", session_id, e) diff --git a/tools/browser_providers/firecrawl.py b/tools/browser_providers/firecrawl.py deleted file mode 100644 index 4a8ae82a2d2..00000000000 --- a/tools/browser_providers/firecrawl.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Firecrawl cloud browser provider.""" - -import logging -import os -import uuid -from typing import Dict - -import requests - -from tools.browser_providers.base import CloudBrowserProvider - -logger = logging.getLogger(__name__) - -_BASE_URL = "https://api.firecrawl.dev" - - -class FirecrawlProvider(CloudBrowserProvider): - """Firecrawl (https://firecrawl.dev) cloud browser backend.""" - - def provider_name(self) -> str: - return "Firecrawl" - - def is_configured(self) -> bool: - return bool(os.environ.get("FIRECRAWL_API_KEY")) - - # ------------------------------------------------------------------ - # Session lifecycle - # ------------------------------------------------------------------ - - def _api_url(self) -> str: - return os.environ.get("FIRECRAWL_API_URL", _BASE_URL) - - def _headers(self) -> Dict[str, str]: - api_key = os.environ.get("FIRECRAWL_API_KEY") - if not api_key: - raise ValueError( - "FIRECRAWL_API_KEY environment variable is required. " - "Get your key at https://firecrawl.dev" - ) - return { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}", - } - - def create_session(self, task_id: str) -> Dict[str, object]: - ttl = int(os.environ.get("FIRECRAWL_BROWSER_TTL", "300")) - - body: Dict[str, object] = {"ttl": ttl} - - try: - response = requests.post( - f"{self._api_url()}/v2/browser", - headers=self._headers(), - json=body, - timeout=30, - ) - except requests.RequestException as exc: - raise RuntimeError( - f"Firecrawl API connection failed: {exc}" - ) from exc - - if not response.ok: - raise RuntimeError( - f"Failed to create Firecrawl browser session: " - f"{response.status_code} {response.text}" - ) - - data = response.json() - session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}" - - logger.info("Created Firecrawl browser session %s", session_name) - - return { - "session_name": session_name, - "bb_session_id": data["id"], - "cdp_url": data["cdpUrl"], - "features": {"firecrawl": True}, - } - - def close_session(self, session_id: str) -> bool: - try: - response = requests.delete( - f"{self._api_url()}/v2/browser/{session_id}", - headers=self._headers(), - timeout=10, - ) - if response.status_code in {200, 201, 204}: - logger.debug("Successfully closed Firecrawl session %s", session_id) - return True - else: - logger.warning( - "Failed to close Firecrawl session %s: HTTP %s - %s", - session_id, - response.status_code, - response.text[:200], - ) - return False - except Exception as e: - logger.error("Exception closing Firecrawl session %s: %s", session_id, e) - return False - - def emergency_cleanup(self, session_id: str) -> None: - try: - requests.delete( - f"{self._api_url()}/v2/browser/{session_id}", - headers=self._headers(), - timeout=5, - ) - except ValueError: - logger.warning("Cannot emergency-cleanup Firecrawl session %s — missing credentials", session_id) - except Exception as e: - logger.debug("Emergency cleanup failed for Firecrawl session %s: %s", session_id, e) From fec0a0da985f42cab63141c9a6a09d2468144a00 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:21:03 +0530 Subject: [PATCH 083/142] test(plugins/browser): coverage for the 3-plugin migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors tests/plugins/web/test_web_search_provider_plugins.py from PR #25182. 31 tests across 5 classes: TestBundledPluginsRegister (8 tests) - Three plugins register (browserbase, browser-use, firecrawl) - Each plugin's name + display_name accessible - get_setup_schema() returns picker-shaped dict with post_setup hook - All three lifecycle methods (create_session, close_session, emergency_cleanup) overridden on every plugin TestIsAvailable (4 tests) - browserbase needs BOTH BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID - browserbase: api_key alone or project_id alone insufficient - browser-use satisfied by BROWSER_USE_API_KEY - firecrawl satisfied by FIRECRAWL_API_KEY TestRegistryResolution (8 tests) — most valuable, locks down pre-migration semantics: - _resolve(None) with no creds returns None (local mode) - _resolve('local') short-circuits to None - _resolve('browserbase') returns provider even when unavailable (so dispatcher surfaces typed credentials error) - _resolve('firecrawl') same: explicit-config wins - _resolve('unknown') falls through to auto-detect - Legacy walk picks browser-use over browserbase - browserbase-only configuration: browserbase wins - **Regression**: firecrawl is NEVER auto-selected even when single-eligible (preserves pre-migration gate; FIRECRAWL_API_KEY shared with web firecrawl must not silently route to paid cloud browser) TestLegacyAbcAliases (6 tests) - is_configured() delegates to is_available() for all three plugins - provider_name() returns display_name for all three plugins TestPickerIntegration (3 tests) - _plugin_browser_providers() exposes all three plugins as rows - Each row carries post_setup='agent_browser' - browser_plugin_name marker matches browser_provider All tests use real imports — no mocking of provider classes — so the suite catches drift in the ABC, registry, picker injection, and plugin glue layer simultaneously. 31/31 passing. --- tests/plugins/browser/__init__.py | 0 .../browser/test_browser_provider_plugins.py | 379 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 tests/plugins/browser/__init__.py create mode 100644 tests/plugins/browser/test_browser_provider_plugins.py diff --git a/tests/plugins/browser/__init__.py b/tests/plugins/browser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/plugins/browser/test_browser_provider_plugins.py b/tests/plugins/browser/test_browser_provider_plugins.py new file mode 100644 index 00000000000..986a1d635bf --- /dev/null +++ b/tests/plugins/browser/test_browser_provider_plugins.py @@ -0,0 +1,379 @@ +"""Plugin-side tests for the browser provider migration (PR #25214). + +Covers: + +- All three bundled plugins (browserbase, browser-use, firecrawl) + instantiate and self-report the expected ABC defaults. +- Each plugin's ``is_available()`` correctly reflects env-var presence. +- The browser_registry resolves an active provider in the documented + scenarios: + * explicit config wins ignoring availability (so dispatcher surfaces + a typed credentials error) + * legacy preference walk: browser-use → browserbase (filtered by + availability) + * firecrawl is NOT in the legacy walk — explicit-only + * unknown name falls through to auto-detect + * ``local`` short-circuits to None + +These tests use *real* imports from the plugin modules — no mocking of +provider classes themselves — so the test catches drift in the ABC +interface, the registry, and the plugin glue layer simultaneously. +Mirrors ``tests/plugins/web/test_web_search_provider_plugins.py`` from +PR #25182. +""" +from __future__ import annotations + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _clear_browser_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip every browser-provider env var so is_available() returns False.""" + for k in ( + "BROWSERBASE_API_KEY", + "BROWSERBASE_PROJECT_ID", + "BROWSERBASE_BASE_URL", + "BROWSER_USE_API_KEY", + "BROWSER_USE_GATEWAY_URL", + "FIRECRAWL_API_KEY", + "FIRECRAWL_API_URL", + "FIRECRAWL_BROWSER_TTL", + "TOOL_GATEWAY_DOMAIN", + "TOOL_GATEWAY_USER_TOKEN", + ): + monkeypatch.delenv(k, raising=False) + + +def _ensure_plugins_loaded() -> None: + """Idempotently load plugins so the registry is populated.""" + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + + +# --------------------------------------------------------------------------- +# Per-test isolation +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Each test starts with a clean browser-provider env.""" + _clear_browser_env(monkeypatch) + + +# --------------------------------------------------------------------------- +# Bundled plugins register +# --------------------------------------------------------------------------- + + +class TestBundledPluginsRegister: + """All three bundled browser plugins discover and register correctly.""" + + def test_all_three_plugins_present_in_registry(self) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import list_providers + + names = sorted(p.name for p in list_providers()) + assert names == ["browser-use", "browserbase", "firecrawl"] + + @pytest.mark.parametrize( + "plugin_name,expected_display", + [ + ("browserbase", "Browserbase"), + ("browser-use", "Browser Use"), + ("firecrawl", "Firecrawl"), + ], + ) + def test_each_plugin_has_name_and_display_name( + self, plugin_name: str, expected_display: str + ) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + provider = get_provider(plugin_name) + assert provider is not None, f"plugin {plugin_name!r} not registered" + assert provider.name == plugin_name + assert provider.display_name == expected_display + + @pytest.mark.parametrize( + "plugin_name", + ["browserbase", "browser-use", "firecrawl"], + ) + def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None: + """``get_setup_schema()`` returns a dict the picker can consume.""" + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + provider = get_provider(plugin_name) + assert provider is not None + schema = provider.get_setup_schema() + assert isinstance(schema, dict) + assert "name" in schema + assert "env_vars" in schema + # Every cloud-browser plugin needs the agent-browser post-setup hook + # so the picker auto-installs the CLI on selection. + assert schema.get("post_setup") == "agent_browser" + + @pytest.mark.parametrize( + "plugin_name", + ["browserbase", "browser-use", "firecrawl"], + ) + def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None: + """The ABC's three lifecycle methods are all overridden.""" + _ensure_plugins_loaded() + from agent.browser_provider import BrowserProvider + from agent.browser_registry import get_provider + + provider = get_provider(plugin_name) + assert provider is not None + # Each method must be a real override, not the ABC's NotImplementedError + # default — we check by comparing the function reference. + assert type(provider).create_session is not BrowserProvider.create_session + assert type(provider).close_session is not BrowserProvider.close_session + assert ( + type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup + ) + + +# --------------------------------------------------------------------------- +# is_available() behavior +# --------------------------------------------------------------------------- + + +class TestIsAvailable: + """Each plugin's ``is_available()`` reflects env-var presence accurately.""" + + def test_browserbase_requires_both_api_key_and_project_id( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider("browserbase") + assert p is not None + assert p.is_available() is False + + # API key alone is insufficient. + monkeypatch.setenv("BROWSERBASE_API_KEY", "key") + assert p.is_available() is False + + # Both env vars set → available. + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj") + assert p.is_available() is True + + def test_browserbase_project_id_alone_insufficient( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider("browserbase") + assert p is not None + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj") + assert p.is_available() is False + + def test_browser_use_satisfied_by_api_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider("browser-use") + assert p is not None + assert p.is_available() is False + monkeypatch.setenv("BROWSER_USE_API_KEY", "key") + assert p.is_available() is True + + def test_firecrawl_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider("firecrawl") + assert p is not None + assert p.is_available() is False + monkeypatch.setenv("FIRECRAWL_API_KEY", "key") + assert p.is_available() is True + + +# --------------------------------------------------------------------------- +# Registry resolution semantics +# --------------------------------------------------------------------------- + + +class TestRegistryResolution: + """``_resolve()`` implements the documented three-rule precedence.""" + + def test_resolve_none_with_no_creds_returns_none(self) -> None: + """No config, no env → local mode (None).""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + assert _resolve(None) is None + + def test_explicit_local_returns_none(self) -> None: + """``cloud_provider: local`` is a positive choice; short-circuits to None.""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + assert _resolve("local") is None + + def test_explicit_browserbase_returns_provider_even_when_unavailable(self) -> None: + """Rule 1: explicit-config wins even when credentials are missing. + + This is critical — the dispatcher needs to surface a typed + credentials error rather than silently switching backends. + """ + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + provider = _resolve("browserbase") + assert provider is not None + assert provider.name == "browserbase" + assert provider.is_available() is False # confirms "ignoring availability" + + def test_explicit_firecrawl_returns_provider_even_when_unavailable(self) -> None: + """Firecrawl behaves the same as browserbase under explicit config.""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + provider = _resolve("firecrawl") + assert provider is not None + assert provider.name == "firecrawl" + + def test_explicit_unknown_falls_back_to_auto_detect(self) -> None: + """Rule 1 miss: unknown name → fall through to legacy walk.""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + # With no credentials anywhere, auto-detect should also fail. + assert _resolve("not-a-real-provider") is None + + def test_legacy_walk_prefers_browser_use_over_browserbase( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Rule 3: walk order is browser-use → browserbase.""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + # Both available — browser-use should win. + monkeypatch.setenv("BROWSER_USE_API_KEY", "k1") + monkeypatch.setenv("BROWSERBASE_API_KEY", "k2") + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p") + + provider = _resolve(None) + assert provider is not None + assert provider.name == "browser-use" + + def test_legacy_walk_falls_through_to_browserbase( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Rule 3: browser-use unavailable → browserbase picked.""" + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + monkeypatch.setenv("BROWSERBASE_API_KEY", "k") + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p") + + provider = _resolve(None) + assert provider is not None + assert provider.name == "browserbase" + + def test_firecrawl_not_in_legacy_walk_even_when_only_one_available( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Regression: firecrawl is NEVER auto-selected even when single-eligible. + + Pre-PR-#25214, the dispatcher only auto-detected between Browser Use + and Browserbase; firecrawl was reachable solely via explicit + config. We preserve that gate because FIRECRAWL_API_KEY is shared + with the *web* firecrawl plugin — auto-routing a web-extract user + to a paid cloud browser would be a real behaviour regression. + """ + _ensure_plugins_loaded() + from agent.browser_registry import _resolve + + monkeypatch.setenv("FIRECRAWL_API_KEY", "k") + + # Only firecrawl is_available() — but it's not in the legacy walk. + assert _resolve(None) is None + + +# --------------------------------------------------------------------------- +# Legacy ABC backward-compat aliases (is_configured / provider_name) +# --------------------------------------------------------------------------- + + +class TestLegacyAbcAliases: + """is_configured() and provider_name() delegate to the new API.""" + + @pytest.mark.parametrize( + "plugin_name", + ["browserbase", "browser-use", "firecrawl"], + ) + def test_is_configured_delegates_to_is_available(self, plugin_name: str) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider(plugin_name) + assert p is not None + assert p.is_configured() is p.is_available() + + @pytest.mark.parametrize( + "plugin_name,expected_label", + [ + ("browserbase", "Browserbase"), + ("browser-use", "Browser Use"), + ("firecrawl", "Firecrawl"), + ], + ) + def test_provider_name_returns_display_name( + self, plugin_name: str, expected_label: str + ) -> None: + _ensure_plugins_loaded() + from agent.browser_registry import get_provider + + p = get_provider(plugin_name) + assert p is not None + assert p.provider_name() == expected_label + + +# --------------------------------------------------------------------------- +# Picker integration +# --------------------------------------------------------------------------- + + +class TestPickerIntegration: + """`_plugin_browser_providers()` exposes all three plugins as picker rows.""" + + def test_picker_rows_match_registered_plugins(self) -> None: + _ensure_plugins_loaded() + from hermes_cli.tools_config import _plugin_browser_providers + + rows = _plugin_browser_providers() + names = sorted(r.get("browser_provider") for r in rows) + assert names == ["browser-use", "browserbase", "firecrawl"] + + def test_picker_rows_carry_post_setup_hook(self) -> None: + """Every browser plugin row has post_setup='agent_browser' so + selecting it triggers the agent-browser CLI install.""" + _ensure_plugins_loaded() + from hermes_cli.tools_config import _plugin_browser_providers + + for row in _plugin_browser_providers(): + assert row.get("post_setup") == "agent_browser", ( + f"plugin row {row['browser_provider']!r} missing post_setup hook" + ) + + def test_picker_rows_carry_browser_plugin_name_marker(self) -> None: + """`browser_plugin_name` matches `browser_provider` so downstream + code can route through the registry when it wants to.""" + _ensure_plugins_loaded() + from hermes_cli.tools_config import _plugin_browser_providers + + for row in _plugin_browser_providers(): + assert row.get("browser_plugin_name") == row.get("browser_provider") From 1bb6f03724590e5755619e97d0fe580d2cc92f9e Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:27:21 +0530 Subject: [PATCH 084/142] fix(browser): ensure plugin discovery before registry lookup; parity harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes that go together: 1. tools/browser_tool.py — add _ensure_browser_plugins_loaded() and call it from _get_cloud_provider() before consulting the registry. Normally model_tools triggers discover_plugins() as an import side-effect, but _get_cloud_provider() can be reached from contexts that haven't gone through model_tools (standalone scripts, certain unit-test paths, the new parity-sweep harness). Without the defensive call, the registry is empty and _registry_get_browser_provider() returns None — silently downgrading users to local mode when they explicitly configured a cloud provider with no credentials yet. The behavior-parity sweep below caught this as 4 scenario regressions (explicit-X-no-creds for all 3 providers, and explicit-firecrawl-with-creds). 2. tests/plugins/browser/check_parity_vs_main.py — subprocess harness that pins one Python invocation to origin/main and one to this PR's worktree via sys.path.insert(), runs _get_cloud_provider() across a 13-scenario config matrix, and diffs the reduced shape tuple (is_local, provider_name, is_available). Provider_name pulls from provider.provider_name() which is the legacy CloudBrowserProvider API and remains as a backward-compat alias on the new BrowserProvider ABC, so the comparison is apples-to-apples regardless of class identity. Final result: PARITY OK across 13 scenarios. The four observable config/credential matrices that exercise the dispatcher all match origin/main bit-for-bit: - no-config + no-env → local - explicit local + any env → local - explicit BB / BU / FC + no creds → provider returned with is_available()==False (so dispatcher surfaces typed credentials error; matches main exactly) - explicit BB / BU / FC + creds → provider returned with is_available()==True - no-config + BU creds → Browser Use - no-config + BB creds → Browserbase - no-config + both → Browser Use (legacy walk first hit) - no-config + FC only → local (firecrawl NOT in legacy walk) - no-config + FC + BB → Browserbase (legacy walk skips firecrawl) Per the dev skill's "behavior-parity for refactor PRs" rule — without this subprocess sweep, 31/31 unit tests pass while the production code path is silently broken for users who type `browser.cloud_provider: browserbase` and run a single browser command without prior model_tools import. Caught + fixed before push. --- tests/plugins/browser/check_parity_vs_main.py | 276 ++++++++++++++++++ tools/browser_tool.py | 22 ++ 2 files changed, 298 insertions(+) create mode 100644 tests/plugins/browser/check_parity_vs_main.py diff --git a/tests/plugins/browser/check_parity_vs_main.py b/tests/plugins/browser/check_parity_vs_main.py new file mode 100644 index 00000000000..11652e94af9 --- /dev/null +++ b/tests/plugins/browser/check_parity_vs_main.py @@ -0,0 +1,276 @@ +"""Behavior-parity check for the browser-provider plugin migration (#25214). + +Spawns one subprocess per (version, scenario) cell — pinned to either +origin/main (legacy in-tree providers + class-instantiation lookup) or +this PR's worktree (plugin-based registry) via `sys.path[0]`. Each +subprocess clears all browser-related env vars + writes a config.yaml, +loads `tools.browser_tool._get_cloud_provider()`, and emits a reduced +"shape tuple" {is_local, provider_name, is_available} as JSON. + +The parent process diffs the shapes per scenario. A diff means the +migration introduced an observable behaviour change vs origin/main — +which would be a real regression for users on the existing config keys. + +Run from the PR worktree: + + cd ~/.hermes/hermes-agent/.worktrees/browser-providers-plugin + python tests/plugins/browser/check_parity_vs_main.py +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +# Pin one path to current main, one to the PR worktree. +# ``REPO_ROOT`` is ``.../.worktrees/browser-providers-plugin``; the main +# checkout lives two levels up at ``~/.hermes/hermes-agent``. +MAIN_DIR = REPO_ROOT.parent.parent # ~/.hermes/hermes-agent +PR_DIR = REPO_ROOT # the worktree we're in +assert (MAIN_DIR / "tools" / "browser_tool.py").exists(), ( + f"MAIN_DIR={MAIN_DIR} doesn't look like a hermes-agent checkout" +) +assert (PR_DIR / "tools" / "browser_tool.py").exists(), ( + f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout" +) + + +# Reduced shape comparison — exact instance addresses obviously differ +# between subprocesses, so we compare the parts that matter for users. +SUBPROCESS_SCRIPT = r""" +import json, os, sys, tempfile +sys.path.insert(0, sys.argv[1]) + +# Isolated HERMES_HOME for the config write. +home = tempfile.mkdtemp() +os.environ["HERMES_HOME"] = home + +# Clear every browser-related env var so is_available() is deterministic. +for k in ( + "BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", "BROWSERBASE_BASE_URL", + "BROWSER_USE_API_KEY", "BROWSER_USE_GATEWAY_URL", + "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_BROWSER_TTL", + "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN", +): + os.environ.pop(k, None) + +# Apply per-scenario env (passed as JSON via argv[2]). +scenario_env = json.loads(sys.argv[2]) +os.environ.update(scenario_env) + +# Apply per-scenario config (passed as YAML body via argv[3]). +config_yaml = sys.argv[3] +config_path = os.path.join(home, "config.yaml") +with open(config_path, "w") as f: + f.write(config_yaml) + +# Fresh import — must not have any browser modules cached. +for name in list(sys.modules): + if name.startswith("tools.") or name.startswith("agent.") or name.startswith("plugins."): + sys.modules.pop(name, None) + +from tools.browser_tool import _get_cloud_provider, _is_local_mode + +provider = _get_cloud_provider() + +# Pull the human-readable backend name via the API that exists on BOTH +# legacy (origin/main: CloudBrowserProvider.provider_name()) and the new +# ABC (BrowserProvider exposes provider_name() as a backward-compat alias +# returning display_name). Both shapes resolve to the same string — +# 'Browserbase' / 'Browser Use' / 'Firecrawl' — so we can compare safely. +provider_name = None +is_available = None +if provider is not None: + pn = getattr(provider, "provider_name", None) + if callable(pn): + provider_name = pn() + elif isinstance(pn, str): + provider_name = pn + is_conf = getattr(provider, "is_configured", None) + if callable(is_conf): + is_available = bool(is_conf()) + +shape = { + "is_local": _is_local_mode(), + "provider_name": provider_name, + "is_available": is_available, +} +print(json.dumps(shape)) +""" + + +SCENARIOS: list[tuple[str, str, dict[str, str]]] = [ + # (label, config.yaml body, extra env vars) + ("no-config-no-env", "", {}), + ("explicit-local-no-env", "browser:\n cloud_provider: local\n", {}), + ( + "explicit-browserbase-no-creds", + "browser:\n cloud_provider: browserbase\n", + {}, + ), + ( + "explicit-browserbase-with-creds", + "browser:\n cloud_provider: browserbase\n", + {"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"}, + ), + ( + "explicit-browser-use-no-creds", + "browser:\n cloud_provider: browser-use\n", + {}, + ), + ( + "explicit-browser-use-with-creds", + "browser:\n cloud_provider: browser-use\n", + {"BROWSER_USE_API_KEY": "k"}, + ), + ( + "explicit-firecrawl-no-creds", + "browser:\n cloud_provider: firecrawl\n", + {}, + ), + ( + "explicit-firecrawl-with-creds", + "browser:\n cloud_provider: firecrawl\n", + {"FIRECRAWL_API_KEY": "k"}, + ), + ( + "no-config-bu-creds", + "", + {"BROWSER_USE_API_KEY": "k"}, + ), + ( + "no-config-bb-creds", + "", + {"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"}, + ), + ( + "no-config-both-creds", + "", + { + "BROWSER_USE_API_KEY": "k", + "BROWSERBASE_API_KEY": "x", + "BROWSERBASE_PROJECT_ID": "y", + }, + ), + ( + "no-config-firecrawl-only", + "", + {"FIRECRAWL_API_KEY": "k"}, + ), + ( + "no-config-firecrawl-and-bb", + "", + { + "FIRECRAWL_API_KEY": "k", + "BROWSERBASE_API_KEY": "x", + "BROWSERBASE_PROJECT_ID": "y", + }, + ), +] + + +def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict: + """Run one (version, scenario) cell. Returns the shape dict.""" + venv_python = repo_path / ".venv" / "bin" / "python" + if not venv_python.exists(): + # Worktrees share the main repo's venv. + venv_python = MAIN_DIR / ".venv" / "bin" / "python" + if not venv_python.exists(): + venv_python = Path("python3") + + out = subprocess.run( + [ + str(venv_python), + "-c", + SUBPROCESS_SCRIPT, + str(repo_path), + json.dumps(env), + config_yaml, + ], + capture_output=True, + text=True, + timeout=30, + ) + if out.returncode != 0: + return { + "error": "subprocess failed", + "stdout": out.stdout, + "stderr": out.stderr[-500:], + } + try: + return json.loads(out.stdout.strip().splitlines()[-1]) + except Exception as exc: + return {"error": f"could not parse output: {exc}", "stdout": out.stdout} + + +def _reduce_for_comparison(shape: dict) -> dict: + """Reduce a shape dict to the parts that matter for user-visible parity. + + We compare ``(is_local, provider_name, is_available)`` — the trio that + decides what the dispatcher does with each tool call. ``provider_name`` + is the legacy ``provider_name()`` return value ('Browserbase' / 'Browser + Use' / 'Firecrawl'), which is identical between legacy and plugin + classes (the plugin's ``display_name`` matches the legacy + ``provider_name()`` return). + """ + return { + "is_local": shape.get("is_local"), + "provider_name": shape.get("provider_name"), + "is_available": shape.get("is_available"), + } + + +def main() -> int: + print(f"main: {MAIN_DIR}") + print(f"pr: {PR_DIR}") + print() + + failures: list[str] = [] + errors: list[str] = [] + for label, config_yaml, env in SCENARIOS: + main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env) + pr_shape = _run_scenario(PR_DIR, label, config_yaml, env) + + if "error" in main_shape or "error" in pr_shape: + print(f" [ERR ] {label}: subprocess failed") + print(f" main: {main_shape}") + print(f" pr: {pr_shape}") + errors.append(label) + continue + + main_reduced = _reduce_for_comparison(main_shape) + pr_reduced = _reduce_for_comparison(pr_shape) + + if main_reduced == pr_reduced: + print(f" [OK] {label}: {main_reduced}") + else: + print(f" [FAIL] {label}") + print(f" main: {main_reduced}") + print(f" pr: {pr_reduced}") + failures.append(label) + + print() + if errors: + print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):") + for e in errors: + print(f" - {e}") + if failures: + print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):") + for f in failures: + print(f" - {f}") + if failures or errors: + return 1 + print(f"PARITY OK across {len(SCENARIOS)} scenarios.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 6fdd8949816..b089ed92133 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -465,6 +465,25 @@ def _is_legacy_provider_registry_overridden() -> bool: return False +def _ensure_browser_plugins_loaded() -> None: + """Idempotently trigger plugin discovery so the browser registry is populated. + + Normally `model_tools` is imported early in any session and that + triggers `discover_plugins()` as a side effect. But `_get_cloud_provider` + can be called from contexts that haven't gone through `model_tools` — + standalone scripts, certain unit-test paths, the parity-sweep harness. + Make discovery idempotent and side-effect-only here so users always + see registered plugins regardless of import order. Cheap: subsequent + calls early-return inside `_ensure_plugins_discovered`. + """ + try: + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + except Exception as exc: + logger.debug("Browser plugin discovery failed (non-fatal): %s", exc) + + def _get_cloud_provider() -> Optional[CloudBrowserProvider]: """Return the configured cloud browser provider, or None for local mode. @@ -509,6 +528,9 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: if factory is not None: resolved = factory() else: + # Ensure plugins are discovered so the registry is + # populated. Idempotent — cheap on subsequent calls. + _ensure_browser_plugins_loaded() resolved = _registry_get_browser_provider(provider_key) except Exception: logger.warning( From c74ff2c8effce1615074820b03e0d13997c62bb5 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Thu, 14 May 2026 14:45:29 +0530 Subject: [PATCH 085/142] =?UTF-8?q?fix(browser):=20self-review=20pass=20?= =?UTF-8?q?=E2=80=94=20dead-import,=20log=20levels,=20future-proofing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses findings from two self-review passes pre-merge. First pass (3-agent parallel review): 1. plugins/browser/browser_use/provider.py: drop the ``_ = managed_nous_tools_enabled`` dead-import-hider in _get_config_or_none(). The import was actively misleading — the helper IS used in _get_config() (separate method, separate import), not here. The "keep static analysis happy" comment was wrong about what the helper does in this scope. 2. agent/browser_provider.py: drop ``pragma: no cover`` from is_configured() / provider_name() backward-compat aliases. They ARE covered by ``TestLegacyAbcAliases`` — the pragma would have masked future regressions. 3. tools/browser_tool.py: refactor _is_legacy_provider_registry_overridden() to compare against a module-frozen _DEFAULT_PROVIDER_REGISTRY snapshot instead of hardcoded set of 3 keys. Future maintainers adding a 4th built-in provider now just extend _PROVIDER_REGISTRY; the override detection adapts automatically. Previously the hardcoded ``set(...) != {"browserbase", "browser-use", "firecrawl"}`` would flip True forever on any 4-key registry, silently routing every install onto the legacy fixture path. 4. tools/browser_tool.py: when explicit ``browser.cloud_provider`` is set but the registry has no matching plugin (typo, uninstalled plugin, discovery failure), emit a WARNING with actionable text instead of silently falling through to auto-detect. Legacy code surfaced a typed credentials error via direct class instantiation; this log restores the signal in the post-migration path. 5. agent/browser_registry.py: trim the triple-redundant _LEGACY_PREFERENCE documentation. Module docstring + 13-line block-comment + 5-line inline comment was repeating the same point. Kept the docstring and trimmed the block-comment to 5 lines. 6. agent/browser_registry.py: upgrade is_available()-raised logging from DEBUG to WARNING with exc_info=True. A provider's availability check throwing is unusual enough that users debugging "no cloud provider" need the traceback in logs. 7. tests/plugins/browser/check_parity_vs_main.py: drop dead top-level imports (os, shutil, tempfile — only referenced inside the SUBPROCESS_SCRIPT string literal that runs in a child process). Second pass (architecture + claim-verification review): 8. tools/browser_tool.py: rewrite the inline comment in _get_cloud_provider auto-detect branch. Prior text claimed it "routes through the plugin registry's legacy preference walk so third-party plugins still get a chance to be selected when they're explicitly configured" — false on both counts. The branch uses module-level legacy class aliases (BrowserUseProvider / BrowserbaseProvider) directly; third-party plugins are intentionally reachable only via explicit ``browser.cloud_provider``. Corrected comment now matches behaviour and cross-references _LEGACY_PREFERENCE for the firecrawl gate rationale. 9. tools/browser_tool.py + tests/tools/test_managed_browserbase_and_modal.py: drop the unused ``get_active_browser_provider as _registry_get_active_browser_provider`` alias from the ``from agent.browser_registry import ...`` block. It was never referenced; matching test-stub line in the agent.browser_registry SimpleNamespace also dropped. ``get_provider`` is still imported (used by the explicit-config dispatch path at line 535). 10. plugins/browser/firecrawl/provider.py: align emergency_cleanup() with the early-guard pattern used in browserbase + browser_use plugins. Previously firecrawl tried the DELETE and relied on ``_headers()`` raising ValueError to trip a "missing credentials" warning; same final outcome but a different control flow that read like a bug to a maintainer skimming the three modules. Now: if is_available() is False, log+return early — identical shape to the other two providers. Verification: 54/54 unit tests + 13/13 parity scenarios still pass. --- agent/browser_provider.py | 4 +- agent/browser_registry.py | 23 +++---- plugins/browser/browser_use/provider.py | 11 +--- plugins/browser/firecrawl/provider.py | 11 ++-- tests/plugins/browser/check_parity_vs_main.py | 3 - .../test_managed_browserbase_and_modal.py | 1 - tools/browser_tool.py | 66 ++++++++++++------- 7 files changed, 62 insertions(+), 57 deletions(-) diff --git a/agent/browser_provider.py b/agent/browser_provider.py index 338dfcd6b07..75e88e584f3 100644 --- a/agent/browser_provider.py +++ b/agent/browser_provider.py @@ -166,10 +166,10 @@ class BrowserProvider(abc.ABC): # and :attr:`name`; they may override ``is_configured`` / ``provider_name`` # for compatibility with the legacy ABC but it is not required. - def is_configured(self) -> bool: # pragma: no cover - trivial delegation + def is_configured(self) -> bool: """Backward-compat alias for :meth:`is_available`.""" return self.is_available() - def provider_name(self) -> str: # pragma: no cover - trivial delegation + def provider_name(self) -> str: """Backward-compat alias returning :attr:`display_name`.""" return self.display_name diff --git a/agent/browser_registry.py b/agent/browser_registry.py index 7b5b8b99b5f..db608744b34 100644 --- a/agent/browser_registry.py +++ b/agent/browser_registry.py @@ -99,19 +99,11 @@ def get_provider(name: str) -> Optional[BrowserProvider]: # --------------------------------------------------------------------------- -# Legacy preference order — preserves behaviour for users who set no -# ``browser.cloud_provider`` config key. Matches the historic auto-detect -# order in :func:`tools.browser_tool._get_cloud_provider` (Browser Use first -# because it covers both managed Nous gateway and direct API key; Browserbase -# second as the older direct-credentials fallback). Filtered by -# ``is_available()`` at walk time so we don't surface a provider the user -# has no credentials for. -# -# Note: ``firecrawl`` is intentionally absent. Pre-migration, the auto-detect -# branch only considered Browser Use → Browserbase; Firecrawl was reachable -# only via an explicit ``browser.cloud_provider: firecrawl`` config key. -# Preserving that gate prevents users with a ``FIRECRAWL_API_KEY`` set for -# web-extract from accidentally getting routed to a (paid) cloud browser. +# Legacy auto-detect order — used when no ``browser.cloud_provider`` is set. +# Matches the pre-migration walk in :func:`tools.browser_tool._get_cloud_provider`. +# Firecrawl is intentionally absent so users with ``FIRECRAWL_API_KEY`` set +# for web-extract don't get silently routed to a paid cloud browser. See +# :func:`_resolve` for the full rationale. _LEGACY_PREFERENCE = ( "browser-use", "browserbase", @@ -159,7 +151,10 @@ def _resolve(configured: Optional[str]) -> Optional[BrowserProvider]: try: return bool(p.is_available()) except Exception as exc: # noqa: BLE001 - logger.debug("provider %s.is_available() raised %s", p.name, exc) + logger.warning( + "Browser provider %s.is_available() raised %s — treating as unavailable", + p.name, exc, exc_info=True, + ) return False # 1. Explicit "local" short-circuit. diff --git a/plugins/browser/browser_use/provider.py b/plugins/browser/browser_use/provider.py index 82bd2420ca1..8c5af5f9f00 100644 --- a/plugins/browser/browser_use/provider.py +++ b/plugins/browser/browser_use/provider.py @@ -130,9 +130,10 @@ class BrowserUseBrowserProvider(BrowserProvider): # managed_tool_gateway pulls in the Nous auth stack which can be # heavy and is not needed for direct-API-key users. from tools.managed_tool_gateway import resolve_managed_tool_gateway - from tools.tool_backend_helpers import managed_nous_tools_enabled, prefers_gateway + from tools.tool_backend_helpers import prefers_gateway - # 1. Direct API key path (unless user explicitly prefers gateway). + # Direct API key wins unless the user has explicitly opted into the + # managed Nous gateway via ``tool_gateway.browser: gateway``. api_key = os.environ.get("BROWSER_USE_API_KEY") if api_key and not prefers_gateway("browser"): return { @@ -141,16 +142,10 @@ class BrowserUseBrowserProvider(BrowserProvider): "managed_mode": False, } - # 2. Managed Nous gateway path. managed = resolve_managed_tool_gateway("browser-use") if managed is None: return None - # Hold reference to managed_nous_tools_enabled so static analysis - # doesn't flag the import as unused — the helper is consulted by - # _get_config() below to compose a more accurate error message. - _ = managed_nous_tools_enabled - return { "api_key": managed.nous_user_token, "base_url": managed.gateway_origin.rstrip("/"), diff --git a/plugins/browser/firecrawl/provider.py b/plugins/browser/firecrawl/provider.py index a3f74d32113..498e4ffad9b 100644 --- a/plugins/browser/firecrawl/provider.py +++ b/plugins/browser/firecrawl/provider.py @@ -130,17 +130,18 @@ class FirecrawlBrowserProvider(BrowserProvider): return False def emergency_cleanup(self, session_id: str) -> None: + if not self.is_available(): + logger.warning( + "Cannot emergency-cleanup Firecrawl session %s — missing credentials", + session_id, + ) + return try: requests.delete( f"{self._api_url()}/v2/browser/{session_id}", headers=self._headers(), timeout=5, ) - except ValueError: - logger.warning( - "Cannot emergency-cleanup Firecrawl session %s — missing credentials", - session_id, - ) except Exception as e: logger.debug( "Emergency cleanup failed for Firecrawl session %s: %s", session_id, e diff --git a/tests/plugins/browser/check_parity_vs_main.py b/tests/plugins/browser/check_parity_vs_main.py index 11652e94af9..b706ce3e9c0 100644 --- a/tests/plugins/browser/check_parity_vs_main.py +++ b/tests/plugins/browser/check_parity_vs_main.py @@ -19,11 +19,8 @@ Run from the PR worktree: from __future__ import annotations import json -import os -import shutil import subprocess import sys -import tempfile from pathlib import Path diff --git a/tests/tools/test_managed_browserbase_and_modal.py b/tests/tools/test_managed_browserbase_and_modal.py index 3d0d7b3419e..d88789706ba 100644 --- a/tests/tools/test_managed_browserbase_and_modal.py +++ b/tests/tools/test_managed_browserbase_and_modal.py @@ -106,7 +106,6 @@ def _install_fake_tools_package(): BrowserProvider=_StubBrowserProvider, ) sys.modules["agent.browser_registry"] = types.SimpleNamespace( - get_active_browser_provider=lambda: None, get_provider=lambda name: None, list_providers=lambda: [], register_provider=lambda provider: None, diff --git a/tools/browser_tool.py b/tools/browser_tool.py index b089ed92133..fb96649cb38 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -90,7 +90,6 @@ except Exception: # shims for callers that import them from this module. from agent.browser_provider import BrowserProvider as CloudBrowserProvider # noqa: F401 (legacy alias) from agent.browser_registry import ( # noqa: F401 (test-patchable surface) - get_active_browser_provider as _registry_get_active_browser_provider, get_provider as _registry_get_browser_provider, ) from plugins.browser.browserbase.provider import ( # noqa: F401 (legacy import surface) @@ -425,6 +424,10 @@ _PROVIDER_REGISTRY: Dict[str, type] = { "browser-use": BrowserUseProvider, "firecrawl": FirecrawlProvider, } +# Frozen copy of the import-time _PROVIDER_REGISTRY, used by +# ``_is_legacy_provider_registry_overridden`` to detect test-time +# monkeypatching. NEVER mutate this dict. +_DEFAULT_PROVIDER_REGISTRY: Dict[str, type] = dict(_PROVIDER_REGISTRY) _cached_cloud_provider: Optional[CloudBrowserProvider] = None _cloud_provider_resolved = False @@ -442,25 +445,23 @@ _browser_engine_resolved = False def _is_legacy_provider_registry_overridden() -> bool: """Return True when a test has patched ``_PROVIDER_REGISTRY`` to a custom value. - Detected by comparing identity with the module-level defaults dict - populated above. Tests that ``monkeypatch.setattr(browser_tool, - "_PROVIDER_REGISTRY", ...)`` swap in a new object; identity differs - even when the contents happen to match. Used by ``_get_cloud_provider`` - to honour test-time overrides (which expect a factory-callable shape) - instead of routing through the plugin registry. + Detected by spotting any registered class that *isn't* the canonical + plugin-backed class for that name. Tests that + ``monkeypatch.setattr(browser_tool, "_PROVIDER_REGISTRY", ...)`` install + custom factories (`exploding_factory`, `lambda: fake_provider`, etc.); + those entries fail the canonical-class identity check below. + + Note: a future maintainer adding a 4th built-in provider only needs to + extend ``_DEFAULT_PROVIDER_REGISTRY`` below — they do NOT need to update + a hardcoded set of keys here. The detection just compares each registered + value against the corresponding canonical class. """ - # The module-level _PROVIDER_REGISTRY is built once at import time. A test - # that swaps it via monkeypatch creates a new dict; we detect that via - # the registered class identities, not by ``is`` on the dict itself - # (the patch may install a dict whose values happen to be the same - # classes; treat that as "not overridden"). try: - return ( - _PROVIDER_REGISTRY.get("browserbase") is not BrowserbaseProvider - or _PROVIDER_REGISTRY.get("browser-use") is not BrowserUseProvider - or _PROVIDER_REGISTRY.get("firecrawl") is not FirecrawlProvider - or set(_PROVIDER_REGISTRY.keys()) != {"browserbase", "browser-use", "firecrawl"} - ) + for key, default_cls in _DEFAULT_PROVIDER_REGISTRY.items(): + if _PROVIDER_REGISTRY.get(key) is not default_cls: + return True + # Extra keys not in the default registry → also an override. + return len(_PROVIDER_REGISTRY) != len(_DEFAULT_PROVIDER_REGISTRY) except Exception: return False @@ -532,6 +533,20 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: # populated. Idempotent — cheap on subsequent calls. _ensure_browser_plugins_loaded() resolved = _registry_get_browser_provider(provider_key) + if resolved is None: + # Explicit config name unknown to the registry — + # might be a typo, an uninstalled plugin, or a + # registry-population failure. Warn the user + # (legacy code would have surfaced a typed + # credentials error via direct class instantiation; + # post-migration we surface this WARNING instead). + logger.warning( + "browser.cloud_provider=%r is not a registered " + "browser plugin; falling back to auto-detect " + "(install the corresponding plugin or fix the " + "config key spelling).", + provider_key, + ) except Exception: logger.warning( "Failed to instantiate explicit cloud_provider %r; will retry on next call", @@ -545,12 +560,15 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: logger.debug("Could not read cloud_provider from config: %s", e) if resolved is None: - # Auto-detect path. When tests have patched the per-class names - # on this module (BrowserUseProvider / BrowserbaseProvider), honour - # them — the test_browser_cloud_provider_cache test relies on this. - # Otherwise route through the plugin registry's legacy preference - # walk so third-party plugins still get a chance to be selected - # when they're explicitly configured. + # Auto-detect path: Browser Use first (managed Nous gateway or + # direct API key), then Browserbase (direct credentials). Uses + # the legacy class names imported at the top of this module so + # tests that ``monkeypatch.setattr(browser_tool, "BrowserUseProvider", ...)`` + # keep driving this branch deterministically. Third-party browser + # plugins are intentionally NOT reachable from auto-detect — they + # participate only via explicit ``browser.cloud_provider: ``, + # mirroring the firecrawl gate documented on + # :data:`agent.browser_registry._LEGACY_PREFERENCE`. try: fallback_provider = BrowserUseProvider() if fallback_provider.is_configured(): From f36c89cd5798da0f313192555739975e57ffdef5 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 04:02:05 -0700 Subject: [PATCH 086/142] fix(plugins/browser): carry forward requests.RequestException wrapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #25580 was authored before #2746 landed on main, so its plugin versions of browser_use/browserbase/firecrawl ship without the requests.RequestException → RuntimeError wrapping that 13c72fb4 added to the legacy tools/browser_providers/ files for #2746. Cherry-picking the PR + git rm'ing the legacy files (the migration's intent) would silently revert that network-error fix. Port the same try/except pattern into the three plugin create_session() methods. Browser Use managed-mode keeps its raw-exception propagation (idempotency-key retry semantics). Co-authored-by: nidhi-singh02 --- plugins/browser/browser_use/provider.py | 22 +++++-- plugins/browser/browserbase/provider.py | 77 +++++++++++++------------ plugins/browser/firecrawl/provider.py | 17 ++++-- 3 files changed, 68 insertions(+), 48 deletions(-) diff --git a/plugins/browser/browser_use/provider.py b/plugins/browser/browser_use/provider.py index 8c5af5f9f00..3d371bdd88a 100644 --- a/plugins/browser/browser_use/provider.py +++ b/plugins/browser/browser_use/provider.py @@ -198,12 +198,22 @@ class BrowserUseBrowserProvider(BrowserProvider): else {} ) - response = requests.post( - f"{config['base_url']}/browsers", - headers=headers, - json=payload, - timeout=30, - ) + try: + response = requests.post( + f"{config['base_url']}/browsers", + headers=headers, + json=payload, + timeout=30, + ) + except requests.RequestException as exc: + # Managed mode: propagate raw so callers can retry with the + # preserved idempotency key. Direct mode: wrap network failures + # into a clean RuntimeError for end users. + if managed_mode: + raise + raise RuntimeError( + f"Browser Use API connection failed: {exc}" + ) from exc if not response.ok: if managed_mode and not _should_preserve_pending_create_key(response): diff --git a/plugins/browser/browserbase/provider.py b/plugins/browser/browserbase/provider.py index 0d1a646c8a6..2b05d01d03b 100644 --- a/plugins/browser/browserbase/provider.py +++ b/plugins/browser/browserbase/provider.py @@ -139,45 +139,50 @@ class BrowserbaseBrowserProvider(BrowserProvider): "X-BB-API-Key": config["api_key"], } - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) + try: + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) - proxies_fallback = False - keepalive_fallback = False + proxies_fallback = False + keepalive_fallback = False - # Handle 402 — paid features unavailable - if response.status_code == 402: - if enable_keep_alive: - keepalive_fallback = True - logger.warning( - "keepAlive may require paid plan (402), retrying without it. " - "Sessions may timeout during long operations." - ) - session_config.pop("keepAlive", None) - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) + # Handle 402 — paid features unavailable + if response.status_code == 402: + if enable_keep_alive: + keepalive_fallback = True + logger.warning( + "keepAlive may require paid plan (402), retrying without it. " + "Sessions may timeout during long operations." + ) + session_config.pop("keepAlive", None) + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) - if response.status_code == 402 and enable_proxies: - proxies_fallback = True - logger.warning( - "Proxies unavailable (402), retrying without proxies. " - "Bot detection may be less effective." - ) - session_config.pop("proxies", None) - response = requests.post( - f"{config['base_url']}/v1/sessions", - headers=headers, - json=session_config, - timeout=30, - ) + if response.status_code == 402 and enable_proxies: + proxies_fallback = True + logger.warning( + "Proxies unavailable (402), retrying without proxies. " + "Bot detection may be less effective." + ) + session_config.pop("proxies", None) + response = requests.post( + f"{config['base_url']}/v1/sessions", + headers=headers, + json=session_config, + timeout=30, + ) + except requests.RequestException as exc: + raise RuntimeError( + f"Browserbase API connection failed: {exc}" + ) from exc if not response.ok: raise RuntimeError( diff --git a/plugins/browser/firecrawl/provider.py b/plugins/browser/firecrawl/provider.py index 498e4ffad9b..2c605134a01 100644 --- a/plugins/browser/firecrawl/provider.py +++ b/plugins/browser/firecrawl/provider.py @@ -82,12 +82,17 @@ class FirecrawlBrowserProvider(BrowserProvider): body: Dict[str, object] = {"ttl": ttl} - response = requests.post( - f"{self._api_url()}/v2/browser", - headers=self._headers(), - json=body, - timeout=30, - ) + try: + response = requests.post( + f"{self._api_url()}/v2/browser", + headers=self._headers(), + json=body, + timeout=30, + ) + except requests.RequestException as exc: + raise RuntimeError( + f"Firecrawl API connection failed: {exc}" + ) from exc if not response.ok: raise RuntimeError( From 3b4dd683263c5895bb6144564e4bea8881d79993 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 11:10:06 -0500 Subject: [PATCH 087/142] fix(tui): align composer cursorLayout with wrap-ansi to kill multiline cursor drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composer's `cursorLayout` (in `ui-tui/src/lib/inputMetrics.ts`) used a hand-rolled word-wrap algorithm to decide where `useDeclaredCursor` should park the hardware cursor. But Ink's `` renders the same text via `wrap-ansi`. The two algorithms disagreed on common real-world inputs — `"branch investigate"` at cols=20, `"hello world"` at cols=8, exact-fill strings like `"abcdefgh"` at cols=8 — so the hardware cursor parked several cells past where Ink actually rendered the last character. Users saw a multi-cell blank gap between their last-typed letter and the cursor block, especially on narrow terminals (the Cursor IDE built-in terminal was the worst offender). Three previous PRs (#26717, #25860, #22197) chased fast-echo displayCursor/cursorDeclaration drift and in-band-vs-native cursor heuristics. None of them touched the underlying wrap-algorithm mismatch, which is why the bug kept resurfacing. Fix: source cursorLayout's line breaks from wrap-ansi directly. Walk its emitted string char-by-char, tracking original-string offsets, push a VisualLine at each '\n'. Also drop the buggy `column >= w` overflow rule in cursorLayout — that's what pushed exact-fill text onto a phantom next row. canFastBackspaceShape now detects the wrap boundary in BOTH coordinate conventions (column === 0 OR column >= columns), since exact-fill now reports as (0, columns) instead of the previous (1, 0). The physical state is identical — the terminal auto-wraps at column N either way — but the layout function reports the position more honestly. Tests: - ui-tui/src/__tests__/textInputWrap.test.ts: 3 tests that pinned the BUGGY behavior were updated to assert wrap-ansi parity (the real invariant). Added a typing-prefix invariant: cursorLayout must agree with wrap-ansi at every character of a long input. - ui-tui/src/__tests__/cursorDriftRegression.test.ts: new file. Walks the user-reported bug message char-by-char at 7 widths and asserts agreement with wrap-ansi at every prefix. Verification: - 791/791 vitest tests pass. - 84/84 tui-gateway pytest tests pass via scripts/run_tests.sh. - PTY repro (typing into a real `hermes --tui` PTY at cols=50/55/60): cursor lands exactly 1 cell past the last typed char in every case the bug previously drifted. --- .../__tests__/cursorDriftRegression.test.ts | 114 +++++++++++++++ ui-tui/src/__tests__/textInputWrap.test.ts | 68 +++++++-- ui-tui/src/components/textInput.tsx | 20 ++- ui-tui/src/lib/inputMetrics.ts | 134 +++++++++--------- 4 files changed, 253 insertions(+), 83 deletions(-) create mode 100644 ui-tui/src/__tests__/cursorDriftRegression.test.ts diff --git a/ui-tui/src/__tests__/cursorDriftRegression.test.ts b/ui-tui/src/__tests__/cursorDriftRegression.test.ts new file mode 100644 index 00000000000..0e562e09789 --- /dev/null +++ b/ui-tui/src/__tests__/cursorDriftRegression.test.ts @@ -0,0 +1,114 @@ +/** + * Pinned regression for the multi-line composer cursor-drift bug. + * + * Symptom: in `hermes --tui`, typing into the composer until the input + * wraps across multiple visual rows would leave several blank cells + * between the last typed character and the (hardware) cursor block. + * Worse on narrow terminals (the Cursor IDE built-in terminal in + * particular). + * + * Root cause: the composer's `cursorLayout` (used by `useDeclaredCursor` + * to place the hardware cursor) ran a hand-rolled word-wrap algorithm, + * while Ink's `` renders via `wrap-ansi`. The two + * disagreed on many real inputs — wrap-ansi would keep "branch + * investigate" on one row while cursorLayout claimed it had wrapped, + * etc. — so the declared cursor position drifted from where the text + * was actually rendered. The fix sources cursorLayout's line breaks + * directly from wrap-ansi, guaranteeing agreement. + * + * This test pins the contract: for every char that would be typed into + * the composer, the cursor position reported by cursorLayout MUST equal + * the end-of-text position that wrap-ansi would render. Any future + * regression that lets the two diverge re-introduces the drift. + */ +import { describe, expect, it } from 'vitest' +import wrapAnsi from 'wrap-ansi' + +import { cursorLayout, inputVisualHeight } from '../lib/inputMetrics.js' + +function wrapAnsiEnd(text: string, cols: number): { line: number; column: number } { + const wrapped = wrapAnsi(text, cols, { hard: true, trim: false }) + const lines = wrapped.split('\n') + const last = lines[lines.length - 1] ?? '' + + return { line: lines.length - 1, column: last.length } +} + +const USER_REPORT_MESSAGE = + // Paraphrase of the user's actual bug report, included verbatim so the + // test is grounded in a realistic typing pattern (long single line, + // mixed-length words, punctuation, no hard newlines). + 'im in cursor terminal using hermes --tui and as i type multiline my caret at the end will often ' + + 'go.. randomly.. like multiple spaces away lol and idk why. theres no rhyme/reason really but ' + + 'there should literally never be a non-user added space at the end of my composer input right? ' + + 'i dont think it happens on new sessions but only existing ones. there have been a few prs to ' + + 'try to fix this and all not working. ok it just happened, to me, nowso attaching screenshot ' + + 'and you can see its multiline, new session. on a new bb/ branch investigate' + +describe('cursor-drift regression — composer cursorLayout matches Ink rendering', () => { + it('agrees with wrap-ansi at every typing-prefix of the user-reported message', () => { + // Walks the message char-by-char (mirroring what the TUI sees when a + // user types). At every prefix, cursorLayout must place the cursor + // exactly where wrap-ansi would render the end of the text. + // + // Pre-fix: this failed on most narrow widths because the hand-rolled + // wrap algorithm broke at slightly different points than wrap-ansi. + for (const cols of [40, 50, 55, 60, 65, 70, 80]) { + let acc = '' + + for (const ch of USER_REPORT_MESSAGE) { + acc += ch + const layout = cursorLayout(acc, acc.length, cols) + const expected = wrapAnsiEnd(acc, cols) + + expect( + layout, + `mismatch at cols=${cols}, len=${acc.length}, last-char=${JSON.stringify(ch)}, ` + + `tail=${JSON.stringify(acc.slice(-30))}` + ).toEqual(expected) + } + } + }) + + it('keeps cursor on the same row when text exactly fills the terminal width', () => { + // wrap-ansi does NOT push exact-fill text onto a phantom next line. + // The previous algorithm did — that's what produced the visible + // "cursor parked one row below the last char" symptom on narrow + // terminals at certain message lengths. + for (const cols of [8, 12, 18, 24]) { + const text = 'a'.repeat(cols) + const layout = cursorLayout(text, text.length, cols) + const inkLines = wrapAnsi(text, cols, { hard: true, trim: false }).split('\n') + + expect(layout.line).toBe(0) + expect(layout.column).toBe(cols) + expect(inkLines).toHaveLength(1) + expect(inputVisualHeight(text, cols)).toBe(1) + } + }) + + it('does not stuff a trailing whitespace word onto a phantom line', () => { + // "branch investigate" at cols=20 fits on one row in wrap-ansi. The + // bug claimed otherwise, parking the cursor at (line=1, col=?) and + // leaving the user's "branch investigate" rendered alone on row 0 + // with the cursor block several cells past it. + const text = 'branch investigate' + const cols = 20 + + expect(cursorLayout(text, text.length, cols)).toEqual({ column: text.length, line: 0 }) + expect(cursorLayout(text, text.length, cols)).toEqual(wrapAnsiEnd(text, cols)) + }) + + it('agrees with wrap-ansi for word-wrap that pushes a word onto the next line', () => { + // "hello world" at cols=8 wraps to ["hello ", "world"] in wrap-ansi. + // The cursor at end-of-text must land at line=1, col=5 — where Ink + // actually renders the last 'd'. The previous algorithm reported + // (line=2, col=0) here (phantom extra wrap), which parked the + // cursor on a row Ink never painted. + const text = 'hello world' + const cols = 8 + + expect(cursorLayout(text, text.length, cols)).toEqual({ column: 5, line: 1 }) + expect(cursorLayout(text, text.length, cols)).toEqual(wrapAnsiEnd(text, cols)) + }) +}) diff --git a/ui-tui/src/__tests__/textInputWrap.test.ts b/ui-tui/src/__tests__/textInputWrap.test.ts index c25c9629e77..a0e70431465 100644 --- a/ui-tui/src/__tests__/textInputWrap.test.ts +++ b/ui-tui/src/__tests__/textInputWrap.test.ts @@ -1,8 +1,20 @@ import { describe, expect, it } from 'vitest' +import wrapAnsi from 'wrap-ansi' import { offsetFromPosition } from '../components/textInput.js' import { composerPromptWidth, cursorLayout, inputVisualHeight, stableComposerColumns } from '../lib/inputMetrics.js' +// Helper: compute the "end of text" position that wrap-ansi would render +// the input to. This is what Ink's uses, so cursorLayout +// MUST agree. Disagreement is the cursor-drift bug. +function wrapAnsiEndPosition(text: string, cols: number): { line: number; column: number } { + const wrapped = wrapAnsi(text, cols, { hard: true, trim: false }) + const lines = wrapped.split('\n') + const last = lines[lines.length - 1] ?? '' + + return { line: lines.length - 1, column: last.length } +} + describe('cursorLayout — word-wrap parity with wrap-ansi', () => { it('places cursor mid-line at its column', () => { expect(cursorLayout('hello world', 6, 40)).toEqual({ column: 6, line: 0 }) @@ -12,19 +24,36 @@ describe('cursorLayout — word-wrap parity with wrap-ansi', () => { expect(cursorLayout('hi', 2, 10)).toEqual({ column: 2, line: 0 }) }) - it('wraps to next line when cursor lands exactly at the right edge', () => { - // 8 chars on an 8-col line: text fills the row exactly; the cursor's - // inverted-space cell overflows to col 0 of the next row. - expect(cursorLayout('abcdefgh', 8, 8)).toEqual({ column: 0, line: 1 }) + it('does not push exact-fill text onto a phantom next line', () => { + // Regression: the previous hand-rolled wrap algorithm forced the cursor + // onto (line+1, 0) when the text exactly filled the row. wrap-ansi keeps + // it on the same row (no soft-wrap), so the cursor must too — otherwise + // useDeclaredCursor parks the hardware cursor below the last char and + // the user sees several blank cells between text and cursor block + // (#cursor-drift-multiline). + expect(cursorLayout('abcdefgh', 8, 8)).toEqual({ column: 8, line: 0 }) + expect(cursorLayout('abcdefgh', 8, 8)).toEqual(wrapAnsiEndPosition('abcdefgh', 8)) + }) + + it('keeps short words on the current line when they fit (no phantom wrap)', () => { + // wrap-ansi: "hello wo" at cols=8 stays as one line "hello wo". + // The old cursorLayout incorrectly pushed to (1,0) because column=8 hit + // the column>=width check, but that disagreed with what Ink actually + // rendered. + expect(cursorLayout('hello wo', 8, 8)).toEqual({ column: 8, line: 0 }) + expect(cursorLayout('hello wo', 8, 8)).toEqual(wrapAnsiEndPosition('hello wo', 8)) }) it('moves words across wrap boundaries instead of splitting them', () => { - // With wordWrap:true, "hello wor" at cols=8 is "hello \nwor" rather - // than "hello wo\nr". - expect(cursorLayout('hello wo', 8, 8)).toEqual({ column: 0, line: 1 }) + // "hello wor" at cols=8: wrap-ansi breaks at the space, "hello \nwor". expect(cursorLayout('hello wor', 9, 8)).toEqual({ column: 3, line: 1 }) expect(cursorLayout('hello worl', 10, 8)).toEqual({ column: 4, line: 1 }) expect(cursorLayout('hello world', 11, 8)).toEqual({ column: 5, line: 1 }) + + // Each must match what wrap-ansi would actually render. + expect(cursorLayout('hello wor', 9, 8)).toEqual(wrapAnsiEndPosition('hello wor', 8)) + expect(cursorLayout('hello worl', 10, 8)).toEqual(wrapAnsiEndPosition('hello worl', 8)) + expect(cursorLayout('hello world', 11, 8)).toEqual(wrapAnsiEndPosition('hello world', 8)) }) it('wraps the next word instead of splitting it at the right edge', () => { @@ -42,12 +71,33 @@ describe('cursorLayout — word-wrap parity with wrap-ansi', () => { it('does not wrap when cursor is before the right edge', () => { expect(cursorLayout('abcdefg', 7, 8)).toEqual({ column: 7, line: 0 }) }) + + it('matches wrap-ansi end-position for typing-style incremental input', () => { + // Pins the actual fix: type a long message char-by-char at a narrow + // width and assert the cursor follows wrap-ansi every step of the way. + // Before the fix, ~5 boundary positions per pass disagreed and Ink + // parked the cursor several cells past the last rendered character. + const MSG = 'on a new bb branch investigate and fix the cursor drift bug here' + + for (const cols of [10, 14, 20, 30, 50, 80]) { + let acc = '' + + for (const ch of MSG) { + acc += ch + expect(cursorLayout(acc, acc.length, cols)).toEqual(wrapAnsiEndPosition(acc, cols)) + } + } + }) }) describe('input metrics helpers', () => { - it('computes visual height from the wrapped cursor line', () => { - expect(inputVisualHeight('abcdefgh', 8)).toBe(2) + it('computes visual height matching wrap-ansi line count', () => { + // Exact-fill text stays on one line in wrap-ansi (no phantom wrap), so + // visual height is 1. The previous implementation reported 2 here. + expect(inputVisualHeight('abcdefgh', 8)).toBe(1) expect(inputVisualHeight('one\ntwo', 40)).toBe(2) + // Multi-line wrap case sanity + expect(inputVisualHeight('hello world', 8)).toBe(2) }) it('counts the prompt gap as its own cell', () => { diff --git a/ui-tui/src/components/textInput.tsx b/ui-tui/src/components/textInput.tsx index ace2f479dc1..92082280a04 100644 --- a/ui-tui/src/components/textInput.tsx +++ b/ui-tui/src/components/textInput.tsx @@ -272,10 +272,22 @@ export function canFastBackspaceShape(current: string, cursor: number, columns?: } // If we know the wrap width, reject at the soft-wrap boundary: the - // caret's visual column is 0, so "\b \b" can't represent the physical - // move back to the previous visual line. - if (columns !== undefined && cursorLayout(current, cursor, columns).column === 0) { - return false + // caret's physical column would be at (or past) the terminal's right + // edge, so the terminal has already auto-wrapped to the next row. + // "\b \b" can't represent the physical move back across that wrap. + // + // We check `column === 0` for the "wrap-ansi broke onto a new line" + // case AND `column >= columns` for the "exact-fill, terminal auto-wraps" + // case. Both manifest as the same physical state (cursor parked at + // col 0 of the next row) but cursorLayout reports them differently + // because it now mirrors wrap-ansi's break points exactly (see the + // cursor-drift-multiline fix in lib/inputMetrics.ts). + if (columns !== undefined) { + const layout = cursorLayout(current, cursor, columns) + + if (layout.column === 0 || layout.column >= columns) { + return false + } } const removed = current.slice(prevPos(current, cursor), cursor) diff --git a/ui-tui/src/lib/inputMetrics.ts b/ui-tui/src/lib/inputMetrics.ts index b5645b43310..208b3533678 100644 --- a/ui-tui/src/lib/inputMetrics.ts +++ b/ui-tui/src/lib/inputMetrics.ts @@ -1,4 +1,5 @@ import { stringWidth } from '@hermes/ink' +import wrapAnsi from 'wrap-ansi' import type { Role } from '../types.js' @@ -12,8 +13,6 @@ interface VisualLine { start: number } -const isWhitespace = (value: string) => /\s/.test(value) - const graphemes = (value: string) => [...seg().segment(value)].map(({ segment, index }) => ({ end: index + segment.length, @@ -22,79 +21,68 @@ const graphemes = (value: string) => width: Math.max(1, stringWidth(segment)) })) -function visualLines(value: string, cols: number): VisualLine[] { +// Build VisualLines from wrap-ansi's output by mapping each emitted character +// back to its original offset in `value`. wrap-ansi only INSERTS '\n' at wrap +// boundaries — it never drops, reorders, or substitutes existing characters — +// so a parallel walk uniquely identifies each line's source range. +// +// This used to be a hand-rolled word-wrap (visualLines below) whose break +// points disagreed with wrap-ansi in subtle but visible ways: exact-fill rows +// pushed the cursor to a phantom next line, mid-word breaks landed one +// grapheme off, etc. The composer's TextInput renders text via Ink's +// , which delegates to wrap-ansi — so any drift between the +// two algorithms parks the hardware cursor several cells away from the last +// rendered character. Sourcing both from wrap-ansi guarantees agreement. +function visualLinesFromWrappedOutput(value: string, cols: number): VisualLine[] { + if (!value.length) { + return [{ start: 0, end: 0 }] + } + const width = Math.max(1, cols) + const wrapped = wrapAnsi(value, width, { hard: true, trim: false }) const lines: VisualLine[] = [] - let sourceLineStart = 0 - for (const sourceLine of value.split('\n')) { - const parts = graphemes(sourceLine) + let originalIdx = 0 + let lineStart = 0 - if (!parts.length) { - lines.push({ start: sourceLineStart, end: sourceLineStart }) - sourceLineStart += 1 + for (let i = 0; i < wrapped.length; i += 1) { + const ch = wrapped[i]! + + if (ch === '\n') { + // wrap-ansi inserts '\n' to mark a soft-wrap boundary OR copies a + // literal '\n' from the input. Either way the next char in `wrapped` + // begins a new visual line. If the source character is a hard '\n', + // consume it (it doesn't appear in either line). Otherwise the '\n' + // is purely a wrap marker and originalIdx stays put. + lines.push({ start: lineStart, end: originalIdx }) + const isHardNewline = originalIdx < value.length && value[originalIdx] === '\n' + + if (isHardNewline) { + originalIdx += 1 + } + + lineStart = originalIdx continue } - let lineStartPart = 0 - let lineStartOffset = sourceLineStart - let column = 0 - let breakPart: null | number = null - let i = 0 - - while (i < parts.length) { - const part = parts[i]! - const partStart = sourceLineStart + part.index - - if (column + part.width > width && i > lineStartPart) { - if (breakPart !== null && breakPart > lineStartPart) { - const breakOffset = sourceLineStart + parts[breakPart - 1]!.end - lines.push({ start: lineStartOffset, end: breakOffset }) - lineStartPart = breakPart - lineStartOffset = breakOffset - } else { - lines.push({ start: lineStartOffset, end: partStart }) - lineStartPart = i - lineStartOffset = partStart - } - - column = 0 - breakPart = null - i = lineStartPart - continue - } - - column += part.width - - if (isWhitespace(part.segment)) { - breakPart = i + 1 - } - - i += 1 - - if (column >= width && i < parts.length) { - const next = parts[i]! - const nextStartsWord = !isWhitespace(next.segment) - - if (breakPart !== null && breakPart > lineStartPart && nextStartsWord) { - const breakOffset = sourceLineStart + parts[breakPart - 1]!.end - lines.push({ start: lineStartOffset, end: breakOffset }) - lineStartPart = breakPart - lineStartOffset = breakOffset - column = 0 - breakPart = null - i = lineStartPart - } - } - } - - lines.push({ start: lineStartOffset, end: sourceLineStart + sourceLine.length }) - sourceLineStart += sourceLine.length + 1 + // Defensive: if wrap-ansi's emitted character ever desyncs from + // `value[originalIdx]` (would only happen if it substituted, which it + // doesn't for the wrap+hard option set we use), fall back to advancing + // by one to stay in lockstep. The lines/cursor map still terminates. + originalIdx += 1 } + lines.push({ start: lineStart, end: originalIdx }) + + // wrap-ansi collapses an empty input into [""] which we already handled + // above; preserve the invariant that lines is never empty for any input. return lines.length ? lines : [{ start: 0, end: 0 }] } +function visualLines(value: string, cols: number): VisualLine[] { + return visualLinesFromWrappedOutput(value, cols) +} + function widthBetween(value: string, start: number, end: number) { let width = 0 @@ -108,6 +96,12 @@ function widthBetween(value: string, start: number, end: number) { /** * Mirrors the word-wrap behavior used by the composer TextInput. * Returns the zero-based visual line and column of the cursor cell. + * + * IMPORTANT: this MUST stay in lock-step with how Ink's `` + * lays the value out (which uses `wrap-ansi`). Any divergence parks the + * hardware cursor several cells off the last rendered character — see the + * "cursor drift past blank cells" bug. visualLinesFromWrappedOutput is + * sourced directly from wrap-ansi to enforce that invariant. */ export function cursorLayout(value: string, cursor: number, cols: number) { const pos = Math.max(0, Math.min(cursor, value.length)) @@ -124,14 +118,14 @@ export function cursorLayout(value: string, cursor: number, cols: number) { } const line = lines[lineIndex]! - let column = widthBetween(value, line.start, Math.min(pos, line.end)) - - // trailing cursor-cell overflows to the next row at the wrap column - if (column >= w) { - lineIndex++ - column = 0 - } + const column = widthBetween(value, line.start, Math.min(pos, line.end)) + // NOTE: the previous implementation forced an extra line break when + // `column >= w` (the "trailing cursor-cell overflows" rule). With + // visualLinesFromWrappedOutput sourcing breaks from wrap-ansi, the line + // wrapping above already matches what Ink will actually render. Pushing + // the cursor onto a phantom next line here would re-introduce the same + // drift we're fixing, so we don't. return { column, line: lineIndex } } From 1c0e59e557d00476e1ac0a35ceeb611e17533761 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 11:34:06 -0500 Subject: [PATCH 088/142] review(tui): address Copilot feedback on cursorLayout wrap-ansi rewrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small follow-ups from the Copilot review on #27489: 1. Declare `wrap-ansi` as a direct dependency of `ui-tui`. It was a phantom dep that resolved via npm hoisting from `@hermes/ink`'s transitive graph — fine on hoisted installs, but breaks under pnpm or `npm install --no-install-strategy=hoisted` style isolated installs. Now listed as `"wrap-ansi": "^9.0.0"` matching the @hermes/ink version. Lockfile regenerated. 2. Implement the defensive resync the comment promised. Previously the comment claimed the loop would "fall back to advancing by one to stay in lockstep" on wrap-ansi desync, but the code unconditionally advanced `originalIdx` with no actual check — so any future wrap-ansi option change or styled-input caller could silently slide `originalIdx` past the end of `value` and emit garbage line ranges. Now actually compares `value[originalIdx] === ch`, re-syncs via `indexOf` on mismatch, and bails out (returning whatever was built so far) if the desync is unrecoverable. Production paths still hit the equality fast-path on every char. 3. Drop the `visualLines` wrapper. It was a one-line indirection over `visualLinesFromWrappedOutput`. Renamed the implementation to `visualLines` and removed the wrapper — same name, no extra layer. No behavior change beyond the defensive realign; all 791 vitest tests still pass. --- ui-tui/package-lock.json | 28 ++------------------ ui-tui/package.json | 3 ++- ui-tui/src/lib/inputMetrics.ts | 48 ++++++++++++++++++++++------------ 3 files changed, 36 insertions(+), 43 deletions(-) diff --git a/ui-tui/package-lock.json b/ui-tui/package-lock.json index bbbf9552399..255c4e1b3cd 100644 --- a/ui-tui/package-lock.json +++ b/ui-tui/package-lock.json @@ -14,7 +14,8 @@ "ink-text-input": "^6.0.0", "nanostores": "^1.2.0", "react": "^19.2.4", - "unicode-animations": "^1.0.3" + "unicode-animations": "^1.0.3", + "wrap-ansi": "^9.0.0" }, "devDependencies": { "@babel/cli": "^7.28.6", @@ -503,31 +504,6 @@ "node": ">=6.9.0" } }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", diff --git a/ui-tui/package.json b/ui-tui/package.json index f28debb313e..1e11f5484da 100644 --- a/ui-tui/package.json +++ b/ui-tui/package.json @@ -22,7 +22,8 @@ "ink-text-input": "^6.0.0", "nanostores": "^1.2.0", "react": "^19.2.4", - "unicode-animations": "^1.0.3" + "unicode-animations": "^1.0.3", + "wrap-ansi": "^9.0.0" }, "devDependencies": { "@babel/cli": "^7.28.6", diff --git a/ui-tui/src/lib/inputMetrics.ts b/ui-tui/src/lib/inputMetrics.ts index 208b3533678..3b66a3dba8e 100644 --- a/ui-tui/src/lib/inputMetrics.ts +++ b/ui-tui/src/lib/inputMetrics.ts @@ -26,14 +26,14 @@ const graphemes = (value: string) => // boundaries — it never drops, reorders, or substitutes existing characters — // so a parallel walk uniquely identifies each line's source range. // -// This used to be a hand-rolled word-wrap (visualLines below) whose break -// points disagreed with wrap-ansi in subtle but visible ways: exact-fill rows -// pushed the cursor to a phantom next line, mid-word breaks landed one -// grapheme off, etc. The composer's TextInput renders text via Ink's -// , which delegates to wrap-ansi — so any drift between the -// two algorithms parks the hardware cursor several cells away from the last -// rendered character. Sourcing both from wrap-ansi guarantees agreement. -function visualLinesFromWrappedOutput(value: string, cols: number): VisualLine[] { +// This used to be a hand-rolled word-wrap whose break points disagreed with +// wrap-ansi in subtle but visible ways: exact-fill rows pushed the cursor to +// a phantom next line, mid-word breaks landed one grapheme off, etc. The +// composer's TextInput renders text via Ink's , which +// delegates to wrap-ansi — so any drift between the two algorithms parks the +// hardware cursor several cells away from the last rendered character. +// Sourcing both from wrap-ansi guarantees agreement. +function visualLines(value: string, cols: number): VisualLine[] { if (!value.length) { return [{ start: 0, end: 0 }] } @@ -65,10 +65,30 @@ function visualLinesFromWrappedOutput(value: string, cols: number): VisualLine[] continue } - // Defensive: if wrap-ansi's emitted character ever desyncs from - // `value[originalIdx]` (would only happen if it substituted, which it - // doesn't for the wrap+hard option set we use), fall back to advancing - // by one to stay in lockstep. The lines/cursor map still terminates. + // Defensive sync check. wrap-ansi (with `hard: true, trim: false`, no + // styled input) is documented to only insert '\n' at break points and + // never substitute, drop, or reorder source characters — so under those + // options `wrapped[i]` should always equal `value[originalIdx]`. But + // future option changes, library upgrades, or callers that start passing + // styled input (ANSI escapes) could violate that invariant silently. If + // they do, we'd slide `originalIdx` past the end of `value` and emit + // garbage line ranges with no diagnostic. Realign by scanning forward + // for the matching character; bail out (return whatever we have) if the + // sync is unrecoverable rather than producing wrong-but-plausible output. + if (originalIdx >= value.length) { + break + } + + if (value[originalIdx] !== ch) { + const reSync = value.indexOf(ch, originalIdx) + + if (reSync === -1) { + break + } + + originalIdx = reSync + } + originalIdx += 1 } @@ -79,10 +99,6 @@ function visualLinesFromWrappedOutput(value: string, cols: number): VisualLine[] return lines.length ? lines : [{ start: 0, end: 0 }] } -function visualLines(value: string, cols: number): VisualLine[] { - return visualLinesFromWrappedOutput(value, cols) -} - function widthBetween(value: string, start: number, end: number) { let width = 0 From 55f13be65de1cc7d9c494b45f7899d9119babd23 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 11:38:33 -0500 Subject: [PATCH 089/142] chore(nix): refresh ui-tui npmDeps hash for wrap-ansi dep addition --- nix/tui.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tui.nix b/nix/tui.nix index b64e8d21fc2..d0828d9438a 100644 --- a/nix/tui.nix +++ b/nix/tui.nix @@ -4,7 +4,7 @@ let src = ../ui-tui; npmDeps = pkgs.fetchNpmDeps { inherit src; - hash = "sha256-9r1EYQ600gNXOnNXwakorpEk7hS/FPxZVbB2JksrhYs="; + hash = "sha256-+2lmAE9K2GorQzIqET+TW0mj+ibBa8pbfOALMnmFp6A="; }; npm = hermesNpmLib.mkNpmPassthru { folder = "ui-tui"; attr = "tui"; pname = "hermes-tui"; }; From 8c78f533ddf988498eda025ed480f71062a82984 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 11:52:21 -0500 Subject: [PATCH 090/142] review(tui): route cursorLayout through @hermes/ink wrapAnsi shim (Bun runtime parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot caught an important runtime parity gap on PR #27489: the fix imported the npm `wrap-ansi` package directly, but Ink's `` uses a runtime-selecting shim (`ui-tui/packages/hermes-ink/src/ink/wrapAnsi.ts`) that prefers `Bun.wrapAnsi` when running under Bun and falls back to the npm package elsewhere. So under Bun, Ink would render via `Bun.wrapAnsi` while `cursorLayout` would compute breaks via the npm package — any disagreement reintroduces the exact cursor-drift symptom the PR is meant to eliminate. Fix: - Export `wrapAnsi` from `@hermes/ink` (`packages/hermes-ink/src/entry-exports.ts` and `packages/hermes-ink/index.d.ts`) so the shim is the public surface. - Switch `ui-tui/src/lib/inputMetrics.ts` from `import wrapAnsi from 'wrap-ansi'` to `import { wrapAnsi } from '@hermes/ink'`. Both renderer (Ink) and cursor layout now traverse the same shim, so they share the runtime-selected implementation by construction. - Same swap in `textInputWrap.test.ts` and `cursorDriftRegression.test.ts` — tests now assert parity through the shim, which means under Bun they actually exercise Bun's implementation instead of asserting a tautology against the npm package. - Drop the direct `"wrap-ansi": "^9.0.0"` from `ui-tui/package.json`. `@hermes/ink` (which IS a declared dep) pulls wrap-ansi in transitively — that's not a phantom dep because the import path goes through `@hermes/ink`'s public exports, not through a hoisting accident. Verified: 791/791 vitest tests pass. `@hermes/ink` rebuilt (`dist/entry-exports.js` includes `wrapAnsi` export). TUI bundle rebuilt clean. --- ui-tui/package-lock.json | 3 +-- ui-tui/package.json | 3 +-- ui-tui/packages/hermes-ink/index.d.ts | 1 + ui-tui/packages/hermes-ink/src/entry-exports.ts | 1 + ui-tui/src/__tests__/cursorDriftRegression.test.ts | 2 +- ui-tui/src/__tests__/textInputWrap.test.ts | 2 +- ui-tui/src/lib/inputMetrics.ts | 3 +-- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ui-tui/package-lock.json b/ui-tui/package-lock.json index 255c4e1b3cd..44e9cbde923 100644 --- a/ui-tui/package-lock.json +++ b/ui-tui/package-lock.json @@ -14,8 +14,7 @@ "ink-text-input": "^6.0.0", "nanostores": "^1.2.0", "react": "^19.2.4", - "unicode-animations": "^1.0.3", - "wrap-ansi": "^9.0.0" + "unicode-animations": "^1.0.3" }, "devDependencies": { "@babel/cli": "^7.28.6", diff --git a/ui-tui/package.json b/ui-tui/package.json index 1e11f5484da..f28debb313e 100644 --- a/ui-tui/package.json +++ b/ui-tui/package.json @@ -22,8 +22,7 @@ "ink-text-input": "^6.0.0", "nanostores": "^1.2.0", "react": "^19.2.4", - "unicode-animations": "^1.0.3", - "wrap-ansi": "^9.0.0" + "unicode-animations": "^1.0.3" }, "devDependencies": { "@babel/cli": "^7.28.6", diff --git a/ui-tui/packages/hermes-ink/index.d.ts b/ui-tui/packages/hermes-ink/index.d.ts index 5d5ae9387c0..66fed32ae60 100644 --- a/ui-tui/packages/hermes-ink/index.d.ts +++ b/ui-tui/packages/hermes-ink/index.d.ts @@ -34,5 +34,6 @@ export { default as measureElement } from './src/ink/measure-element.ts' export { createRoot, forceRedraw, default as render, renderSync } from './src/ink/root.ts' export type { Instance, RenderOptions, Root } from './src/ink/root.ts' export { stringWidth } from './src/ink/stringWidth.ts' +export { wrapAnsi } from './src/ink/wrapAnsi.ts' export { default as TextInput, UncontrolledTextInput } from 'ink-text-input' export type { Props as TextInputProps } from 'ink-text-input' diff --git a/ui-tui/packages/hermes-ink/src/entry-exports.ts b/ui-tui/packages/hermes-ink/src/entry-exports.ts index d173e0c9bb1..a113660385f 100644 --- a/ui-tui/packages/hermes-ink/src/entry-exports.ts +++ b/ui-tui/packages/hermes-ink/src/entry-exports.ts @@ -26,5 +26,6 @@ export { default as measureElement } from './ink/measure-element.js' export { scrollFastPathStats, type ScrollFastPathStats } from './ink/render-node-to-output.js' export { createRoot, forceRedraw, default as render, renderSync } from './ink/root.js' export { stringWidth } from './ink/stringWidth.js' +export { wrapAnsi } from './ink/wrapAnsi.js' export { isXtermJs } from './ink/terminal.js' export { default as TextInput, UncontrolledTextInput } from 'ink-text-input' diff --git a/ui-tui/src/__tests__/cursorDriftRegression.test.ts b/ui-tui/src/__tests__/cursorDriftRegression.test.ts index 0e562e09789..3f9082dcefc 100644 --- a/ui-tui/src/__tests__/cursorDriftRegression.test.ts +++ b/ui-tui/src/__tests__/cursorDriftRegression.test.ts @@ -21,8 +21,8 @@ * the end-of-text position that wrap-ansi would render. Any future * regression that lets the two diverge re-introduces the drift. */ +import { wrapAnsi } from '@hermes/ink' import { describe, expect, it } from 'vitest' -import wrapAnsi from 'wrap-ansi' import { cursorLayout, inputVisualHeight } from '../lib/inputMetrics.js' diff --git a/ui-tui/src/__tests__/textInputWrap.test.ts b/ui-tui/src/__tests__/textInputWrap.test.ts index a0e70431465..22b33c9480e 100644 --- a/ui-tui/src/__tests__/textInputWrap.test.ts +++ b/ui-tui/src/__tests__/textInputWrap.test.ts @@ -1,5 +1,5 @@ +import { wrapAnsi } from '@hermes/ink' import { describe, expect, it } from 'vitest' -import wrapAnsi from 'wrap-ansi' import { offsetFromPosition } from '../components/textInput.js' import { composerPromptWidth, cursorLayout, inputVisualHeight, stableComposerColumns } from '../lib/inputMetrics.js' diff --git a/ui-tui/src/lib/inputMetrics.ts b/ui-tui/src/lib/inputMetrics.ts index 3b66a3dba8e..3d8a0c61bb8 100644 --- a/ui-tui/src/lib/inputMetrics.ts +++ b/ui-tui/src/lib/inputMetrics.ts @@ -1,5 +1,4 @@ -import { stringWidth } from '@hermes/ink' -import wrapAnsi from 'wrap-ansi' +import { stringWidth, wrapAnsi } from '@hermes/ink' import type { Role } from '../types.js' From 220736f41726cbd2445c2904a97c1971b2612730 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 11:54:48 -0500 Subject: [PATCH 091/142] chore(nix): refresh ui-tui npmDeps hash after wrap-ansi direct-dep drop --- nix/tui.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tui.nix b/nix/tui.nix index d0828d9438a..33ede8b2dcc 100644 --- a/nix/tui.nix +++ b/nix/tui.nix @@ -4,7 +4,7 @@ let src = ../ui-tui; npmDeps = pkgs.fetchNpmDeps { inherit src; - hash = "sha256-+2lmAE9K2GorQzIqET+TW0mj+ibBa8pbfOALMnmFp6A="; + hash = "sha256-uod1G7SWEjhYNTQ2/MG1Q1JDrQ41H0by9tspv8zh0h4="; }; npm = hermesNpmLib.mkNpmPassthru { folder = "ui-tui"; attr = "tui"; pname = "hermes-tui"; }; From 711f46e4bdbf1ec07d949f0c6726a6e034ac4509 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 12:32:29 -0500 Subject: [PATCH 092/142] review(tui): update stale comment refs to renamed visualLines helper --- ui-tui/src/lib/inputMetrics.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-tui/src/lib/inputMetrics.ts b/ui-tui/src/lib/inputMetrics.ts index 3d8a0c61bb8..4c624da167a 100644 --- a/ui-tui/src/lib/inputMetrics.ts +++ b/ui-tui/src/lib/inputMetrics.ts @@ -115,8 +115,8 @@ function widthBetween(value: string, start: number, end: number) { * IMPORTANT: this MUST stay in lock-step with how Ink's `` * lays the value out (which uses `wrap-ansi`). Any divergence parks the * hardware cursor several cells off the last rendered character — see the - * "cursor drift past blank cells" bug. visualLinesFromWrappedOutput is - * sourced directly from wrap-ansi to enforce that invariant. + * "cursor drift past blank cells" bug. `visualLines` is sourced directly + * from wrap-ansi to enforce that invariant. */ export function cursorLayout(value: string, cursor: number, cols: number) { const pos = Math.max(0, Math.min(cursor, value.length)) @@ -137,9 +137,9 @@ export function cursorLayout(value: string, cursor: number, cols: number) { // NOTE: the previous implementation forced an extra line break when // `column >= w` (the "trailing cursor-cell overflows" rule). With - // visualLinesFromWrappedOutput sourcing breaks from wrap-ansi, the line - // wrapping above already matches what Ink will actually render. Pushing - // the cursor onto a phantom next line here would re-introduce the same + // `visualLines` sourcing breaks from wrap-ansi, the line wrapping + // above already matches what Ink will actually render. Pushing the + // cursor onto a phantom next line here would re-introduce the same // drift we're fixing, so we don't. return { column, line: lineIndex } } From caac54796bbdd28131ee2c105fe7585ca245674c Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 17 May 2026 13:33:10 -0500 Subject: [PATCH 093/142] chore: revert unrelated package-lock + nix hash churn to keep PR diff minimal --- nix/tui.nix | 2 +- ui-tui/package-lock.json | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nix/tui.nix b/nix/tui.nix index 33ede8b2dcc..b64e8d21fc2 100644 --- a/nix/tui.nix +++ b/nix/tui.nix @@ -4,7 +4,7 @@ let src = ../ui-tui; npmDeps = pkgs.fetchNpmDeps { inherit src; - hash = "sha256-uod1G7SWEjhYNTQ2/MG1Q1JDrQ41H0by9tspv8zh0h4="; + hash = "sha256-9r1EYQ600gNXOnNXwakorpEk7hS/FPxZVbB2JksrhYs="; }; npm = hermesNpmLib.mkNpmPassthru { folder = "ui-tui"; attr = "tui"; pname = "hermes-tui"; }; diff --git a/ui-tui/package-lock.json b/ui-tui/package-lock.json index 44e9cbde923..bbbf9552399 100644 --- a/ui-tui/package-lock.json +++ b/ui-tui/package-lock.json @@ -503,6 +503,31 @@ "node": ">=6.9.0" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", From e89d78ff09cc0bcca4396cb50faa2e9da4301e48 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sun, 17 May 2026 03:40:22 +0300 Subject: [PATCH 094/142] fix(doctor): suppress stale XAI_API_KEY issue when xAI OAuth is healthy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _has_healthy_oauth_fallback_for_apikey_provider() covers Gemini and MiniMax (added by #26853) but omits xAI. The xAI provider profile (plugins/model-providers/xai/__init__.py) has auth_type="api_key" and env_vars=("XAI_API_KEY",), so it enters the generic API-key connectivity loop. When XAI_API_KEY fails a 401 probe but xAI OAuth is healthy, the failure is promoted to the blocking summary even though xAI works fine via OAuth — the same false-positive #26853 fixed for Gemini and MiniMax. Fix: import get_xai_oauth_auth_status alongside the existing two helpers and add the "xai" branch. get_xai_oauth_auth_status() already exists in hermes_cli/auth.py and returns {"logged_in": True} when a valid OAuth token is present. Symmetric with the Gemini and MiniMax branches introduced in #26853. No behavior change for providers without an OAuth path. --- hermes_cli/doctor.py | 3 +++ tests/hermes_cli/test_doctor.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index ef668e07940..04cfffef922 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -164,6 +164,7 @@ def _has_healthy_oauth_fallback_for_apikey_provider(provider_label: str) -> bool from hermes_cli.auth import ( get_gemini_oauth_auth_status, get_minimax_oauth_auth_status, + get_xai_oauth_auth_status, ) except Exception: return False @@ -173,6 +174,8 @@ def _has_healthy_oauth_fallback_for_apikey_provider(provider_label: str) -> bool return bool((get_gemini_oauth_auth_status() or {}).get("logged_in")) if normalized == "minimax": return bool((get_minimax_oauth_auth_status() or {}).get("logged_in")) + if normalized == "xai": + return bool((get_xai_oauth_auth_status() or {}).get("logged_in")) return False diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index ee419656a71..d99947a9886 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -944,3 +944,21 @@ def test_run_doctor_ignores_invalid_direct_keys_when_oauth_fallback_is_healthy( assert "invalid API key" in out assert unexpected_issue not in out + + +class TestHasHealthyOauthFallbackForXai: + def test_returns_true_when_xai_oauth_healthy(self, monkeypatch): + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {"logged_in": True}) + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("xai") is True + + def test_returns_false_when_xai_oauth_not_logged_in(self, monkeypatch): + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {"logged_in": False}) + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("xai") is False + + def test_returns_false_for_unknown_provider(self): + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("unknown-provider") is False From e10bb9dffa5908f21f6a97d7e2c4466de76b61e5 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sun, 17 May 2026 03:53:23 +0300 Subject: [PATCH 095/142] fix(doctor): isolate per-provider OAuth imports to prevent fallback regression Shared try/except import block meant that if any one status function was missing, all providers lost their OAuth fallback suppression. Split into per-provider try/except so each branch is independently safe. Add end-to-end test for xAI: bad XAI_API_KEY with healthy OAuth does not surface a blocking issue in run_doctor output. Add tests for None return, import failure isolation (xAI missing does not break Gemini), and move test_returns_false_for_unknown_provider out of the xAI-specific class. --- hermes_cli/doctor.py | 27 ++++++++++--------- tests/hermes_cli/test_doctor.py | 48 ++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 04cfffef922..a3d5764835f 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -160,22 +160,25 @@ def _has_healthy_oauth_fallback_for_apikey_provider(provider_label: str) -> bool still show a failed API-key connectivity row, but it should not promote that direct-key problem into the final blocking summary. """ - try: - from hermes_cli.auth import ( - get_gemini_oauth_auth_status, - get_minimax_oauth_auth_status, - get_xai_oauth_auth_status, - ) - except Exception: - return False - normalized = (provider_label or "").strip().lower() if normalized in {"google / gemini", "gemini"}: - return bool((get_gemini_oauth_auth_status() or {}).get("logged_in")) + try: + from hermes_cli.auth import get_gemini_oauth_auth_status + return bool((get_gemini_oauth_auth_status() or {}).get("logged_in")) + except Exception: + return False if normalized == "minimax": - return bool((get_minimax_oauth_auth_status() or {}).get("logged_in")) + try: + from hermes_cli.auth import get_minimax_oauth_auth_status + return bool((get_minimax_oauth_auth_status() or {}).get("logged_in")) + except Exception: + return False if normalized == "xai": - return bool((get_xai_oauth_auth_status() or {}).get("logged_in")) + try: + from hermes_cli.auth import get_xai_oauth_auth_status + return bool((get_xai_oauth_auth_status() or {}).get("logged_in")) + except Exception: + return False return False diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index d99947a9886..4f9a9e93cba 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -850,6 +850,7 @@ def _run_doctor_with_healthy_oauth_fallback( failing_host: str, gemini_oauth_status: dict, minimax_oauth_status: dict, + xai_oauth_status: dict | None = None, ) -> str: home = tmp_path / ".hermes" home.mkdir(parents=True, exist_ok=True) @@ -886,6 +887,8 @@ def _run_doctor_with_healthy_oauth_fallback( monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: gemini_oauth_status) monkeypatch.setattr(_auth_mod, "get_minimax_oauth_auth_status", lambda: minimax_oauth_status) + _xai_status = xai_oauth_status if xai_oauth_status is not None else {} + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: _xai_status) def fake_get(url, headers=None, timeout=None): status = 401 if failing_host in url else 200 @@ -902,7 +905,7 @@ def _run_doctor_with_healthy_oauth_fallback( @pytest.mark.parametrize( - ("env_key", "bad_key", "failing_host", "gemini_oauth_status", "minimax_oauth_status", "unexpected_issue"), + ("env_key", "bad_key", "failing_host", "gemini_oauth_status", "minimax_oauth_status", "xai_oauth_status", "unexpected_issue"), [ ( "GOOGLE_API_KEY", @@ -910,6 +913,7 @@ def _run_doctor_with_healthy_oauth_fallback( "googleapis.com", {"logged_in": True, "email": "user@example.com"}, {}, + None, "Check GOOGLE_API_KEY in .env", ), ( @@ -918,8 +922,18 @@ def _run_doctor_with_healthy_oauth_fallback( "minimax.io", {}, {"logged_in": True, "region": "global"}, + None, "Check MINIMAX_API_KEY in .env", ), + ( + "XAI_API_KEY", + "bad-xai-key", + "api.x.ai", + {}, + {}, + {"logged_in": True, "auth_mode": "oauth_pkce"}, + "Check XAI_API_KEY in .env", + ), ], ) def test_run_doctor_ignores_invalid_direct_keys_when_oauth_fallback_is_healthy( @@ -930,6 +944,7 @@ def test_run_doctor_ignores_invalid_direct_keys_when_oauth_fallback_is_healthy( failing_host, gemini_oauth_status, minimax_oauth_status, + xai_oauth_status, unexpected_issue, ): out = _run_doctor_with_healthy_oauth_fallback( @@ -940,12 +955,18 @@ def test_run_doctor_ignores_invalid_direct_keys_when_oauth_fallback_is_healthy( failing_host=failing_host, gemini_oauth_status=gemini_oauth_status, minimax_oauth_status=minimax_oauth_status, + xai_oauth_status=xai_oauth_status, ) assert "invalid API key" in out assert unexpected_issue not in out +def test_has_healthy_oauth_fallback_returns_false_for_unknown_provider(): + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("unknown-provider") is False + + class TestHasHealthyOauthFallbackForXai: def test_returns_true_when_xai_oauth_healthy(self, monkeypatch): from hermes_cli import auth as _auth_mod @@ -959,6 +980,27 @@ class TestHasHealthyOauthFallbackForXai: from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider assert _has_healthy_oauth_fallback_for_apikey_provider("xai") is False - def test_returns_false_for_unknown_provider(self): + def test_returns_false_when_xai_oauth_returns_none(self, monkeypatch): + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: None) from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider - assert _has_healthy_oauth_fallback_for_apikey_provider("unknown-provider") is False + assert _has_healthy_oauth_fallback_for_apikey_provider("xai") is False + + def test_returns_false_when_xai_import_unavailable(self, monkeypatch): + import sys + # Simulate get_xai_oauth_auth_status missing from auth module + monkeypatch.delattr("hermes_cli.auth.get_xai_oauth_auth_status", raising=False) + # Force doctor module to re-import the function + monkeypatch.delitem(sys.modules, "hermes_cli.doctor", raising=False) + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("xai") is False + + def test_xai_import_failure_does_not_affect_gemini(self, monkeypatch): + import sys + from hermes_cli import auth as _auth_mod + # xAI function missing, but Gemini is healthy + monkeypatch.delattr(_auth_mod, "get_xai_oauth_auth_status", raising=False) + monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: {"logged_in": True}) + monkeypatch.delitem(sys.modules, "hermes_cli.doctor", raising=False) + from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider + assert _has_healthy_oauth_fallback_for_apikey_provider("gemini") is True From 016893f5e47b32dba0c16a3c38279de0cb590243 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sun, 17 May 2026 04:01:29 +0300 Subject: [PATCH 096/142] feat(status): show xAI OAuth login state in hermes status hermes status listed Nous Portal, OpenAI Codex, Qwen OAuth, and MiniMax OAuth in the Auth Providers section but omitted xAI OAuth entirely. Users who authenticated via `hermes auth add xai-oauth` had no way to verify their session state from the status output. Add xAI OAuth display using the same field shape as OpenAI Codex: auth_store (Auth file:), last_refresh (Refreshed:), and error when not logged in. The import is isolated in its own try/except so an import failure cannot affect the already-printed rows above it. Tests cover: - logged in: check mark, auth_store, last_refresh, error suppressed - not logged in: login command hint, error shown, error absent = no line - resilience: import failure, status function raises, returns None - isolation: xAI import failure does not break Nous/MiniMax display --- hermes_cli/status.py | 21 +++ tests/hermes_cli/test_status.py | 223 ++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+) diff --git a/hermes_cli/status.py b/hermes_cli/status.py index f2164ac8a4d..5629da03fe3 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -259,6 +259,27 @@ def show_status(args): if minimax_status.get("error") and not minimax_logged_in: print(f" Error: {minimax_status.get('error')}") + # xAI OAuth — separate try/except so an import failure here cannot + # disrupt the already-printed Nous/Codex/Qwen/MiniMax rows above. + try: + from hermes_cli.auth import get_xai_oauth_auth_status + xai_oauth_status = get_xai_oauth_auth_status() or {} + except Exception: + xai_oauth_status = {} + + xai_oauth_logged_in = bool(xai_oauth_status.get("logged_in")) + print( + f" {'xAI OAuth':<12} {check_mark(xai_oauth_logged_in)} " + f"{'logged in' if xai_oauth_logged_in else 'not logged in (run: hermes auth add xai-oauth)'}" + ) + xai_auth_file = xai_oauth_status.get("auth_store") + if xai_auth_file: + print(f" Auth file: {xai_auth_file}") + if xai_oauth_status.get("last_refresh"): + print(f" Refreshed: {_format_iso_timestamp(xai_oauth_status.get('last_refresh'))}") + if xai_oauth_status.get("error") and not xai_oauth_logged_in: + print(f" Error: {xai_oauth_status.get('error')}") + # ========================================================================= # Nous Subscription Features # ========================================================================= diff --git a/tests/hermes_cli/test_status.py b/tests/hermes_cli/test_status.py index a13e843faf8..3cee9ab10ba 100644 --- a/tests/hermes_cli/test_status.py +++ b/tests/hermes_cli/test_status.py @@ -29,6 +29,7 @@ def test_show_status_termux_gateway_section_skips_systemctl(monkeypatch, capsys, monkeypatch.setattr(status_mod, "provider_label", lambda provider: "OpenAI Codex", raising=False) monkeypatch.setattr(auth_mod, "get_nous_auth_status", lambda: {}, raising=False) monkeypatch.setattr(auth_mod, "get_codex_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", lambda: {}, raising=False) monkeypatch.setattr(gateway_mod, "find_gateway_pids", lambda exclude_pids=None: [], raising=False) def _unexpected_systemctl(*args, **kwargs): @@ -70,6 +71,7 @@ def test_show_status_reports_nous_auth_error(monkeypatch, capsys, tmp_path): ) monkeypatch.setattr(auth_mod, "get_codex_auth_status", lambda: {}, raising=False) monkeypatch.setattr(auth_mod, "get_qwen_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", lambda: {}, raising=False) monkeypatch.setattr(gateway_mod, "find_gateway_pids", lambda exclude_pids=None: [], raising=False) status_mod.show_status(SimpleNamespace(all=False, deep=False)) @@ -96,6 +98,7 @@ def test_show_status_reports_vercel_backend_contract(monkeypatch, capsys, tmp_pa monkeypatch.setattr(auth_mod, "get_nous_auth_status", lambda: {}, raising=False) monkeypatch.setattr(auth_mod, "get_codex_auth_status", lambda: {}, raising=False) monkeypatch.setattr(auth_mod, "get_qwen_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", lambda: {}, raising=False) monkeypatch.setattr(gateway_mod, "find_gateway_pids", lambda exclude_pids=None: [], raising=False) status_mod.show_status(SimpleNamespace(all=False, deep=False)) @@ -109,3 +112,223 @@ def test_show_status_reports_vercel_backend_contract(monkeypatch, capsys, tmp_pa assert "oidc-token" not in output assert "snapshot filesystem" in output assert "live processes do not survive" in output + + +# --------------------------------------------------------------------------- +# Helpers shared by xAI OAuth status tests +# --------------------------------------------------------------------------- + +def _base_xai_mocks(monkeypatch, tmp_path): + """Set up the minimal environment for show_status, returning status_mod.""" + from hermes_cli import status as status_mod + import hermes_cli.auth as auth_mod + import hermes_cli.gateway as gateway_mod + + monkeypatch.setattr(status_mod, "get_env_path", lambda: tmp_path / ".env", raising=False) + monkeypatch.setattr(status_mod, "get_hermes_home", lambda: tmp_path, raising=False) + monkeypatch.setattr(status_mod, "load_config", lambda: {"model": "gpt-5.4"}, raising=False) + monkeypatch.setattr(status_mod, "resolve_requested_provider", lambda requested=None: "openai-codex", raising=False) + monkeypatch.setattr(status_mod, "resolve_provider", lambda requested=None, **kwargs: "openai-codex", raising=False) + monkeypatch.setattr(status_mod, "provider_label", lambda provider: "OpenAI Codex", raising=False) + monkeypatch.setattr(auth_mod, "get_nous_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_codex_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_qwen_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(auth_mod, "get_minimax_oauth_auth_status", lambda: {}, raising=False) + monkeypatch.setattr(gateway_mod, "find_gateway_pids", lambda exclude_pids=None: [], raising=False) + return status_mod + + +class TestShowStatusXaiOAuth: + """xAI OAuth row in hermes status.""" + + # ------------------------------------------------------------------ + # Logged-in branch + # ------------------------------------------------------------------ + + def test_logged_in_shows_check_mark_and_label(self, monkeypatch, capsys, tmp_path): + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": True, "auth_store": "/a/auth.json"}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "xAI OAuth" in out + # The logged-in label must appear; the "not logged in" label must not + assert "✓" in out or "logged in" in out + assert "not logged in" not in out.split("xAI OAuth", 1)[1].split("\n")[0] + + def test_logged_in_shows_auth_store(self, monkeypatch, capsys, tmp_path): + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": True, "auth_store": "/home/u/.hermes/auth.json"}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "Auth file: /home/u/.hermes/auth.json" in out + + def test_logged_in_shows_last_refresh(self, monkeypatch, capsys, tmp_path): + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: { + "logged_in": True, + "auth_store": "/a/auth.json", + "last_refresh": "2026-05-17T10:00:00+00:00", + }, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "Refreshed:" in out + + def test_logged_in_does_not_show_error_line(self, monkeypatch, capsys, tmp_path): + """Error field must be suppressed when logged_in is True.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: { + "logged_in": True, + "auth_store": "/a/auth.json", + "error": "stale-error-must-not-appear", + }, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + xai_section = out.split("xAI OAuth", 1)[1] + assert "stale-error-must-not-appear" not in xai_section + + def test_no_auth_store_line_when_field_absent(self, monkeypatch, capsys, tmp_path): + """Auth file line must not appear when auth_store is missing.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": True}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + xai_section = out.split("xAI OAuth", 1)[1].split("◆", 1)[0] + assert "Auth file:" not in xai_section + + def test_no_refreshed_line_when_last_refresh_absent(self, monkeypatch, capsys, tmp_path): + """Refreshed line must not appear when last_refresh is not present.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": True, "auth_store": "/a/auth.json"}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + xai_section = out.split("xAI OAuth", 1)[1].split("◆", 1)[0] + assert "Refreshed:" not in xai_section + + # ------------------------------------------------------------------ + # Not-logged-in branch + # ------------------------------------------------------------------ + + def test_not_logged_in_shows_login_command(self, monkeypatch, capsys, tmp_path): + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": False, "error": "no credentials"}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "not logged in (run: hermes auth add xai-oauth)" in out + + def test_not_logged_in_shows_error(self, monkeypatch, capsys, tmp_path): + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": False, "error": "Token has expired"}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "Error: Token has expired" in out + + def test_not_logged_in_omits_error_line_when_error_absent(self, monkeypatch, capsys, tmp_path): + """No Error: line when not logged in but error key is missing.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: {"logged_in": False}, + raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + xai_section = out.split("xAI OAuth", 1)[1].split("◆", 1)[0] + assert "Error:" not in xai_section + + # ------------------------------------------------------------------ + # Resilience: import failure and runtime exception + # ------------------------------------------------------------------ + + def test_import_failure_does_not_crash_show_status(self, monkeypatch, capsys, tmp_path): + """show_status must complete even when get_xai_oauth_auth_status cannot be imported.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.delattr(auth_mod, "get_xai_oauth_auth_status", raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "◆ Auth Providers" in out + + def test_import_failure_does_not_break_other_oauth_providers(self, monkeypatch, capsys, tmp_path): + """Nous/Codex/MiniMax rows must still appear when xAI import fails.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_nous_auth_status", + lambda: {"logged_in": True}, raising=False) + monkeypatch.delattr(auth_mod, "get_xai_oauth_auth_status", raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "Nous Portal" in out + assert "MiniMax OAuth" in out + + def test_status_function_exception_does_not_crash(self, monkeypatch, capsys, tmp_path): + """show_status must not propagate an exception raised by get_xai_oauth_auth_status.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + + def _raises(): + raise RuntimeError("backend unreachable") + + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", _raises, raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "◆ Auth Providers" in out + + def test_status_function_returns_none_does_not_crash(self, monkeypatch, capsys, tmp_path): + """get_xai_oauth_auth_status returning None must be handled gracefully.""" + import hermes_cli.auth as auth_mod + status_mod = _base_xai_mocks(monkeypatch, tmp_path) + monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", + lambda: None, raising=False) + + status_mod.show_status(SimpleNamespace(all=False, deep=False)) + out = capsys.readouterr().out + + assert "xAI OAuth" in out + assert "not logged in (run: hermes auth add xai-oauth)" in out From d0f551b44e98c36e61aba31c5b2b65a564d0c3f8 Mon Sep 17 00:00:00 2001 From: EloquentBrush0x <283442588+EloquentBrush0x@users.noreply.github.com> Date: Sun, 17 May 2026 04:27:23 +0300 Subject: [PATCH 097/142] fix(doctor): show xAI OAuth login state in hermes doctor Auth Providers section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `hermes doctor` displayed OAuth status for Nous, Codex, Gemini, and MiniMax but silently omitted xAI OAuth, even though `get_xai_oauth_auth_status()` exists and the same information is already surfaced in `hermes status`. Add xAI OAuth as a *separate* try/except block so an import failure cannot silence the already-printed provider rows above it — consistent with the per-provider isolation introduced in the doctor fallback fix. Tests: - 9 new tests in TestDoctorXaiOAuthStatus covering: logged-in ok, not-logged-in warn, error line present/absent, import failure isolation, runtime exception and None-return safety. - 9 existing run_doctor helpers updated to mock get_xai_oauth_auth_status for deterministic output. --- hermes_cli/doctor.py | 14 +++ tests/hermes_cli/test_doctor.py | 177 ++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index a3d5764835f..6f036426fa5 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -823,6 +823,20 @@ def run_doctor(args): except Exception as e: check_warn("Auth provider status", f"(could not check: {e})") + # xAI OAuth — separate try/except so an import failure here cannot + # disrupt the already-printed Nous/Codex/Gemini/MiniMax rows above. + try: + from hermes_cli.auth import get_xai_oauth_auth_status + xai_oauth_status = get_xai_oauth_auth_status() or {} + if xai_oauth_status.get("logged_in"): + check_ok("xAI OAuth", "(logged in)") + else: + check_warn("xAI OAuth", "(not logged in)") + if xai_oauth_status.get("error"): + check_info(xai_oauth_status["error"]) + except Exception: + pass + if _safe_which("codex"): check_ok("codex CLI") else: diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index 4f9a9e93cba..a5b058fe452 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -320,6 +320,7 @@ class TestDoctorMemoryProviderSection: from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -426,6 +427,7 @@ def test_run_doctor_accepts_named_provider_from_providers_section(monkeypatch, t from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -463,6 +465,7 @@ def test_run_doctor_accepts_bare_custom_provider(monkeypatch, tmp_path): from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -510,6 +513,7 @@ def test_run_doctor_accepts_hermes_provider_ids_that_catalog_aliases( from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -556,6 +560,7 @@ def test_run_doctor_accepts_kimi_coding_cn_provider(monkeypatch, tmp_path): monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_auth_status", lambda provider: {"logged_in": True}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -594,6 +599,7 @@ def test_run_doctor_termux_does_not_mark_browser_available_without_agent_browser from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -633,6 +639,7 @@ def test_run_doctor_kimi_cn_env_is_detected_and_probe_is_null_safe(monkeypatch, from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except Exception: pass @@ -681,6 +688,7 @@ def test_run_doctor_dashscope_retries_china_endpoint_after_intl_unauthorized(mon from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except ImportError: pass @@ -739,6 +747,7 @@ def test_run_doctor_opencode_go_skips_invalid_models_probe(monkeypatch, tmp_path from hermes_cli import auth as _auth_mod monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", lambda: {}) except ImportError: pass @@ -1004,3 +1013,171 @@ class TestHasHealthyOauthFallbackForXai: monkeypatch.delitem(sys.modules, "hermes_cli.doctor", raising=False) from hermes_cli.doctor import _has_healthy_oauth_fallback_for_apikey_provider assert _has_healthy_oauth_fallback_for_apikey_provider("gemini") is True + + +# --------------------------------------------------------------------------- +# ◆ Auth Providers — xAI OAuth display in run_doctor() +# --------------------------------------------------------------------------- + + +class TestDoctorXaiOAuthStatus: + """The ◆ Auth Providers section must show xAI OAuth login state. + + xAI OAuth is checked in a *separate* try/except block so that an import + failure (or runtime exception) cannot silence the Nous / Codex / Gemini / + MiniMax rows that were already printed above it. + """ + + def _run(self, monkeypatch, tmp_path, *, xai_auth_fn) -> str: + """Run doctor with a controlled xAI auth callable; return stdout.""" + home = tmp_path / ".hermes" + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text("memory: {}\n", encoding="utf-8") + project = tmp_path / "project" + project.mkdir(exist_ok=True) + + monkeypatch.setattr(doctor_mod, "HERMES_HOME", home) + monkeypatch.setattr(doctor_mod, "PROJECT_ROOT", project) + monkeypatch.setattr(doctor_mod, "_DHH", str(home)) + + fake_model_tools = types.SimpleNamespace( + check_tool_availability=lambda *a, **kw: ([], []), + TOOLSET_REQUIREMENTS={}, + ) + monkeypatch.setitem(sys.modules, "model_tools", fake_model_tools) + + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_minimax_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_xai_oauth_auth_status", xai_auth_fn) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + doctor_mod.run_doctor(Namespace(fix=False)) + return buf.getvalue() + + def test_logged_in_shows_ok(self, monkeypatch, tmp_path): + out = self._run( + monkeypatch, tmp_path, + xai_auth_fn=lambda: {"logged_in": True}, + ) + assert "xAI OAuth" in out + assert "(logged in)" in out + + def test_not_logged_in_shows_warn(self, monkeypatch, tmp_path): + out = self._run( + monkeypatch, tmp_path, + xai_auth_fn=lambda: {"logged_in": False}, + ) + assert "xAI OAuth" in out + assert "(not logged in)" in out + + def test_error_shown_when_not_logged_in_and_error_present(self, monkeypatch, tmp_path): + out = self._run( + monkeypatch, tmp_path, + xai_auth_fn=lambda: {"logged_in": False, "error": "refresh token expired"}, + ) + assert "xAI OAuth" in out + assert "refresh token expired" in out + + def test_no_error_line_when_error_key_absent(self, monkeypatch, tmp_path): + out = self._run( + monkeypatch, tmp_path, + xai_auth_fn=lambda: {"logged_in": False}, + ) + assert "xAI OAuth" in out + # The check_info line is only emitted when the "error" key is present. + # Pick a token that would appear in no ordinary doctor output. + assert "refresh token expired" not in out + + def test_logged_in_does_not_emit_not_logged_in_on_xai_line(self, monkeypatch, tmp_path): + out = self._run( + monkeypatch, tmp_path, + xai_auth_fn=lambda: {"logged_in": True}, + ) + assert "xAI OAuth" in out + # The xAI OAuth line itself must say "(logged in)", not "(not logged in)". + xai_line = next(l for l in out.splitlines() if "xAI OAuth" in l) + assert "(logged in)" in xai_line + assert "(not logged in)" not in xai_line + + def test_import_failure_does_not_crash_doctor(self, monkeypatch, tmp_path): + """Doctor must not crash when get_xai_oauth_auth_status cannot be imported.""" + home = tmp_path / ".hermes" + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text("memory: {}\n", encoding="utf-8") + project = tmp_path / "project" + project.mkdir(exist_ok=True) + + monkeypatch.setattr(doctor_mod, "HERMES_HOME", home) + monkeypatch.setattr(doctor_mod, "PROJECT_ROOT", project) + monkeypatch.setattr(doctor_mod, "_DHH", str(home)) + + fake_model_tools = types.SimpleNamespace( + check_tool_availability=lambda *a, **kw: ([], []), + TOOLSET_REQUIREMENTS={}, + ) + monkeypatch.setitem(sys.modules, "model_tools", fake_model_tools) + + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_minimax_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.delattr(_auth_mod, "get_xai_oauth_auth_status", raising=False) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + doctor_mod.run_doctor(Namespace(fix=False)) + out = buf.getvalue() + # The ◆ Auth Providers header must still appear — other providers unaffected. + assert "Auth Providers" in out + + def test_import_failure_does_not_affect_other_providers(self, monkeypatch, tmp_path): + """Nous / Codex / Gemini / MiniMax rows must survive an xAI import failure.""" + home = tmp_path / ".hermes" + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text("memory: {}\n", encoding="utf-8") + project = tmp_path / "project" + project.mkdir(exist_ok=True) + + monkeypatch.setattr(doctor_mod, "HERMES_HOME", home) + monkeypatch.setattr(doctor_mod, "PROJECT_ROOT", project) + monkeypatch.setattr(doctor_mod, "_DHH", str(home)) + + fake_model_tools = types.SimpleNamespace( + check_tool_availability=lambda *a, **kw: ([], []), + TOOLSET_REQUIREMENTS={}, + ) + monkeypatch.setitem(sys.modules, "model_tools", fake_model_tools) + + from hermes_cli import auth as _auth_mod + monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {"logged_in": True}) + monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.setattr(_auth_mod, "get_minimax_oauth_auth_status", lambda: {"logged_in": False}) + monkeypatch.delattr(_auth_mod, "get_xai_oauth_auth_status", raising=False) + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + doctor_mod.run_doctor(Namespace(fix=False)) + out = buf.getvalue() + assert "Nous Portal auth" in out + assert "logged in" in out + + def test_function_raises_does_not_crash_doctor(self, monkeypatch, tmp_path): + """A runtime exception from get_xai_oauth_auth_status must be swallowed.""" + def _raise(): + raise RuntimeError("simulated xAI status failure") + + out = self._run(monkeypatch, tmp_path, xai_auth_fn=_raise) + assert "Auth Providers" in out + + def test_function_returns_none_does_not_crash_doctor(self, monkeypatch, tmp_path): + """None return is normalised to {} via `or {}` — must not AttributeError.""" + out = self._run(monkeypatch, tmp_path, xai_auth_fn=lambda: None) + # None → {} → logged_in falsy → shows not-logged-in warn + assert "xAI OAuth" in out + assert "(not logged in)" in out From 37286a5bcd4fe2b43ea365140e71abb0add05fbb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 11:37:09 -0700 Subject: [PATCH 098/142] chore(release): map QuenVix, Mind-Dragon, soynchux emails for Tier 4 salvage --- scripts/release.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index 31bf7020ce3..c0d743bef9d 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1150,6 +1150,11 @@ AUTHOR_MAP = { "alaamohanad169-ship-it@users.noreply.github.com": "alaamohanad169-ship-it", # PR #26036 (telegram typing after send) "vigo@hermes": "hawknewton", # PR #26294 (bedrock boto3 lazy_deps) "211668+hawknewton@users.noreply.github.com": "hawknewton", + "quenvix00@gmail.com": "QuenVix", # PR #26761/26772 salvage + "164776164+QuenVix@users.noreply.github.com": "QuenVix", + "262945885+Mind-Dragon@users.noreply.github.com": "Mind-Dragon", # PR #26966 salvage + "soynchuux@gmail.com": "soynchux", # PR #27060 salvage + "209694554+soynchux@users.noreply.github.com": "soynchux", } From d5a0815c3dd9e4c9ca2fd37d0f51f5d1cc0b1e3e Mon Sep 17 00:00:00 2001 From: QuenVix <164776164+QuenVix@users.noreply.github.com> Date: Sat, 16 May 2026 08:00:48 +0300 Subject: [PATCH 099/142] fix(transports): use monotonic deadlines in codex app-server turn loop --- agent/transports/codex_app_server_session.py | 10 ++-- .../test_codex_app_server_session.py | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/agent/transports/codex_app_server_session.py b/agent/transports/codex_app_server_session.py index a72599ae719..d9ee92dfbf5 100644 --- a/agent/transports/codex_app_server_session.py +++ b/agent/transports/codex_app_server_session.py @@ -404,7 +404,7 @@ class CodexAppServerSession: return result result.turn_id = (ts.get("turn") or {}).get("id") - deadline = time.time() + turn_timeout + deadline = time.monotonic() + turn_timeout turn_complete = False # Post-tool watchdog state. last_tool_completion_at is set whenever # a tool-shaped item completes; if no further notification arrives @@ -412,7 +412,7 @@ class CodexAppServerSession: # fast-fail and retire the session. last_tool_completion_at: Optional[float] = None - while time.time() < deadline and not turn_complete: + while time.monotonic() < deadline and not turn_complete: if self._interrupt_event.is_set(): self._issue_interrupt(result.turn_id) result.interrupted = True @@ -440,7 +440,7 @@ class CodexAppServerSession: # up on this turn instead of waiting for the outer deadline. if ( last_tool_completion_at is not None - and (time.time() - last_tool_completion_at) + and (time.monotonic() - last_tool_completion_at) > post_tool_quiet_timeout ): self._issue_interrupt(result.turn_id) @@ -471,7 +471,7 @@ class CodexAppServerSession: result.projected_messages.extend(proj.messages) if proj.is_tool_iteration: result.tool_iterations += 1 - last_tool_completion_at = time.time() + last_tool_completion_at = time.monotonic() if proj.final_text is not None: result.final_text = proj.final_text if _has_turn_aborted_marker(proj.final_text): @@ -514,7 +514,7 @@ class CodexAppServerSession: result.tool_iterations += 1 # Arm/refresh the post-tool quiet watchdog whenever a # tool-shaped item completes. - last_tool_completion_at = time.time() + last_tool_completion_at = time.monotonic() else: # Any non-tool projected activity (assistant message, # status update, etc.) means codex is still producing diff --git a/tests/agent/transports/test_codex_app_server_session.py b/tests/agent/transports/test_codex_app_server_session.py index f51996dd067..b192d64e1c8 100644 --- a/tests/agent/transports/test_codex_app_server_session.py +++ b/tests/agent/transports/test_codex_app_server_session.py @@ -9,10 +9,12 @@ from __future__ import annotations import threading import time +from unittest.mock import patch from typing import Any, Optional import pytest +import agent.transports.codex_app_server_session as session_mod from agent.transports.codex_app_server_session import ( CodexAppServerSession, TurnResult, @@ -344,6 +346,23 @@ class TestRunTurn: assert r.interrupted is True assert r.error and "timed out" in r.error + def test_deadline_uses_monotonic_clock(self): + client = FakeClient() + s = make_session(client) + monotonic_values = iter([1000.0, 999.0, 999.0, 1001.0]) + with patch.object( + session_mod.time, + "monotonic", + side_effect=lambda: next(monotonic_values), + ): + r = s.run_turn( + "never finishes", + turn_timeout=0.1, + notification_poll_timeout=0.0, + ) + assert r.interrupted is True + assert r.error and "timed out" in r.error + def test_failed_turn_records_error_from_turn_completed(self): client = FakeClient() client.queue_notification( @@ -666,6 +685,35 @@ class TestSessionRetirement: # Confirm we issued turn/interrupt to free codex compute assert any(method == "turn/interrupt" for (method, _) in client.requests) + def test_post_tool_watchdog_uses_monotonic_clock(self): + client = FakeClient() + client.queue_notification( + "item/completed", + item={ + "type": "commandExecution", "id": "ex1", + "command": "echo hi", "cwd": "/tmp", + "status": "completed", "aggregatedOutput": "hi", + "exitCode": 0, "commandActions": [], + }, + threadId="t", turnId="tu1", + ) + s = make_session(client) + monotonic_values = iter([1000.0, 999.0, 999.0, 999.0, 1000.2]) + with patch.object( + session_mod.time, + "monotonic", + side_effect=lambda: next(monotonic_values), + ): + r = s.run_turn( + "tool then silence", + turn_timeout=5.0, + notification_poll_timeout=0.0, + post_tool_quiet_timeout=0.15, + ) + assert r.interrupted is True + assert r.should_retire is True + assert r.error and "silent" in r.error + def test_post_tool_watchdog_resets_on_further_activity(self): """A tool completion followed by an agent message should NOT trip the watchdog — further activity = codex still alive.""" From 2f28b60a474c880367be612c682f52b8ca9dbb4d Mon Sep 17 00:00:00 2001 From: QuenVix <164776164+QuenVix@users.noreply.github.com> Date: Sat, 16 May 2026 08:26:41 +0300 Subject: [PATCH 100/142] fix(send_message): preserve Slack and Matrix thread targets resolved from channel directory --- tests/tools/test_send_message_tool.py | 96 ++++++++++++++++++++++++++- tools/send_message_tool.py | 10 +++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_send_message_tool.py b/tests/tools/test_send_message_tool.py index fa810eb5c54..dac476749fd 100644 --- a/tests/tools/test_send_message_tool.py +++ b/tests/tools/test_send_message_tool.py @@ -182,6 +182,81 @@ class TestSendMessageTool: force_document=False, ) + def test_resolved_slack_thread_name_preserves_thread_id(self): + slack_cfg = SimpleNamespace(enabled=True, token="xoxb-test", extra={}) + config = SimpleNamespace( + platforms={Platform.SLACK: slack_cfg}, + get_home_channel=lambda _platform: None, + ) + + with patch("gateway.config.load_gateway_config", return_value=config), \ + patch("tools.interrupt.is_interrupted", return_value=False), \ + patch("gateway.channel_directory.resolve_channel_name", return_value="C123ABCDEF:171.000001"), \ + patch("model_tools._run_async", side_effect=_run_async_immediately), \ + patch("tools.send_message_tool._send_to_platform", new=AsyncMock(return_value={"success": True})) as send_mock, \ + patch("gateway.mirror.mirror_to_session", return_value=True): + result = json.loads( + send_message_tool( + { + "action": "send", + "target": "slack:ops / topic 171.000001", + "message": "hello", + } + ) + ) + + assert result["success"] is True + send_mock.assert_awaited_once_with( + Platform.SLACK, + slack_cfg, + "C123ABCDEF", + "hello", + thread_id="171.000001", + media_files=[], + force_document=False, + ) + + def test_resolved_matrix_thread_name_preserves_thread_id(self): + matrix_cfg = SimpleNamespace( + enabled=True, + token="tok", + extra={"homeserver": "https://matrix.example.com"}, + ) + config = SimpleNamespace( + platforms={Platform.MATRIX: matrix_cfg}, + get_home_channel=lambda _platform: None, + ) + + with patch("gateway.config.load_gateway_config", return_value=config), \ + patch("tools.interrupt.is_interrupted", return_value=False), \ + patch( + "gateway.channel_directory.resolve_channel_name", + return_value="!roomid:matrix.example.org:$thread123:matrix.example.org", + ), \ + patch("model_tools._run_async", side_effect=_run_async_immediately), \ + patch("tools.send_message_tool._send_to_platform", new=AsyncMock(return_value={"success": True})) as send_mock, \ + patch("gateway.mirror.mirror_to_session", return_value=True): + result = json.loads( + send_message_tool( + { + "action": "send", + "target": "matrix:Ops / topic $thread123", + "message": "hello", + } + ) + ) + + assert result["success"] is True + send_mock.assert_awaited_once_with( + Platform.MATRIX, + matrix_cfg, + "!roomid:matrix.example.org", + "hello", + thread_id="$thread123:matrix.example.org", + media_files=[], + force_document=False, + ) + def test_mirror_receives_current_session_user_id(self): config, _telegram_cfg = _make_config() @@ -503,9 +578,8 @@ class TestSendToPlatformChunking: assert all(call == [] for call in sent_calls[:-1]) assert sent_calls[-1] == media - def test_matrix_media_uses_native_adapter_helper(self): - - doc_path = Path("/tmp/test-send-message-matrix.pdf") + def test_matrix_media_uses_native_adapter_helper(self, tmp_path): + doc_path = tmp_path / "test-send-message-matrix.pdf" doc_path.write_bytes(b"%PDF-1.4 test") try: @@ -847,6 +921,16 @@ class TestParseTargetRefDiscord: class TestParseTargetRefMatrix: """_parse_target_ref correctly handles Matrix room IDs and user MXIDs.""" + def test_matrix_thread_target_is_explicit(self): + """Session-derived Matrix thread targets round-trip as room + event id.""" + chat_id, thread_id, is_explicit = _parse_target_ref( + "matrix", + "!HLOQwxYGgFPMPJUSNR:matrix.org:$thread123:matrix.org", + ) + assert chat_id == "!HLOQwxYGgFPMPJUSNR:matrix.org" + assert thread_id == "$thread123:matrix.org" + assert is_explicit is True + def test_matrix_room_id_is_explicit(self): """Matrix room IDs (!) are recognized as explicit targets.""" chat_id, thread_id, is_explicit = _parse_target_ref("matrix", "!HLOQwxYGgFPMPJUSNR:matrix.org") @@ -919,6 +1003,12 @@ class TestParseTargetRefE164: class TestParseTargetRefSlack: """_parse_target_ref recognizes Slack channel/user IDs as explicit.""" + def test_thread_target_is_explicit(self): + chat_id, thread_id, is_explicit = _parse_target_ref("slack", "C0B0QV5434G:171.000001") + assert chat_id == "C0B0QV5434G" + assert thread_id == "171.000001" + assert is_explicit is True + def test_public_channel_id_is_explicit(self): chat_id, thread_id, is_explicit = _parse_target_ref("slack", "C0B0QV5434G") assert chat_id == "C0B0QV5434G" diff --git a/tools/send_message_tool.py b/tools/send_message_tool.py index d5b2c0c782c..bfe1a630707 100644 --- a/tools/send_message_tool.py +++ b/tools/send_message_tool.py @@ -28,6 +28,8 @@ _FEISHU_TARGET_RE = re.compile(r"^\s*((?:oc|ou|on|chat|open)_[-A-Za-z0-9]+)(?::( # conversations.open to obtain a D... ID. Without this gate, Slack IDs fall # through to channel-name resolution, which only matches by name and fails. _SLACK_TARGET_RE = re.compile(r"^\s*([CGD][A-Z0-9]{8,})\s*$") +# Session-derived Slack thread targets use ":". +_SLACK_THREAD_TARGET_RE = re.compile(r"^\s*([CGD][A-Z0-9]{8,}):([^\s:]+)\s*$") _WEIXIN_TARGET_RE = re.compile(r"^\s*((?:wxid|gh|v\d+|wm|wb)_[A-Za-z0-9_-]+|[A-Za-z0-9._-]+@chatroom|filehelper)\s*$") _YUANBAO_TARGET_RE = re.compile(r"^\s*((?:group|direct):[^:]+)\s*$") # Discord snowflake IDs are numeric, same regex pattern as Telegram topic targets. @@ -330,9 +332,17 @@ def _parse_target_ref(platform_name: str, target_ref: str): if match: return match.group(1), match.group(2), True if platform_name == "slack": + match = _SLACK_THREAD_TARGET_RE.fullmatch(target_ref) + if match: + return match.group(1), match.group(2), True match = _SLACK_TARGET_RE.fullmatch(target_ref) if match: return match.group(1), None, True + if platform_name == "matrix": + trimmed = target_ref.strip() + split_idx = trimmed.rfind(":$") + if split_idx > 0: + return trimmed[:split_idx], trimmed[split_idx + 1 :], True if platform_name == "weixin": match = _WEIXIN_TARGET_RE.fullmatch(target_ref) if match: From 55d6a1636bb1f38b01b708582c527b91cc9fe578 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 11:36:15 -0700 Subject: [PATCH 101/142] fix(agent): honor provider timeout config in streaming API calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #25249 (and supersedes PR #25260) in spirit. Two bugs in the streaming chat-completions path caused provider timeout configuration to be silently ignored: 1. Hardcoded connect/pool timeout. The httpx.Timeout for streaming calls used hardcoded connect=30.0 and pool=30.0 regardless of the user's providers..request_timeout_seconds config. If the custom provider (e.g. Ollama) was unreachable, the call always waited exactly 30s before failing, ignoring any configured timeout. Fix: use min(_base_timeout, 60.0) for connect and pool when a provider timeout is configured, falling back to 30.0 otherwise. The 60s cap addresses review feedback (TCP handshake shouldn't wait the inference timeout — connect/pool cover the connection layer, not model latency). 2. Streaming stale-stream detector ignored provider config. The stale detector read only HERMES_STREAM_STALE_TIMEOUT (env default 180s). The providers..stale_timeout_seconds key (correctly used in the non-streaming path) was never consulted. Fix: check get_provider_stale_timeout(provider, model) first, then fall back to the env var. Aligns the streaming path with the non-streaming path's priority chain (config > env > default). Salvage shape diverged from PR #25260: the function moved to agent/chat_completion_helpers.py and the contributor's two commits (initial fix + 60s-cap review follow-up) are squashed into one final commit applied at the new location. Original diagnosis, fix shape, AND the 60s-cap review response from @zccyman in PR #25260; credited via Co-authored-by. Co-authored-by: zccyman <16263913+zccyman@users.noreply.github.com> --- agent/chat_completion_helpers.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 1bf1ebc651e..e536db95eb1 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -33,7 +33,7 @@ from types import SimpleNamespace from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse, parse_qs, urlunparse -from hermes_cli.timeouts import get_provider_request_timeout +from hermes_cli.timeouts import get_provider_request_timeout, get_provider_stale_timeout from agent.error_classifier import classify_api_error, FailoverReason from agent.model_metadata import is_local_endpoint from agent.message_sanitization import ( @@ -1272,15 +1272,18 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= "Local provider detected (%s) — stream read timeout raised to %.0fs", agent.base_url, _stream_read_timeout, ) + # Cap connect/pool at 60s even when provider timeout is higher. + # connect/pool cover TCP handshake, not model inference. + _conn_cap = min(_base_timeout, 60.0) if _provider_timeout_cfg is not None else 30.0 stream_kwargs = { **api_kwargs, "stream": True, "stream_options": {"include_usage": True}, "timeout": _httpx.Timeout( - connect=30.0, + connect=_conn_cap, read=_stream_read_timeout, write=_base_timeout, - pool=30.0, + pool=_conn_cap, ), } request_client_holder["client"] = agent._create_request_openai_client( @@ -1868,7 +1871,12 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if request_client is not None: agent._close_request_openai_client(request_client, reason="stream_request_complete") - _stream_stale_timeout_base = float(os.getenv("HERMES_STREAM_STALE_TIMEOUT", 180.0)) + # Provider-configured stale timeout takes priority over env default. + _cfg_stale = get_provider_stale_timeout(agent.provider, agent.model) + if _cfg_stale is not None: + _stream_stale_timeout_base = _cfg_stale + else: + _stream_stale_timeout_base = float(os.getenv("HERMES_STREAM_STALE_TIMEOUT", 180.0)) # Local providers (Ollama, oMLX, llama-cpp) can take 300+ seconds # for prefill on large contexts. Disable the stale detector unless # the user explicitly set HERMES_STREAM_STALE_TIMEOUT. From 4afd479f51631ea39f8403df6b1e0467fc81c466 Mon Sep 17 00:00:00 2001 From: bird <6666242+bird@users.noreply.github.com> Date: Wed, 13 May 2026 16:06:06 -0400 Subject: [PATCH 102/142] fix(gateway): use service restart path in Docker/Podman containers The /restart command used a detached subprocess approach to restart the gateway. In Docker, when the gateway process exits, tini (PID 1) also exits, causing Docker to stop the container and kill the detached helper before it can restart the gateway. This made /restart effectively a /shutdown in containerized deployments. Detect Docker (/.dockerenv) and Podman (/run/.containerenv) containers and use the service restart path (exit code 75) instead, letting the container restart policy handle the actual restart. Note: requires restart policy that restarts on non-zero exit (e.g. unless-stopped or on-failure). --- gateway/run.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index db7066281c3..a0ab84e850d 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -8971,13 +8971,15 @@ class GatewayRunner: logger.debug("Failed to write restart dedup marker: %s", e) active_agents = self._running_agent_count() - # When running under a service manager (systemd/launchd), use the - # service restart path: exit with code 75 so the service manager - # restarts us. The detached subprocess approach (setsid + bash) - # doesn't work under systemd because KillMode=mixed kills all - # processes in the cgroup, including the detached helper. + # When running under a service manager (systemd/launchd) or inside a + # Docker/Podman container, use the service restart path: exit with + # code 75 so the service manager / container restart policy restarts + # us. The detached subprocess approach (setsid + bash) doesn't work + # under systemd (KillMode=mixed kills the cgroup) or Docker (tini + # exits when the gateway dies, taking the detached helper with it). _under_service = bool(os.environ.get("INVOCATION_ID")) # systemd sets this - if _under_service: + _in_container = os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv") + if _under_service or _in_container: self.request_restart(detached=False, via_service=True) else: self.request_restart(detached=True, via_service=False) From 714b3b2bd885c070d6404391b390fe349bf6cbf6 Mon Sep 17 00:00:00 2001 From: davidcampbelldc <165905879+davidcampbelldc@users.noreply.github.com> Date: Sun, 17 May 2026 11:36:29 -0700 Subject: [PATCH 103/142] fix(web_server): pass proxy_headers=False to uvicorn.run so the dashboard's loopback gate sees the real connection peer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_ws_client_is_allowed()` enforces a loopback-only client check on every dashboard WebSocket upgrade (`/api/ws`, `/api/events`, `/api/pty`, `/api/pub`): def _ws_client_is_allowed(ws): if _is_public_bind(): return True client_host = ws.client.host if ws.client else "" if not client_host: return True return client_host in _LOOPBACK_HOSTS The intent is: when bound to 127.0.0.1, only accept WS upgrades from loopback peers. Public bind (--insecure) trades that for token-only. However, `uvicorn.run(app, host=host, port=port, log_level="warning")` omits `proxy_headers`. In modern uvicorn (>= 0.20) `proxy_headers` defaults to True and `forwarded_allow_ips` defaults to "127.0.0.1". With those defaults, any reverse proxy connecting from loopback (nginx, in-cluster proxy, Cloudflare Tunnel sidecar in HTTP mode, K8s ingress-nginx) causes uvicorn to rewrite `ws.client.host` from the request's `X-Forwarded-For` header. So the gate sees the original client's IP (a public address) instead of the loopback peer, returns False, and closes every browser WS with code=4403 (surfaces as HTTP 403 to the proxy). Passing `proxy_headers=False` keeps the loopback gate's view of `ws.client.host` at the immediate transport peer (the proxy on 127.0.0.1), which is exactly what the gate is designed to check. The bug is invisible in dev (no proxy → no XFF → ws.client.host stays loopback). It surfaces in proxied production: dashboard chat tab opens, events feed banner shows "disconnected — tool calls may not appear", all WS endpoints return 403. Reproduces with: curl -i -H "Connection: Upgrade" -H "Upgrade: websocket" \ -H "Sec-WebSocket-Version: 13" -H "Sec-WebSocket-Key: ..." \ -H "X-Forwarded-For: 1.2.3.4" \ "http://127.0.0.1:9119/api/ws?token=\$TOKEN" # Before: HTTP/1.1 403 Forbidden # After: HTTP/1.1 101 Switching Protocols Without the XFF header, both behave the same (101) — confirming the single-variable trigger. Discovered while diagnosing why the Hermes dashboard at mandy.loadmagic.ai (behind nginx + Cloudflare Tunnel + CF Access) refused all browser WS upgrades despite Access app config matching a known-working sibling deployment (Simone, which doesn't have nginx in the path). --- hermes_cli/web_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index bdb24554f87..8a1e4aca2e1 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -4434,4 +4434,7 @@ def start_server( ) print(f" Hermes Web UI → http://{host}:{port}") - uvicorn.run(app, host=host, port=port, log_level="warning") + # proxy_headers=False so _ws_client_is_allowed sees the real connection peer + # rather than X-Forwarded-For's rewritten value (which would defeat the + # loopback gate when behind a reverse proxy). + uvicorn.run(app, host=host, port=port, log_level="warning", proxy_headers=False) From 74031e1e2aab77881c8e1eddb5f1766b47dfcfdc Mon Sep 17 00:00:00 2001 From: wesleysimplicio <6108320+wesleysimplicio@users.noreply.github.com> Date: Sun, 17 May 2026 11:36:29 -0700 Subject: [PATCH 104/142] fix(dashboard): respect HERMES_BASE_PATH in WebSocket URLs (#25547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the dashboard is reverse-proxied under a path prefix (`X-Forwarded-Prefix: /dashboard`), the SPA already routes its `/api/...` REST traffic through `HERMES_BASE_PATH` via `web/src/lib/api.ts`. Three WebSocket URLs constructed elsewhere were still hardcoded to root `/api/...` and so opened `wss://host/api/...` instead of `wss://host/dashboard/api/...`, forcing operators to forward selected root API/WS paths through the reverse proxy as a workaround (see issue #25547). Add `HERMES_BASE_PATH` between `host` and `/api/...` in the three constructed WebSocket URLs: - `web/src/pages/ChatPage.tsx` — PTY WebSocket - `web/src/components/ChatSidebar.tsx` — events subscriber - `web/src/lib/gatewayClient.ts` — JSON-RPC gateway WebSocket When the dashboard is served at root, `HERMES_BASE_PATH === """ and the URLs are bit-for-bit identical to before. Under a prefix, the WebSocket connections now go through the same proxy path the REST calls already use. Note: bundled dashboard plugins (kanban, hermes-achievements) embed `"/api/plugins/..."` in their compiled `dist/index.js` and remain out of scope here — those need source-side fixes per plugin. Fixes #25547. --- web/src/components/ChatSidebar.tsx | 3 ++- web/src/lib/gatewayClient.ts | 4 +++- web/src/pages/ChatPage.tsx | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/web/src/components/ChatSidebar.tsx b/web/src/components/ChatSidebar.tsx index 38f1cf80abd..c311673fafc 100644 --- a/web/src/components/ChatSidebar.tsx +++ b/web/src/components/ChatSidebar.tsx @@ -30,6 +30,7 @@ import { Card } from "@/components/ui/card"; import { ModelPickerDialog } from "@/components/ModelPickerDialog"; import { ToolCall, type ToolEntry } from "@/components/ToolCall"; import { GatewayClient, type ConnectionState } from "@/lib/gatewayClient"; +import { HERMES_BASE_PATH } from "@/lib/api"; import { cn } from "@/lib/utils"; import { AlertCircle, ChevronDown, RefreshCw } from "lucide-react"; @@ -160,7 +161,7 @@ export function ChatSidebar({ channel, className }: ChatSidebarProps) { const proto = window.location.protocol === "https:" ? "wss:" : "ws:"; const qs = new URLSearchParams({ token, channel }); const ws = new WebSocket( - `${proto}//${window.location.host}/api/events?${qs.toString()}`, + `${proto}//${window.location.host}${HERMES_BASE_PATH}/api/events?${qs.toString()}`, ); // `unmounting` suppresses the banner during cleanup — `ws.close()` diff --git a/web/src/lib/gatewayClient.ts b/web/src/lib/gatewayClient.ts index fa58841ce18..9092ef2d32d 100644 --- a/web/src/lib/gatewayClient.ts +++ b/web/src/lib/gatewayClient.ts @@ -13,6 +13,8 @@ * await gw.request("prompt.submit", { session_id, text: "hi" }) */ +import { HERMES_BASE_PATH } from "@/lib/api"; + export type GatewayEventName = | "gateway.ready" | "session.info" @@ -117,7 +119,7 @@ export class GatewayClient { const scheme = location.protocol === "https:" ? "wss:" : "ws:"; const ws = new WebSocket( - `${scheme}//${location.host}/api/ws?token=${encodeURIComponent(resolved)}`, + `${scheme}//${location.host}${HERMES_BASE_PATH}/api/ws?token=${encodeURIComponent(resolved)}`, ); this.ws = ws; diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index 6fd32fa43fc..3e3c2e3268b 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -24,6 +24,7 @@ import { Terminal } from "@xterm/xterm"; import "@xterm/xterm/css/xterm.css"; import { Button } from "@nous-research/ui/ui/components/button"; import { Typography } from "@/components/NouiTypography"; +import { HERMES_BASE_PATH } from "@/lib/api"; import { cn } from "@/lib/utils"; import { Copy, PanelRight, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; @@ -44,7 +45,7 @@ function buildWsUrl( const proto = window.location.protocol === "https:" ? "wss:" : "ws:"; const qs = new URLSearchParams({ token, channel }); if (resume) qs.set("resume", resume); - return `${proto}//${window.location.host}/api/pty?${qs.toString()}`; + return `${proto}//${window.location.host}${HERMES_BASE_PATH}/api/pty?${qs.toString()}`; } // Channel id ties this chat tab's PTY child (publisher) to its sidebar From 3f01e9493c4105bc52a9366a833fb17bb155527d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 11:37:00 -0700 Subject: [PATCH 105/142] chore(release): AUTHOR_MAP entries for batch salvage group 6 contributors Final LHF run group. Adds release-note attribution mappings for: - @bird (PR #25219) - @davidcampbelldc (PR #26834) (zccyman, wesleysimplicio already mapped from prior groups.) --- scripts/release.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index c0d743bef9d..fa1ed739d48 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1155,6 +1155,10 @@ AUTHOR_MAP = { "262945885+Mind-Dragon@users.noreply.github.com": "Mind-Dragon", # PR #26966 salvage "soynchuux@gmail.com": "soynchux", # PR #27060 salvage "209694554+soynchux@users.noreply.github.com": "soynchux", + # batch salvage (May 2026 LHF run, group 6 — final) + "6666242+bird@users.noreply.github.com": "bird", # PR #25219 (gateway docker exit-75 restart) + "david@loadmagic.ai": "davidcampbelldc", # PR #26834 (web_server proxy_headers=False) + "165905879+davidcampbelldc@users.noreply.github.com": "davidcampbelldc", } From 84667cbc21dc09c4e53793eb31d3b7f2c4fd9d0f Mon Sep 17 00:00:00 2001 From: Mind-Dragon <262945885+Mind-Dragon@users.noreply.github.com> Date: Sat, 16 May 2026 16:28:40 +0200 Subject: [PATCH 106/142] fix(delegation): preserve configured_provider name when runtime returns 'custom' Named custom providers (e.g. crof.ai) resolve to provider='custom' at the runtime level, causing subagents to lose their intended provider identity. On retry/fallback, resolve_provider_client('custom', model=...) searches all providers advertising that model and picks non-deterministically, routing to Z.AI or Bailian instead of the configured target. The fix preserves configured_provider when runtime['provider'] == 'custom', restoring the original provider name so routing stays correct through retries. Adds a named constant _RUNTIME_PROVIDER_CUSTOM instead of a magic string. Adds three regression tests: - test_named_custom_provider_preserves_provider_name: the #26954 case - test_standard_provider_not_overwritten_by_configured_name: openrouter/nous must still return their own identity, not the configured name - test_custom_provider_with_empty_configured_provider_falls_back_to_runtime: empty provider triggers the early-return None path as before --- tests/tools/test_delegate.py | 65 ++++++++++++++++++++++++++++++++++++ tools/delegate_tool.py | 7 +++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index 684f24f5da8..4a40f82b9aa 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -1014,6 +1014,71 @@ class TestDelegationCredentialResolution(unittest.TestCase): self.assertIsNone(creds["model"]) self.assertIsNone(creds["provider"]) + @patch("hermes_cli.runtime_provider.resolve_runtime_provider") + def test_named_custom_provider_preserves_provider_name(self, mock_resolve): + """Named custom provider (e.g. crof.ai) resolves to 'custom' at runtime level + but the subagent must retain the original provider identity so that + resolve_provider_client routes to the correct endpoint on retry/fallback. + Regression test for #26954. + """ + mock_resolve.return_value = { + "provider": "custom", # runtime marks it as "custom" type + "model": "deepseek-v4-pro-CEER", + "base_url": "https://api.crof.ai/v1", + "api_key": "crof-key-abc", + "api_mode": "chat_completions", + } + parent = _make_mock_parent(depth=0) + cfg = {"model": "deepseek-v4-pro-CEER", "provider": "crof.ai"} + creds = _resolve_delegation_credentials(cfg, parent) + # The key assertion: subagent must keep "crof.ai", NOT "custom" + self.assertEqual(creds["provider"], "crof.ai") + self.assertEqual(creds["model"], "deepseek-v4-pro-CEER") + self.assertEqual(creds["base_url"], "https://api.crof.ai/v1") + self.assertEqual(creds["api_key"], "crof-key-abc") + # Verify resolve_runtime_provider was called with the configured name + mock_resolve.assert_called_once_with( + requested="crof.ai", target_model="deepseek-v4-pro-CEER" + ) + + @patch("hermes_cli.runtime_provider.resolve_runtime_provider") + def test_standard_provider_not_overwritten_by_configured_name(self, mock_resolve): + """Standard (non-custom) providers must still return runtime identity, + not the configured name, to preserve existing behaviour for openrouter, + nous, etc. + """ + mock_resolve.return_value = { + "provider": "openrouter", + "model": "anthropic/claude-sonnet-4", + "base_url": "https://openrouter.ai/api/v1", + "api_key": "or-key-xyz", + "api_mode": "chat_completions", + } + parent = _make_mock_parent(depth=0) + cfg = {"model": "anthropic/claude-sonnet-4", "provider": "openrouter"} + creds = _resolve_delegation_credentials(cfg, parent) + # Standard provider returns its own name, not "custom" + self.assertEqual(creds["provider"], "openrouter") + + @patch("hermes_cli.runtime_provider.resolve_runtime_provider") + def test_custom_provider_with_empty_configured_provider_falls_back_to_runtime(self, mock_resolve): + """When configured_provider is empty/None, the early return kicks in and + we return provider=None regardless of what runtime resolved. The runtime + path is only reached when configured_provider is a non-empty string. + """ + mock_resolve.return_value = { + "provider": "custom", + "model": "some-model", + "base_url": "https://fallback.example.com/v1", + "api_key": "key-fallback", + "api_mode": "chat_completions", + } + parent = _make_mock_parent(depth=0) + cfg = {"model": "some-model", "provider": ""} + creds = _resolve_delegation_credentials(cfg, parent) + # Empty provider → early return with None (child inherits parent) + self.assertIsNone(creds["provider"]) + class TestDelegationProviderIntegration(unittest.TestCase): """Integration tests: delegation config → _run_single_child → AIAgent construction.""" diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index e9ad32e0d3a..86dcd0715cc 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -31,6 +31,11 @@ from concurrent.futures import ( from typing import Any, Dict, List, Optional from toolsets import TOOLSETS + +# Sentinel value used by the runtime provider system for providers that are +# not natively known (named custom providers, third-party aggregators, etc.). +# Must match hermes_cli.runtime_provider.RUNTIME_PROVIDER_TYPE_CUSTOM. +_RUNTIME_PROVIDER_CUSTOM = "custom" from tools import file_state from tools.terminal_tool import set_approval_callback as _set_subagent_approval_cb from utils import base_url_hostname, is_truthy_value @@ -2442,7 +2447,7 @@ def _resolve_delegation_credentials(cfg: dict, parent_agent) -> dict: return { "model": configured_model or runtime.get("model") or None, - "provider": runtime.get("provider"), + "provider": configured_provider if runtime.get("provider") == _RUNTIME_PROVIDER_CUSTOM else runtime.get("provider"), "base_url": runtime.get("base_url"), "api_key": api_key, "api_mode": runtime.get("api_mode"), From 874dad5cc1886ed79cddbc4d12c8cc62f8f3db5e Mon Sep 17 00:00:00 2001 From: Mind-Dragon <262945885+Mind-Dragon@users.noreply.github.com> Date: Sat, 16 May 2026 16:49:28 +0200 Subject: [PATCH 107/142] test(delegation): add regression test for runtime missing 'provider' key Addresses reviewer feedback: when resolve_runtime_provider returns a dict without the 'provider' key, the result must be None regardless of configured_provider. This guards against malformed runtime responses. Test: test_runtime_missing_provider_key_returns_none --- tests/tools/test_delegate.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index 4a40f82b9aa..72c4c67f570 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -1079,6 +1079,24 @@ class TestDelegationCredentialResolution(unittest.TestCase): # Empty provider → early return with None (child inherits parent) self.assertIsNone(creds["provider"]) + @patch("hermes_cli.runtime_provider.resolve_runtime_provider") + def test_runtime_missing_provider_key_returns_none(self, mock_resolve): + """When resolve_runtime_provider returns a dict without 'provider' key, + the result must be None regardless of configured_provider. + This protects against malformed runtime responses. + """ + mock_resolve.return_value = { + # deliberately missing "provider" + "model": "some-model", + "base_url": "https://example.com/v1", + "api_key": "key-123", + "api_mode": "chat_completions", + } + parent = _make_mock_parent(depth=0) + cfg = {"model": "some-model", "provider": "crof.ai"} + creds = _resolve_delegation_credentials(cfg, parent) + self.assertIsNone(creds["provider"]) + class TestDelegationProviderIntegration(unittest.TestCase): """Integration tests: delegation config → _run_single_child → AIAgent construction.""" From 280c63ce91629f9e16d0c2fa82acbbc79c51152b Mon Sep 17 00:00:00 2001 From: soynchux <209694554+soynchux@users.noreply.github.com> Date: Sat, 16 May 2026 22:05:34 +0300 Subject: [PATCH 108/142] fix(mcp): prevent parallel-safe prefix collisions --- tests/run_agent/test_run_agent.py | 16 ++++-- tests/tools/test_mcp_tool.py | 81 +++++++++++++++++++++++++++++-- tools/mcp_tool.py | 47 +++++++++++++----- 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 11b58e5faa1..a72359227a6 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2282,9 +2282,11 @@ class TestMcpParallelToolBatch: def test_mcp_tools_parallel_when_server_opted_in(self): """MCP tools from a parallel-safe server can run concurrently.""" from run_agent import _should_parallelize_tool_batch - from tools.mcp_tool import _parallel_safe_servers, _lock + from tools.mcp_tool import _mcp_tool_server_names, _parallel_safe_servers, _lock with _lock: _parallel_safe_servers.add("github") + _mcp_tool_server_names["mcp_github_list_repos"] = "github" + _mcp_tool_server_names["mcp_github_search_code"] = "github" try: tc1 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c1") tc2 = _mock_tool_call(name="mcp_github_search_code", arguments='{"q":"test"}', call_id="c2") @@ -2292,13 +2294,16 @@ class TestMcpParallelToolBatch: finally: with _lock: _parallel_safe_servers.discard("github") + _mcp_tool_server_names.pop("mcp_github_list_repos", None) + _mcp_tool_server_names.pop("mcp_github_search_code", None) def test_mixed_mcp_and_builtin_parallel(self): """MCP parallel tools mixed with built-in parallel-safe tools.""" from run_agent import _should_parallelize_tool_batch - from tools.mcp_tool import _parallel_safe_servers, _lock + from tools.mcp_tool import _mcp_tool_server_names, _parallel_safe_servers, _lock with _lock: _parallel_safe_servers.add("docs") + _mcp_tool_server_names["mcp_docs_search"] = "docs" try: tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1") tc2 = _mock_tool_call(name="web_search", arguments='{"query":"test"}', call_id="c2") @@ -2306,14 +2311,17 @@ class TestMcpParallelToolBatch: finally: with _lock: _parallel_safe_servers.discard("docs") + _mcp_tool_server_names.pop("mcp_docs_search", None) def test_mixed_parallel_and_serial_mcp_servers(self): """One parallel MCP server + one non-parallel MCP server = sequential.""" from run_agent import _should_parallelize_tool_batch - from tools.mcp_tool import _parallel_safe_servers, _lock + from tools.mcp_tool import _mcp_tool_server_names, _parallel_safe_servers, _lock with _lock: _parallel_safe_servers.add("docs") # "github" is NOT in _parallel_safe_servers + _mcp_tool_server_names["mcp_docs_search"] = "docs" + _mcp_tool_server_names["mcp_github_list_repos"] = "github" try: tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1") tc2 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c2") @@ -2321,6 +2329,8 @@ class TestMcpParallelToolBatch: finally: with _lock: _parallel_safe_servers.discard("docs") + _mcp_tool_server_names.pop("mcp_docs_search", None) + _mcp_tool_server_names.pop("mcp_github_list_repos", None) class TestHandleMaxIterations: diff --git a/tests/tools/test_mcp_tool.py b/tests/tools/test_mcp_tool.py index 0a094eb5467..3212a350c37 100644 --- a/tests/tools/test_mcp_tool.py +++ b/tests/tools/test_mcp_tool.py @@ -3781,16 +3781,26 @@ class TestMcpParallelToolCalls: def test_is_mcp_tool_parallel_safe_no_servers(self): """MCP tool from unknown server returns False.""" - from tools.mcp_tool import is_mcp_tool_parallel_safe, _parallel_safe_servers, _lock + from tools.mcp_tool import ( + is_mcp_tool_parallel_safe, _mcp_tool_server_names, + _parallel_safe_servers, _lock, + ) with _lock: _parallel_safe_servers.clear() + _mcp_tool_server_names.clear() assert is_mcp_tool_parallel_safe("mcp_docs_search") is False def test_is_mcp_tool_parallel_safe_with_flag(self): """MCP tool from a parallel-safe server returns True.""" - from tools.mcp_tool import is_mcp_tool_parallel_safe, _parallel_safe_servers, _lock + from tools.mcp_tool import ( + is_mcp_tool_parallel_safe, _mcp_tool_server_names, + _parallel_safe_servers, _lock, + ) with _lock: _parallel_safe_servers.add("docs") + _mcp_tool_server_names["mcp_docs_search"] = "docs" + _mcp_tool_server_names["mcp_docs_read_file"] = "docs" + _mcp_tool_server_names["mcp_github_list_repos"] = "github" try: assert is_mcp_tool_parallel_safe("mcp_docs_search") is True assert is_mcp_tool_parallel_safe("mcp_docs_read_file") is True @@ -3799,23 +3809,86 @@ class TestMcpParallelToolCalls: finally: with _lock: _parallel_safe_servers.discard("docs") + _mcp_tool_server_names.pop("mcp_docs_search", None) + _mcp_tool_server_names.pop("mcp_docs_read_file", None) + _mcp_tool_server_names.pop("mcp_github_list_repos", None) def test_is_mcp_tool_parallel_safe_server_with_underscores(self): """Server names containing underscores are correctly matched.""" - from tools.mcp_tool import is_mcp_tool_parallel_safe, _parallel_safe_servers, _lock + from tools.mcp_tool import ( + is_mcp_tool_parallel_safe, _mcp_tool_server_names, + _parallel_safe_servers, _lock, + ) with _lock: _parallel_safe_servers.add("my_server") + _mcp_tool_server_names["mcp_my_server_query"] = "my_server" try: assert is_mcp_tool_parallel_safe("mcp_my_server_query") is True finally: with _lock: _parallel_safe_servers.discard("my_server") + _mcp_tool_server_names.pop("mcp_my_server_query", None) + + def test_is_mcp_tool_parallel_safe_uses_exact_registered_server(self): + """Ambiguous MCP names must not match a shorter parallel-safe prefix.""" + from tools.mcp_tool import ( + is_mcp_tool_parallel_safe, _mcp_tool_server_names, + _parallel_safe_servers, _lock, + ) + with _lock: + _parallel_safe_servers.add("a") + _mcp_tool_server_names["mcp_a_search"] = "a" + _mcp_tool_server_names["mcp_a_b_tool"] = "a_b" + try: + assert is_mcp_tool_parallel_safe("mcp_a_search") is True + assert is_mcp_tool_parallel_safe("mcp_a_b_tool") is False + finally: + with _lock: + _parallel_safe_servers.discard("a") + _mcp_tool_server_names.pop("mcp_a_search", None) + _mcp_tool_server_names.pop("mcp_a_b_tool", None) + + def test_registered_tool_provenance_prevents_prefix_collision(self): + """Registration records exact server ownership for ambiguous names.""" + from tools.registry import registry + from tools.mcp_tool import ( + _mcp_tool_server_names, _parallel_safe_servers, + _register_server_tools, is_mcp_tool_parallel_safe, _lock, + ) + + server = _make_mock_server( + "a_b", + tools=[_make_mcp_tool("tool", "Ambiguous tool name")], + ) + registered = _register_server_tools("a_b", server, {}) + try: + assert registered == ["mcp_a_b_tool"] + with _lock: + assert _mcp_tool_server_names["mcp_a_b_tool"] == "a_b" + _parallel_safe_servers.add("a") + assert is_mcp_tool_parallel_safe("mcp_a_b_tool") is False + + with _lock: + _parallel_safe_servers.add("a_b") + assert is_mcp_tool_parallel_safe("mcp_a_b_tool") is True + finally: + for tool_name in registered: + registry.deregister(tool_name) + with _lock: + _parallel_safe_servers.discard("a") + _parallel_safe_servers.discard("a_b") + _mcp_tool_server_names.pop("mcp_a_b_tool", None) def test_is_mcp_tool_parallel_safe_no_tool_suffix(self): """Tool name that is just 'mcp_{server}' without a tool part returns False.""" - from tools.mcp_tool import is_mcp_tool_parallel_safe, _parallel_safe_servers, _lock + from tools.mcp_tool import ( + is_mcp_tool_parallel_safe, _mcp_tool_server_names, + _parallel_safe_servers, _lock, + ) with _lock: _parallel_safe_servers.add("docs") + _mcp_tool_server_names.pop("mcp_docs", None) + _mcp_tool_server_names.pop("mcp_docs_", None) try: # "mcp_docs" has no tool part after the server name assert is_mcp_tool_parallel_safe("mcp_docs") is False diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index 9cec72524af..e1d87389d42 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -1161,6 +1161,7 @@ class MCPServerTask: } for tool_name in stale_tool_names: registry.deregister(tool_name) + _forget_mcp_tool_server(tool_name) # 3. Re-register with fresh tool list self._tools = new_mcp_tools @@ -1696,6 +1697,7 @@ class MCPServerTask: self._pending_refresh_tasks.clear() for tool_name in list(getattr(self, "_registered_tool_names", [])): registry.deregister(tool_name) + _forget_mcp_tool_server(tool_name) self._registered_tool_names = [] self.session = None @@ -2066,11 +2068,20 @@ def _handle_session_expired_and_retry( # ``is_mcp_tool_parallel_safe()`` for the parallel-execution check in run_agent. _parallel_safe_servers: set = set() +# Exact MCP tool-name provenance. MCP tool names are formatted as +# ``mcp_{sanitized_server}_{sanitized_tool}``, which is ambiguous when server +# names contain underscores (``mcp_a_b_tool`` could be server ``a`` + tool +# ``b_tool`` or server ``a_b`` + tool ``tool``). Keep the server component +# captured at registration time so parallel safety never relies on prefix +# guessing. +_mcp_tool_server_names: Dict[str, str] = {} + # Dedicated event loop running in a background daemon thread. _mcp_loop: Optional[asyncio.AbstractEventLoop] = None _mcp_thread: Optional[threading.Thread] = None -# Protects _mcp_loop, _mcp_thread, _servers, _parallel_safe_servers, and _stdio_pids. +# Protects _mcp_loop, _mcp_thread, _servers, _parallel_safe_servers, +# _mcp_tool_server_names, and _stdio_pids. _lock = threading.Lock() # PIDs of stdio MCP server subprocesses. Tracked so we can force-kill @@ -2953,6 +2964,19 @@ _UTILITY_CAPABILITY_ATTRS = { } +def _track_mcp_tool_server(tool_name: str, server_name: str) -> None: + """Remember the exact MCP server that registered *tool_name*.""" + safe_server_name = sanitize_mcp_name_component(server_name) + with _lock: + _mcp_tool_server_names[tool_name] = safe_server_name + + +def _forget_mcp_tool_server(tool_name: str) -> None: + """Forget MCP server provenance for a deregistered tool.""" + with _lock: + _mcp_tool_server_names.pop(tool_name, None) + + def _select_utility_schemas(server_name: str, server: MCPServerTask, config: dict) -> List[dict]: """Select utility schemas based on config and server capabilities.""" tools_filter = config.get("tools") or {} @@ -3087,6 +3111,7 @@ def _register_server_tools(name: str, server: MCPServerTask, config: dict) -> Li is_async=False, description=schema["description"], ) + _track_mcp_tool_server(tool_name_prefixed, name) registered_names.append(tool_name_prefixed) # Register MCP Resources & Prompts utility tools, filtered by config and @@ -3123,6 +3148,7 @@ def _register_server_tools(name: str, server: MCPServerTask, config: dict) -> Li is_async=False, description=schema["description"], ) + _track_mcp_tool_server(util_name, name) registered_names.append(util_name) if registered_names: @@ -3307,24 +3333,19 @@ def discover_mcp_tools() -> List[str]: def is_mcp_tool_parallel_safe(tool_name: str) -> bool: """Check if an MCP tool belongs to a server that supports parallel tool calls. - MCP tool names follow the pattern ``mcp_{server}_{tool}``. This extracts - the server component and checks it against the set of servers whose config - includes ``supports_parallel_tool_calls: true``. + MCP tool names follow the pattern ``mcp_{server}_{tool}``, but that string + shape is ambiguous when server names contain underscores. Use the exact + server provenance captured at registration time rather than prefix + matching, then check whether that server's config includes + ``supports_parallel_tool_calls: true``. Returns False for non-MCP tools or tools from servers without the flag. """ if not tool_name.startswith("mcp_"): return False - # Strip the "mcp_" prefix and extract the server name. - # Tool names are: mcp_{sanitized_server}_{sanitized_tool} - # We need to check all possible server prefixes because the server name - # itself may contain underscores after sanitization. - rest = tool_name[4:] # strip "mcp_" with _lock: - for server_name in _parallel_safe_servers: - if rest.startswith(server_name + "_") and len(rest) > len(server_name) + 1: - return True - return False + server_name = _mcp_tool_server_names.get(tool_name) + return bool(server_name and server_name in _parallel_safe_servers) def get_mcp_status() -> List[dict]: From ee7cd10281c8d6e2cdb9f2f0583c96c0ce2b1639 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 11:50:15 -0700 Subject: [PATCH 109/142] chore(release): map hehehe0803 email for #26212 salvage --- scripts/release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/release.py b/scripts/release.py index fa1ed739d48..2ccdf56aec2 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1159,6 +1159,8 @@ AUTHOR_MAP = { "6666242+bird@users.noreply.github.com": "bird", # PR #25219 (gateway docker exit-75 restart) "david@loadmagic.ai": "davidcampbelldc", # PR #26834 (web_server proxy_headers=False) "165905879+davidcampbelldc@users.noreply.github.com": "davidcampbelldc", + "hoangv.pham0803@gmail.com": "hehehe0803", # PR #26212 salvage (codex kanban writable root) + "26063003+hehehe0803@users.noreply.github.com": "hehehe0803", } From 4a7cd2e16dfacbbed4762f7625ab6eb6e0332447 Mon Sep 17 00:00:00 2001 From: "Hoang V. Pham" Date: Fri, 15 May 2026 15:01:27 +0700 Subject: [PATCH 110/142] fix(codex): allow kanban worker board writes --- agent/transports/codex_app_server.py | 33 ++++++++++- .../test_codex_app_server_runtime.py | 55 +++++++++++++++++++ .../features/codex-app-server-runtime.md | 4 +- 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/agent/transports/codex_app_server.py b/agent/transports/codex_app_server.py index b1aeaa00786..7128de9c4fa 100644 --- a/agent/transports/codex_app_server.py +++ b/agent/transports/codex_app_server.py @@ -74,12 +74,43 @@ class CodexAppServerClient: env: Optional[dict[str, str]] = None, ) -> None: self._codex_bin = codex_bin - cmd = [codex_bin, "app-server"] + list(extra_args or []) spawn_env = os.environ.copy() if env: spawn_env.update(env) if codex_home: spawn_env["CODEX_HOME"] = codex_home + + app_server_args = list(extra_args or []) + # Kanban workers must be able to write their handoff/status back to + # the board DB, which lives outside the per-task workspace. Keep the + # Codex sandbox on, but add the Kanban root as the only extra writable + # root. Without this, codex-runtime workers finish their actual work + # but crash/block when kanban_complete/kanban_block writes SQLite. + if spawn_env.get("HERMES_KANBAN_TASK"): + kanban_db = spawn_env.get("HERMES_KANBAN_DB") + kanban_root = ( + os.path.dirname(kanban_db) + if kanban_db + else spawn_env.get( + "HERMES_KANBAN_ROOT", + os.path.join( + spawn_env.get("HERMES_HOME", os.path.expanduser("~/.hermes")), + "kanban", + ), + ) + ) + app_server_args.extend( + [ + "-c", + 'sandbox_mode="workspace-write"', + "-c", + f'sandbox_workspace_write.writable_roots=["{kanban_root}"]', + "-c", + "sandbox_workspace_write.network_access=false", + ] + ) + + cmd = [codex_bin, "app-server"] + app_server_args # Codex emits tracing to stderr; default WARN keeps it quiet for users. spawn_env.setdefault("RUST_LOG", "warn") diff --git a/tests/agent/transports/test_codex_app_server_runtime.py b/tests/agent/transports/test_codex_app_server_runtime.py index d12ac227254..55bbc8bc6d3 100644 --- a/tests/agent/transports/test_codex_app_server_runtime.py +++ b/tests/agent/transports/test_codex_app_server_runtime.py @@ -241,3 +241,58 @@ class TestSpawnEnvIsolation: assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex" # And HOME still passes through unchanged assert captured["env"].get("HOME") == "/users/alice" + + def test_kanban_worker_adds_only_kanban_writable_root(self, monkeypatch): + """Codex-runtime Kanban workers need to write board state outside + their scratch/worktree workspace, but should not fall back to + danger-full-access. Hermes passes a narrow app-server config override + for the Kanban root only. + """ + import subprocess + from agent.transports import codex_app_server as cas + + captured = {} + + class FakePopen: + def __init__(self, cmd, *args, **kwargs): + captured["cmd"] = list(cmd) + captured["env"] = kwargs.get("env", {}).copy() + self.stdin = None + self.stdout = None + self.stderr = None + self.pid = 1 + self.returncode = None + + def poll(self): + return None + + def terminate(self): + pass + + def wait(self, timeout=None): + return 0 + + def kill(self): + pass + + monkeypatch.setattr(subprocess, "Popen", FakePopen) + monkeypatch.setenv("HOME", "/users/alice") + monkeypatch.setenv("HERMES_HOME", "/users/alice/.hermes/profiles/backend-worker") + monkeypatch.setenv("HERMES_KANBAN_TASK", "t_smoke") + monkeypatch.setenv( + "HERMES_KANBAN_DB", + "/users/alice/.hermes/kanban/boards/smoke/kanban.db", + ) + + client = cas.CodexAppServerClient(codex_bin="codex") + client._closed = True + + cmd = captured["cmd"] + assert cmd[:2] == ["codex", "app-server"] + assert 'sandbox_mode="workspace-write"' in cmd + assert ( + 'sandbox_workspace_write.writable_roots=["/users/alice/.hermes/kanban/boards/smoke"]' + in cmd + ) + assert "sandbox_workspace_write.network_access=false" in cmd + assert all("danger" not in part for part in cmd) diff --git a/website/docs/user-guide/features/codex-app-server-runtime.md b/website/docs/user-guide/features/codex-app-server-runtime.md index a1aa6a0776e..575250d9b01 100644 --- a/website/docs/user-guide/features/codex-app-server-runtime.md +++ b/website/docs/user-guide/features/codex-app-server-runtime.md @@ -91,11 +91,11 @@ What works inside a codex-runtime worker: - The Hermes tool callback for browser_*, vision, image_gen, skills, TTS What also works because the MCP callback exposes them: -- **`kanban_complete` / `kanban_block` / `kanban_comment` / `kanban_heartbeat`** — the worker handoff tools. These read `HERMES_KANBAN_TASK` from env (set by the dispatcher), gate access correctly, and write to `~/.hermes/kanban.db`. Without these in the callback, a worker on this runtime could do its task but couldn't report back, hanging until the dispatcher's timeout. +- **`kanban_complete` / `kanban_block` / `kanban_comment` / `kanban_heartbeat`** — the worker handoff tools. These read `HERMES_KANBAN_TASK` from env (set by the dispatcher), gate access correctly, and write to the per-board SQLite DB pinned by `HERMES_KANBAN_DB`. Without these in the callback, a worker on this runtime could do its task but couldn't report back, hanging until the dispatcher's timeout. - **`kanban_show` / `kanban_list`** — read-only board queries for the worker to check its own context. - **`kanban_create` / `kanban_unblock` / `kanban_link`** — orchestrator-only operations. Available for orchestrator agents running on the codex runtime that need to dispatch new tasks. -The kanban tools are gated by `HERMES_KANBAN_TASK` env var the dispatcher sets — that var is propagated to the codex subprocess (codex inherits env) and from there to the spawned `hermes-tools` MCP server subprocess. So the tools see the right task id and gate correctly. +The kanban tools are gated by `HERMES_KANBAN_TASK` env var the dispatcher sets — that var is propagated to the codex subprocess (codex inherits env) and from there to the spawned `hermes-tools` MCP server subprocess. So the tools see the right task id and gate correctly. For Codex app-server workers, Hermes also passes narrow app-server sandbox overrides when `HERMES_KANBAN_TASK` is present: keep `workspace-write` sandboxing, add only the current board directory (derived from `HERMES_KANBAN_DB`) as an extra writable root, and keep network disabled by default. This avoids the brittle `:danger-no-sandbox` workaround while letting `kanban_complete` / `kanban_block` update the board DB. ### Cron jobs From 7847a58b3a9735a4214d1ee081725f8c8e8d063d Mon Sep 17 00:00:00 2001 From: vaddisrinivas <38348871+vaddisrinivas@users.noreply.github.com> Date: Fri, 15 May 2026 10:27:07 -0400 Subject: [PATCH 111/142] fix(docker): preload messaging gateway deps --- Dockerfile | 18 ++++++++++-------- tests/tools/test_dockerfile_pid1_reaping.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8655c51f34c..bde3412ed7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,9 +66,11 @@ RUN npm install --prefer-offline --no-audit && \ # frontend stats the readme path during dep resolution, so we `touch` an # empty placeholder — the real README is restored by `COPY . .` below. # -# `uv sync --frozen --no-install-project --extra all` installs only the -# deps reachable through the composite `[all]` extra (handpicked set -# intended for the production image). We do NOT use `--all-extras`: +# `uv sync --frozen --no-install-project --extra all --extra messaging` +# installs the deps reachable through the composite `[all]` extra +# (handpicked set intended for the production image), plus gateway +# messaging adapters that should work in the published image without a +# first-boot lazy install. We do NOT use `--all-extras`: # that would pull in `[rl]` (atroposlib + tinker + torch + wandb from # git), `[yc-bench]` (another git dep), and `[termux-all]` (Android # redundancy), none of which belong in the published container. @@ -76,7 +78,7 @@ RUN npm install --prefer-offline --no-audit && \ # The editable link is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md -RUN uv sync --frozen --no-install-project --extra all +RUN uv sync --frozen --no-install-project --extra all --extra messaging # ---------- Source code ---------- # .dockerignore excludes node_modules, so the installs above survive. @@ -94,10 +96,10 @@ RUN cd web && npm run build && \ # hermes_cli/main.py succeeds (see #18800). /opt/hermes/web is build-time # only (HERMES_WEB_DIST points at hermes_cli/web_dist) and is intentionally # not chowned here. -# The .venv MUST be hermes-writable so lazy_deps.py can install platform -# packages (discord.py, telegram, slack, etc.) at first gateway boot. -# Without this, `uv pip install` fails with EACCES and all messaging -# adapters silently fail to load. See tools/lazy_deps.py. +# The .venv MUST remain hermes-writable so lazy_deps.py can install +# remaining optional platform packages and future pin bumps at first use. +# Without this, `uv pip install` fails with EACCES and adapters silently +# fail to load. See tools/lazy_deps.py. USER root RUN chmod -R a+rX /opt/hermes && \ chown -R hermes:hermes /opt/hermes/.venv /opt/hermes/ui-tui /opt/hermes/node_modules diff --git a/tests/tools/test_dockerfile_pid1_reaping.py b/tests/tools/test_dockerfile_pid1_reaping.py index e578d8a69fd..70d95807aa7 100644 --- a/tests/tools/test_dockerfile_pid1_reaping.py +++ b/tests/tools/test_dockerfile_pid1_reaping.py @@ -121,6 +121,20 @@ def test_dockerfile_installs_tui_dependencies(dockerfile_text): ) +def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text): + sync_steps = [ + step for step in _run_steps(dockerfile_text) + if "uv sync" in step and "--no-install-project" in step + ] + + assert sync_steps, "Dockerfile must install Python dependencies with uv sync" + assert any("--extra messaging" in step for step in sync_steps), ( + "Published Docker images must preload the [messaging] extra so " + "Telegram/Discord gateway adapters do not depend on first-boot " + "lazy installation (#24698)." + ) + + def test_dockerfile_builds_tui_assets(dockerfile_text): assert any( "ui-tui" in step and "npm" in step and "run build" in step From a2cc30544c8107a3d0610bc9b796e11d05a3f9a8 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 11:51:36 -0700 Subject: [PATCH 112/142] chore(release): map vaddisrinivas for #26394 salvage --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 2ccdf56aec2..6bb3d200583 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1161,6 +1161,7 @@ AUTHOR_MAP = { "165905879+davidcampbelldc@users.noreply.github.com": "davidcampbelldc", "hoangv.pham0803@gmail.com": "hehehe0803", # PR #26212 salvage (codex kanban writable root) "26063003+hehehe0803@users.noreply.github.com": "hehehe0803", + "38348871+vaddisrinivas@users.noreply.github.com": "vaddisrinivas", # PR #26394 salvage (Docker messaging extra) } From 73df329214a89eddbd45b0fa84ee99aefa8aea30 Mon Sep 17 00:00:00 2001 From: worlldz <101180447+worlldz@users.noreply.github.com> Date: Fri, 15 May 2026 18:45:02 +0300 Subject: [PATCH 113/142] fix(doctor): flag missing credentials for active openrouter provider --- hermes_cli/doctor.py | 52 ++++++++++++++++++++------------- tests/hermes_cli/test_doctor.py | 42 ++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 6f036426fa5..87043bc2611 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -651,31 +651,41 @@ def run_doctor(args): # Check credentials for the configured provider. # Limit to API-key providers in PROVIDER_REGISTRY — other provider - # types (OAuth, SDK, openrouter/anthropic/custom/auto) have their - # own env-var checks elsewhere in doctor, and get_auth_status() - # returns a bare {logged_in: False} for anything it doesn't - # explicitly dispatch, which would produce false positives. - if runtime_provider and runtime_provider not in {"auto", "custom", "openrouter"}: + # types (OAuth, SDK, anthropic/custom/auto) have their own env-var + # checks elsewhere in doctor, and get_auth_status() returns a bare + # {logged_in: False} for anything it doesn't explicitly dispatch, + # which would produce false positives. + if runtime_provider and runtime_provider not in ("auto", "custom"): try: - from hermes_cli.auth import PROVIDER_REGISTRY, get_auth_status - pconfig = PROVIDER_REGISTRY.get(runtime_provider) - if pconfig and getattr(pconfig, "auth_type", "") == "api_key": - status = get_auth_status(runtime_provider) or {} + if runtime_provider == "openrouter": + from hermes_cli.config import get_env_value + configured = bool( - status.get("configured") - or status.get("logged_in") - or status.get("api_key") + str(get_env_value("OPENROUTER_API_KEY") or "").strip() + or str(get_env_value("OPENAI_API_KEY") or "").strip() ) - if not configured: - check_fail( - f"model.provider '{runtime_provider}' is set but no API key is configured", - "(check ~/.hermes/.env or run 'hermes setup')", - ) - issues.append( - f"No credentials found for provider '{runtime_provider}'. " - f"Run 'hermes setup' or set the provider's API key in {_DHH}/.env, " - f"or switch providers with 'hermes config set model.provider '" + else: + from hermes_cli.auth import PROVIDER_REGISTRY, get_auth_status + + pconfig = PROVIDER_REGISTRY.get(runtime_provider) + configured = True + if pconfig and getattr(pconfig, "auth_type", "") == "api_key": + status = get_auth_status(runtime_provider) or {} + configured = bool( + status.get("configured") + or status.get("logged_in") + or status.get("api_key") ) + if not configured: + check_fail( + f"model.provider '{runtime_provider}' is set but no API key is configured", + "(check ~/.hermes/.env or run 'hermes setup')", + ) + issues.append( + f"No credentials found for provider '{runtime_provider}'. " + f"Run 'hermes setup' or set the provider's API key in {_DHH}/.env, " + f"or switch providers with 'hermes config set model.provider '" + ) except Exception: pass diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index a5b058fe452..be8c35239b3 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -477,6 +477,48 @@ def test_run_doctor_accepts_bare_custom_provider(monkeypatch, tmp_path): assert "model.provider 'custom' is not a recognised provider" not in out +def test_run_doctor_flags_missing_credentials_for_active_openrouter_provider(monkeypatch, tmp_path): + home = tmp_path / ".hermes" + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text( + "model:\n" + " provider: openrouter\n" + " default: openai/gpt-4.1-mini\n", + encoding="utf-8", + ) + + monkeypatch.setattr(doctor_mod, "HERMES_HOME", home) + monkeypatch.setattr(doctor_mod, "PROJECT_ROOT", tmp_path / "project") + monkeypatch.setattr(doctor_mod, "_DHH", str(home)) + (tmp_path / "project").mkdir(exist_ok=True) + + fake_model_tools = types.SimpleNamespace( + check_tool_availability=lambda *a, **kw: ([], []), + TOOLSET_REQUIREMENTS={}, + ) + monkeypatch.setitem(sys.modules, "model_tools", fake_model_tools) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + try: + from hermes_cli import auth as _auth_mod + + monkeypatch.setattr(_auth_mod, "get_nous_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_codex_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_gemini_oauth_auth_status", lambda: {}) + monkeypatch.setattr(_auth_mod, "get_minimax_oauth_auth_status", lambda: {}) + except Exception: + pass + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + doctor_mod.run_doctor(Namespace(fix=False)) + + out = buf.getvalue() + assert "model.provider 'openrouter' is set but no API key is configured" in out + assert "No credentials found for provider 'openrouter'." in out + + @pytest.mark.parametrize( ("provider", "default_model"), [ From 1a82b7a1ff00a389bd39f92f8203793492fd9e5f Mon Sep 17 00:00:00 2001 From: aqilaziz <46887634+aqilaziz@users.noreply.github.com> Date: Sat, 16 May 2026 03:25:01 +0700 Subject: [PATCH 114/142] fix(tests): stabilize xai env and provider parity --- tests/run_agent/test_provider_parity.py | 22 ++++++++++++--- .../test_transcription_dotenv_fallback.py | 27 +++++++++++++++++++ tools/xai_http.py | 12 ++++----- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/tests/run_agent/test_provider_parity.py b/tests/run_agent/test_provider_parity.py index c65c22004a9..cf619ea9743 100644 --- a/tests/run_agent/test_provider_parity.py +++ b/tests/run_agent/test_provider_parity.py @@ -254,8 +254,12 @@ class TestDeveloperRoleSwap: assert messages[0]["role"] == "system" def test_developer_role_via_nous_portal(self, monkeypatch): - agent = _make_agent(monkeypatch, "nous", base_url="https://inference-api.nousresearch.com/v1") - agent.model = "gpt-5" + agent = _make_agent( + monkeypatch, + "nous", + base_url="https://inference-api.nousresearch.com/v1", + model="gpt-5", + ) messages = [ {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "hi"}, @@ -346,14 +350,24 @@ class TestBuildApiKwargsAIGateway: class TestBuildApiKwargsNousPortal: def test_includes_nous_product_tags(self, monkeypatch): from agent.portal_tags import nous_portal_tags - agent = _make_agent(monkeypatch, "nous", base_url="https://inference-api.nousresearch.com/v1") + agent = _make_agent( + monkeypatch, + "nous", + base_url="https://inference-api.nousresearch.com/v1", + model="gpt-5", + ) messages = [{"role": "user", "content": "hi"}] kwargs = agent._build_api_kwargs(messages) extra = kwargs.get("extra_body", {}) assert extra.get("tags") == nous_portal_tags() def test_uses_chat_completions_format(self, monkeypatch): - agent = _make_agent(monkeypatch, "nous", base_url="https://inference-api.nousresearch.com/v1") + agent = _make_agent( + monkeypatch, + "nous", + base_url="https://inference-api.nousresearch.com/v1", + model="gpt-5", + ) messages = [{"role": "user", "content": "hi"}] kwargs = agent._build_api_kwargs(messages) assert "messages" in kwargs diff --git a/tests/tools/test_transcription_dotenv_fallback.py b/tests/tools/test_transcription_dotenv_fallback.py index a28c777a8f1..365b910d4cc 100644 --- a/tests/tools/test_transcription_dotenv_fallback.py +++ b/tests/tools/test_transcription_dotenv_fallback.py @@ -58,6 +58,33 @@ class TestProviderSelectionGate: finally: importlib.reload(tt) + def test_xai_resolver_import_after_config_env_patch_uses_restored_dotenv_loader(self): + """xAI HTTP auth must not cache a temporarily patched env helper.""" + import importlib + import hermes_cli.config as config_mod + from tools import xai_http + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(config_mod, "get_env_value", lambda name, default=None: "") + xai_http = importlib.reload(xai_http) + + try: + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=RuntimeError("no oauth"), + ), patch( + "hermes_cli.auth.resolve_xai_oauth_runtime_credentials", + return_value={}, + ), patch( + "hermes_cli.config.load_env", + return_value={"XAI_API_KEY": "dotenv-secret"}, + ): + creds = xai_http.resolve_xai_http_credentials() + finally: + importlib.reload(xai_http) + + assert creds["api_key"] == "dotenv-secret" + def test_explicit_groq_sees_dotenv(self): from tools import transcription_tools as tt diff --git a/tools/xai_http.py b/tools/xai_http.py index 216a51ff10d..848ad8fc748 100644 --- a/tools/xai_http.py +++ b/tools/xai_http.py @@ -5,12 +5,6 @@ from __future__ import annotations import os from typing import Dict -try: - from hermes_cli.config import get_env_value as _hermes_get_env_value -except Exception: - _hermes_get_env_value = None - - def get_env_value(name: str, default=None): """Read ``name`` from ``~/.hermes/.env`` first, then ``os.environ``. @@ -18,10 +12,14 @@ def get_env_value(name: str, default=None): ``tools.xai_http.get_env_value`` to inject dotenv-only secrets into the xAI credential resolver. """ - if _hermes_get_env_value is not None: + try: + from hermes_cli.config import get_env_value as _hermes_get_env_value + value = _hermes_get_env_value(name) if value is not None: return value + except Exception: + pass return os.environ.get(name, default) From bc7c608d54367ff11a10e18b48e82999005c3ea7 Mon Sep 17 00:00:00 2001 From: aqilaziz <46887634+aqilaziz@users.noreply.github.com> Date: Sat, 16 May 2026 05:56:28 +0700 Subject: [PATCH 115/142] fix(gateway): ignore inaccessible service path dirs --- hermes_cli/gateway.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index c5303e32799..ef57d5ce9fe 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -2110,24 +2110,30 @@ def _build_service_path_dirs(project_root: Path | None = None) -> list[str]: if project_root is None: project_root = PROJECT_ROOT + def _is_dir(path: Path) -> bool: + try: + return path.is_dir() + except OSError: + return False + candidates = [] venv_bin = project_root / "venv" / "bin" - if venv_bin.is_dir(): + if _is_dir(venv_bin): candidates.append(str(venv_bin)) elif sys.prefix != sys.base_prefix: candidates.append(str(Path(sys.prefix) / "bin")) node_bin = project_root / "node_modules" / ".bin" - if node_bin.is_dir(): + if _is_dir(node_bin): candidates.append(str(node_bin)) hermes_home = get_hermes_home() hermes_node = hermes_home / "node" / "bin" - if hermes_node.is_dir(): + if _is_dir(hermes_node): candidates.append(str(hermes_node)) hermes_nm = hermes_home / "node_modules" / ".bin" - if hermes_nm.is_dir(): + if _is_dir(hermes_nm): candidates.append(str(hermes_nm)) return candidates From cb53c40e459f1913d086a3ba942746eb605ec6f5 Mon Sep 17 00:00:00 2001 From: xxxigm Date: Sat, 16 May 2026 23:11:21 +0700 Subject: [PATCH 116/142] fix(xai-oauth): echo code_challenge in token POST so PKCE exchange succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xAI's OAuth implementation at ``auth.x.ai`` validates the PKCE ``code_challenge`` at the **token** endpoint, not just at the authorize step. When Hermes sends the standards-compliant token POST with ``code_verifier`` alone — exactly what RFC 7636 §4.5 prescribes — xAI rejects the exchange with ``code_challenge is required`` and the user is stuck with no working OAuth login. The fix: * Extract the token POST into ``_xai_oauth_exchange_code_for_tokens`` so the wire format is unit-testable in isolation. * Send the original ``code_challenge`` and ``code_challenge_method`` in the form body alongside ``code_verifier``. Strict RFC-compliant servers ignore the extras at the token endpoint, and xAI's permissive implementation accepts the exchange. This is the standard "defensive echo" workaround used by every OAuth client that targets a server with this quirk. * Refuse to fire the POST when ``code_verifier`` is empty — leaking the authorization code to a server that can't redeem it is worse than failing locally with an actionable error. The new error code is ``xai_pkce_verifier_missing`` and the message points at this issue for context. * Surface the HTTP status code prominently in the 4xx error message (``xAI token exchange failed (HTTP 400). Response: …``) so users and maintainers can tell a 400 (bad request / PKCE problem) from a 403 (tier denied, see #26847) at a glance instead of parsing the JSON body by eye. Closes #26990 --- hermes_cli/auth.py | 150 ++++++++++++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 41 deletions(-) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 6752b65829f..8b154db7468 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -5312,6 +5312,107 @@ def _xai_oauth_build_authorize_url( return f"{authorization_endpoint}?{urlencode(authorize_params)}" +def _xai_oauth_exchange_code_for_tokens( + *, + token_endpoint: str, + code: str, + redirect_uri: str, + code_verifier: str, + code_challenge: str, + timeout_seconds: float = 20.0, +) -> Dict[str, Any]: + """POST the authorization code to xAI's token endpoint and return + the parsed JSON payload. + + Sends ``code_verifier`` as required by RFC 7636 §4.5. Also echoes + ``code_challenge`` + ``code_challenge_method`` in the request body + as a defense-in-depth measure for OAuth servers (xAI's among them, + per #26990) that re-validate the challenge at the token step + instead of relying solely on server-side session state captured + during the authorize step. Echoing the challenge is harmless for + strict RFC-compliant servers — RFC 7636 doesn't forbid additional + parameters at the token endpoint — and decisively fixes the + ``code_challenge is required`` failure mode users hit on the + loopback flow. + + Raises :class:`AuthError` on any non-2xx response or transport + failure; the error message embeds the HTTP status code and the + full response body so users can disambiguate cause at a glance. + """ + # Paranoia: if upstream call sites ever drop ``code_verifier`` we + # want to surface a precise, local error rather than send a + # missing-PKCE request to xAI and receive their generic "code + # challenge required" message back. + if not code_verifier: + raise AuthError( + "xAI token exchange refused locally: PKCE code_verifier is empty. " + "This is a bug in Hermes — please report at " + "https://github.com/NousResearch/hermes-agent/issues/26990.", + provider="xai-oauth", + code="xai_pkce_verifier_missing", + ) + + data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": redirect_uri, + "client_id": XAI_OAUTH_CLIENT_ID, + "code_verifier": code_verifier, + } + # Defense-in-depth: include the original ``code_challenge`` and + # ``code_challenge_method``. Some OAuth servers (including xAI's + # auth.x.ai implementation, per the symptom reported in #26990) + # validate these at the token endpoint instead of relying purely on + # state captured during the authorize step — without them, xAI + # rejects the exchange with ``code_challenge is required`` even + # though we sent a valid ``code_verifier``. + if code_challenge: + data["code_challenge"] = code_challenge + data["code_challenge_method"] = "S256" + + try: + response = httpx.post( + token_endpoint, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + }, + data=data, + timeout=max(20.0, timeout_seconds), + ) + except Exception as exc: + raise AuthError( + f"xAI token exchange failed: {exc}", + provider="xai-oauth", + code="xai_token_exchange_failed", + ) from exc + + if response.status_code != 200: + body = response.text.strip() + raise AuthError( + f"xAI token exchange failed (HTTP {response.status_code})." + + (f" Response: {body}" if body else ""), + provider="xai-oauth", + code="xai_token_exchange_failed", + ) + + try: + payload = response.json() + except Exception as exc: + raise AuthError( + f"xAI token exchange returned invalid JSON: {exc}", + provider="xai-oauth", + code="xai_token_exchange_invalid", + ) from exc + if not isinstance(payload, dict): + raise AuthError( + "xAI token exchange response was not a JSON object.", + provider="xai-oauth", + code="xai_token_exchange_invalid", + ) + return payload + + def _xai_oauth_loopback_login( *, timeout_seconds: float = 20.0, @@ -5392,47 +5493,14 @@ def _xai_oauth_loopback_login( code="xai_code_missing", ) - try: - response = httpx.post( - token_endpoint, - headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"}, - data={ - "grant_type": "authorization_code", - "code": code, - "redirect_uri": redirect_uri, - "client_id": XAI_OAUTH_CLIENT_ID, - "code_verifier": code_verifier, - }, - timeout=max(20.0, timeout_seconds), - ) - except Exception as exc: - raise AuthError( - f"xAI token exchange failed: {exc}", - provider="xai-oauth", - code="xai_token_exchange_failed", - ) from exc - if response.status_code != 200: - detail = response.text.strip() - raise AuthError( - "xAI token exchange failed." - + (f" Response: {detail}" if detail else ""), - provider="xai-oauth", - code="xai_token_exchange_failed", - ) - try: - payload = response.json() - except Exception as exc: - raise AuthError( - f"xAI token exchange returned invalid JSON: {exc}", - provider="xai-oauth", - code="xai_token_exchange_invalid", - ) from exc - if not isinstance(payload, dict): - raise AuthError( - "xAI token exchange response was not a JSON object.", - provider="xai-oauth", - code="xai_token_exchange_invalid", - ) + payload = _xai_oauth_exchange_code_for_tokens( + token_endpoint=token_endpoint, + code=code, + redirect_uri=redirect_uri, + code_verifier=code_verifier, + code_challenge=code_challenge, + timeout_seconds=timeout_seconds, + ) access_token = str(payload.get("access_token", "") or "").strip() refresh_token = str(payload.get("refresh_token", "") or "").strip() if not access_token: From e3f7ff1123fc8e0dc156807fb0935c89f613d6f4 Mon Sep 17 00:00:00 2001 From: xxxigm Date: Sat, 16 May 2026 23:11:34 +0700 Subject: [PATCH 117/142] test(xai-oauth): pin PKCE token-exchange wire format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 14 focused tests on the extracted helper ``_xai_oauth_exchange_code_for_tokens`` cover: Core contract: * ``code_verifier`` is on the wire (RFC 7636 §4.5). * ``code_challenge`` + ``code_challenge_method=S256`` are echoed (the #26990 defense-in-depth that makes xAI's token endpoint stop rejecting valid exchanges). * ``grant_type=authorization_code``, ``code``, ``redirect_uri``, and ``client_id`` are all locked. * Content-Type is ``application/x-www-form-urlencoded`` (xAI rejects ``application/json`` on this endpoint). * The supplied ``token_endpoint`` URL is used verbatim — no hard-coded constant sneaks in via a future refactor. * ``timeout_seconds`` is forwarded; floored at 20s. Sanity guard: * Empty ``code_verifier`` raises ``xai_pkce_verifier_missing`` with a link to #26990 — and NOTHING is sent. Leaking the auth code to a server that can't redeem it is the wrong failure mode. * Empty ``code_challenge`` omits only the defensive echo; the standards-compliant ``code_verifier`` request still goes out so RFC-compliant servers keep working. Error surfacing: * Non-200 responses include both ``HTTP `` and the body verbatim — disambiguates 400 (PKCE / bad request) from 403 (tier denied, see #26847). * Transport errors are wrapped as ``AuthError`` with the ``xai_token_exchange_failed`` code, so the surrounding ``format_auth_error`` UI mapping still fires. * Non-dict JSON payloads raise ``xai_token_exchange_invalid``. * 200 happy path returns the parsed payload dict verbatim. End-to-end wire-format guard: * A real ``httpx.Client`` with a stub transport captures the bytes on the wire and asserts every PKCE field round-trips through ``urlencode``. Catches a future refactor that swaps ``data=`` for ``json=`` (which xAI would silently reject). --- .../test_xai_oauth_pkce_token_exchange.py | 359 ++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 tests/hermes_cli/test_xai_oauth_pkce_token_exchange.py diff --git a/tests/hermes_cli/test_xai_oauth_pkce_token_exchange.py b/tests/hermes_cli/test_xai_oauth_pkce_token_exchange.py new file mode 100644 index 00000000000..98b81ff140e --- /dev/null +++ b/tests/hermes_cli/test_xai_oauth_pkce_token_exchange.py @@ -0,0 +1,359 @@ +"""Regression coverage for xAI OAuth PKCE token exchange (issue #26990). + +Issue [#26990] reported that ``hermes auth add xai-oauth`` succeeds at the +browser-side authorize step but fails at the token endpoint with +``code_challenge is required`` — the symptom of an OAuth server that +re-validates PKCE at the token step instead of relying purely on +state captured during the authorize redirect. + +The fix in ``hermes_cli/auth.py`` extracts the token POST into +:func:`_xai_oauth_exchange_code_for_tokens` and: + +* Sends ``code_verifier`` (RFC 7636 §4.5 requirement). +* **Also** echoes ``code_challenge`` and ``code_challenge_method`` + in the request body as defense-in-depth — strictly compliant + servers ignore extras at the token endpoint, but xAI's server + needs them. +* Refuses to fire the POST locally when ``code_verifier`` is empty + (avoids leaking the auth code to a server that can't redeem it). +* Surfaces the HTTP status code prominently in the error message so + users / maintainers can tell a 400 (bad request) from a 403 + (entitlement denied) at a glance. + +These tests pin all three behaviors so the fix can't silently regress. +""" + +from __future__ import annotations + +from typing import Any, Dict, List +from urllib.parse import parse_qs + +import httpx +import pytest + +from hermes_cli.auth import ( + AuthError, + XAI_OAUTH_CLIENT_ID, + _xai_oauth_exchange_code_for_tokens, +) + + +# --------------------------------------------------------------------------- +# httpx.post recorder +# --------------------------------------------------------------------------- + + +class _PostRecorder: + """Capture every ``httpx.post`` call without touching the network.""" + + def __init__(self, response: httpx.Response) -> None: + self.response = response + self.calls: List[Dict[str, Any]] = [] + + def __call__(self, url, *, headers=None, data=None, timeout=None, **kw): + self.calls.append( + {"url": url, "headers": headers or {}, "data": data or {}, + "timeout": timeout, "extra": kw} + ) + return self.response + + +def _ok_response(payload: dict) -> httpx.Response: + return httpx.Response(200, json=payload) + + +def _err_response(status: int, body: str) -> httpx.Response: + return httpx.Response(status, text=body) + + +@pytest.fixture +def post_recorder(monkeypatch): + """Default: 200 response with a full xAI token payload.""" + recorder = _PostRecorder( + _ok_response( + { + "access_token": "AT-fresh", + "refresh_token": "RT-fresh", + "id_token": "ID", + "expires_in": 3600, + "token_type": "Bearer", + } + ) + ) + monkeypatch.setattr("hermes_cli.auth.httpx.post", recorder) + return recorder + + +# --------------------------------------------------------------------------- +# Core contract: which fields go on the wire? +# --------------------------------------------------------------------------- + + +def test_token_exchange_includes_code_verifier(post_recorder): + """RFC 7636 §4.5 — ``code_verifier`` MUST be sent.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="theVerifier_43_to_128_chars_____________________", + code_challenge="aBcDeF", + ) + sent = post_recorder.calls[-1]["data"] + assert sent["code_verifier"] == "theVerifier_43_to_128_chars_____________________" + + +def test_token_exchange_also_echoes_code_challenge_for_xai(post_recorder): + """Defense-in-depth for #26990 — xAI re-validates the challenge + at the token endpoint, not just at authorize. Without this echo + we get ``code_challenge is required`` even though we send a valid + ``code_verifier``.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="aBcDeF", + ) + sent = post_recorder.calls[-1]["data"] + assert sent["code_challenge"] == "aBcDeF" + assert sent["code_challenge_method"] == "S256" + + +def test_token_exchange_uses_correct_grant_and_client(post_recorder): + """Lock the static fields too — a future refactor must not flip + these to ``client_credentials`` or drop ``client_id``.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + sent = post_recorder.calls[-1]["data"] + assert sent["grant_type"] == "authorization_code" + assert sent["code"] == "AUTHCODE" + assert sent["redirect_uri"] == "http://127.0.0.1:56121/callback" + assert sent["client_id"] == XAI_OAUTH_CLIENT_ID + + +def test_token_exchange_uses_form_urlencoded_content_type(post_recorder): + """xAI's token endpoint expects ``application/x-www-form-urlencoded``.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + headers = post_recorder.calls[-1]["headers"] + assert headers["Content-Type"] == "application/x-www-form-urlencoded" + assert headers["Accept"] == "application/json" + + +def test_token_exchange_targets_the_supplied_endpoint(post_recorder): + """Some test fixtures sniff the discovered token endpoint dynamically. + We must POST to the URL the caller passed, not a hard-coded constant.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/some/other/token/path", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + assert post_recorder.calls[-1]["url"] == "https://auth.x.ai/some/other/token/path" + + +def test_token_exchange_passes_timeout_through(post_recorder): + """Operators on slow networks pass a higher ``timeout_seconds``; + the helper must forward it (and bump the floor to 20s).""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + timeout_seconds=45.0, + ) + assert post_recorder.calls[-1]["timeout"] == 45.0 + + +def test_token_exchange_floor_timeout_is_20s(post_recorder): + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + timeout_seconds=2.0, + ) + assert post_recorder.calls[-1]["timeout"] == 20.0 + + +# --------------------------------------------------------------------------- +# Sanity guard: refuse to POST with an empty code_verifier +# --------------------------------------------------------------------------- + + +def test_empty_code_verifier_raises_without_posting(post_recorder): + """If ``code_verifier`` is somehow lost upstream, we must refuse to + send the request — leaking an authorization code to xAI without a + verifier is worse than failing locally with an actionable error.""" + with pytest.raises(AuthError) as exc_info: + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="", + code_challenge="c" * 43, + ) + assert exc_info.value.code == "xai_pkce_verifier_missing" + assert "26990" in str(exc_info.value) + # And critically: nothing was sent. + assert post_recorder.calls == [] + + +def test_missing_code_challenge_omits_echo_but_still_sends_verifier(post_recorder): + """``code_challenge`` is defensive — if a caller doesn't have it + handy, we must still send the standards-compliant request rather + than refusing. This keeps RFC-compliant servers happy.""" + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="", + ) + sent = post_recorder.calls[-1]["data"] + assert sent["code_verifier"] == "v" * 64 + assert "code_challenge" not in sent + assert "code_challenge_method" not in sent + + +# --------------------------------------------------------------------------- +# Error surfacing +# --------------------------------------------------------------------------- + + +def test_non_200_response_surfaces_status_and_body(monkeypatch): + """When xAI returns a 4xx, the operator needs both the HTTP status + code (to tell 400 from 401 from 403 at a glance) and the response + body (the actual server-side reason).""" + recorder = _PostRecorder( + _err_response(400, '{"error":"invalid_grant","error_description":"code_challenge is required"}') + ) + monkeypatch.setattr("hermes_cli.auth.httpx.post", recorder) + with pytest.raises(AuthError) as exc_info: + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + msg = str(exc_info.value) + assert "HTTP 400" in msg, ( + "Status code must be in the error so callers can disambiguate " + "tier-denied (403) from bad-request (400) without inspecting " + "exc.code." + ) + assert "code_challenge is required" in msg + assert exc_info.value.code == "xai_token_exchange_failed" + + +def test_transport_error_wraps_as_auth_error(monkeypatch): + """A connection failure must come back as ``AuthError`` so the + surrounding ``format_auth_error`` UI mapping fires correctly.""" + + def _boom(*args, **kwargs): + raise httpx.ConnectError("dns failure") + + monkeypatch.setattr("hermes_cli.auth.httpx.post", _boom) + with pytest.raises(AuthError) as exc_info: + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + assert exc_info.value.code == "xai_token_exchange_failed" + assert "dns failure" in str(exc_info.value) + + +def test_non_dict_payload_raises_invalid_json(monkeypatch): + """xAI returning ``[]`` or a string at 200 is a server bug — fail + with a precise error rather than crashing later in token storage.""" + recorder = _PostRecorder(_ok_response([1, 2, 3])) # type: ignore[arg-type] + monkeypatch.setattr("hermes_cli.auth.httpx.post", recorder) + with pytest.raises(AuthError) as exc_info: + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + assert exc_info.value.code == "xai_token_exchange_invalid" + + +def test_success_returns_full_payload_dict(post_recorder): + """200 happy path: the parsed JSON dict comes back verbatim so the + caller can pluck ``access_token`` / ``refresh_token`` etc.""" + out = _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="v" * 64, + code_challenge="c" * 43, + ) + assert out["access_token"] == "AT-fresh" + assert out["refresh_token"] == "RT-fresh" + + +# --------------------------------------------------------------------------- +# Wire-format guard: httpx must serialise ``data`` as form-urlencoded +# --------------------------------------------------------------------------- + + +def test_wire_format_is_form_urlencoded_with_all_pkce_fields(monkeypatch): + """End-to-end check on the actual bytes httpx puts on the wire. + If anyone ever swaps ``data=`` for ``json=`` or refactors the dict, + xAI will start rejecting again — this catches it locally.""" + + captured: Dict[str, Any] = {} + + class _Transport(httpx.BaseTransport): + def handle_request(self, request): + captured["body"] = bytes(request.read()) + captured["content_type"] = request.headers.get("content-type", "") + return httpx.Response( + 200, + json={"access_token": "AT", "refresh_token": "RT", + "id_token": "", "expires_in": 60, "token_type": "Bearer"}, + ) + + real_post = httpx.post + + def _post(*args, **kwargs): + with httpx.Client(transport=_Transport()) as c: + return c.post(*args, **kwargs) + + monkeypatch.setattr("hermes_cli.auth.httpx.post", _post) + + _xai_oauth_exchange_code_for_tokens( + token_endpoint="https://auth.x.ai/oauth2/token", + code="AUTHCODE", + redirect_uri="http://127.0.0.1:56121/callback", + code_verifier="theVerifier_43+", + code_challenge="theChallenge_43+", + ) + + assert "application/x-www-form-urlencoded" in captured["content_type"] + parsed = parse_qs(captured["body"].decode()) + assert parsed["grant_type"] == ["authorization_code"] + assert parsed["code"] == ["AUTHCODE"] + assert parsed["redirect_uri"] == ["http://127.0.0.1:56121/callback"] + assert parsed["client_id"] == [XAI_OAUTH_CLIENT_ID] + assert parsed["code_verifier"] == ["theVerifier_43+"] + assert parsed["code_challenge"] == ["theChallenge_43+"] + assert parsed["code_challenge_method"] == ["S256"] From 822e92edb313193494d397064f9d3a8572a74b63 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:11 -0700 Subject: [PATCH 118/142] fix(aux): default OpenRouter auxiliary to gemini-3-flash-preview --- agent/auxiliary_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index e02fa1911f7..a7fcd311f11 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -424,7 +424,7 @@ NOUS_EXTRA_BODY = _nous_extra_body() auxiliary_is_nous: bool = False # Default auxiliary models per provider -_OPENROUTER_MODEL = "google/gemini-2.5-flash" +_OPENROUTER_MODEL = "google/gemini-3-flash-preview" _NOUS_MODEL = "google/gemini-3-flash-preview" _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1" _ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com" From e66a3e86efbc9e428bb5ace45501d6f6ac92d36e Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:11 -0700 Subject: [PATCH 119/142] chore(acp): bump registry manifest to 0.14.0 matching pyproject --- acp_registry/agent.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acp_registry/agent.json b/acp_registry/agent.json index b94a48e089f..b23d1642a94 100644 --- a/acp_registry/agent.json +++ b/acp_registry/agent.json @@ -1,7 +1,7 @@ { "id": "hermes-agent", "name": "Hermes Agent", - "version": "0.13.0", + "version": "0.14.0", "description": "Self-improving open-source AI agent by Nous Research with ACP editor integration, persistent memory, skills, and rich tool support.", "repository": "https://github.com/NousResearch/hermes-agent", "website": "https://hermes-agent.nousresearch.com/docs/user-guide/features/acp", @@ -9,7 +9,7 @@ "license": "MIT", "distribution": { "uvx": { - "package": "hermes-agent[acp]==0.13.0", + "package": "hermes-agent[acp]==0.14.0", "args": ["hermes-acp"] } } From 06924e827cb8184a933899dbb365b1cb51c9eaa2 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:11 -0700 Subject: [PATCH 120/142] test(gateway): accept trust_env in fake aiohttp ClientSession lambdas --- tests/gateway/test_google_chat.py | 2 +- tests/gateway/test_teams.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gateway/test_google_chat.py b/tests/gateway/test_google_chat.py index 3f093bcea1d..9d36945a357 100644 --- a/tests/gateway/test_google_chat.py +++ b/tests/gateway/test_google_chat.py @@ -2740,7 +2740,7 @@ class _FakeAiohttpSession: def _install_fake_aiohttp(monkeypatch, session): fake_aiohttp = types.SimpleNamespace( - ClientSession=lambda timeout=None: session, + ClientSession=lambda timeout=None, **kwargs: session, ClientTimeout=lambda total=None: None, ) monkeypatch.setitem(sys.modules, "aiohttp", fake_aiohttp) diff --git a/tests/gateway/test_teams.py b/tests/gateway/test_teams.py index 58b8c35a5c2..6c7173fe931 100644 --- a/tests/gateway/test_teams.py +++ b/tests/gateway/test_teams.py @@ -763,7 +763,7 @@ def _install_fake_aiohttp(monkeypatch, session): """Replace ``aiohttp`` in ``sys.modules`` so ``import aiohttp as _aiohttp`` inside ``_standalone_send`` picks up our fake.""" fake_aiohttp = types.SimpleNamespace( - ClientSession=lambda timeout=None: session, + ClientSession=lambda timeout=None, **kwargs: session, ClientTimeout=lambda total=None: None, ) monkeypatch.setitem(sys.modules, "aiohttp", fake_aiohttp) From dfc6ea72c16ee971ac1d6f4b3118cd8cc1f47a7d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:11 -0700 Subject: [PATCH 121/142] test(gateway): include direct_messages_topic_id in telegram DM metadata assertions --- tests/gateway/test_background_command.py | 1 + tests/gateway/test_telegram_thread_fallback.py | 1 + tests/gateway/test_voice_command.py | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/gateway/test_background_command.py b/tests/gateway/test_background_command.py index 9c156960c70..9e0d71921cd 100644 --- a/tests/gateway/test_background_command.py +++ b/tests/gateway/test_background_command.py @@ -316,6 +316,7 @@ class TestRunBackgroundTask: assert mock_adapter.send.call_args.kwargs["metadata"] == { "thread_id": "20197", "telegram_dm_topic_reply_fallback": True, + "direct_messages_topic_id": "20197", "telegram_reply_to_message_id": "463", } diff --git a/tests/gateway/test_telegram_thread_fallback.py b/tests/gateway/test_telegram_thread_fallback.py index f310d017946..f46997f0b92 100644 --- a/tests/gateway/test_telegram_thread_fallback.py +++ b/tests/gateway/test_telegram_thread_fallback.py @@ -408,6 +408,7 @@ async def test_gateway_runner_busy_ack_replies_to_triggering_message_for_telegra assert adapter.calls[0]["metadata"] == { "thread_id": "20197", "telegram_dm_topic_reply_fallback": True, + "direct_messages_topic_id": "20197", "telegram_reply_to_message_id": "463", } diff --git a/tests/gateway/test_voice_command.py b/tests/gateway/test_voice_command.py index a877730dcec..d792a48e0cf 100644 --- a/tests/gateway/test_voice_command.py +++ b/tests/gateway/test_voice_command.py @@ -461,6 +461,7 @@ class TestSendVoiceReply: assert call_kwargs["metadata"] == { "thread_id": "20197", "telegram_dm_topic_reply_fallback": True, + "direct_messages_topic_id": "20197", "telegram_reply_to_message_id": "462", } From f27416dc80b2419b0a1dc7c3197077fe3e27e311 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:12 -0700 Subject: [PATCH 122/142] fix(cli): include send in _BUILTIN_SUBCOMMANDS for plugin discovery gating --- hermes_cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 662bc57b78d..6ea8dd122fc 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -9631,7 +9631,8 @@ _BUILTIN_SUBCOMMANDS = frozenset( "config", "cron", "curator", "dashboard", "debug", "doctor", "dump", "fallback", "gateway", "hooks", "import", "insights", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", - "model", "pairing", "plugins", "postinstall", "profile", "proxy", "sessions", "setup", + "model", "pairing", "plugins", "postinstall", "profile", "proxy", + "send", "sessions", "setup", "skills", "slack", "status", "tools", "uninstall", "update", "version", "webhook", "whatsapp", "chat", # Help-ish invocations — plugin commands not being listed in From bfcab25dcdb07e639b72cdabe473cbb42edad241 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:44:12 -0700 Subject: [PATCH 123/142] test(tools_config): align post_setup parametrize with current browser provider catalog --- tests/hermes_cli/test_tools_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index 89dc33258a0..787292d83a4 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -1048,9 +1048,6 @@ def test_reconfigure_browser_provider_overwrites_stale_use_gateway(): @pytest.mark.parametrize("provider_name,post_setup_key", [ - ("Browserbase", "agent_browser"), - ("Browser Use", "agent_browser"), - ("Firecrawl", "agent_browser"), ("Camofox", "camofox"), ]) def test_reconfigure_provider_runs_post_setup_for_env_var_providers( From 0b491c466a9493a1522bcdaaa3f7ead96dffe2d2 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:49:38 -0700 Subject: [PATCH 124/142] fix(model_switch): preserve explicit custom-provider model list when no api_key --- hermes_cli/model_switch.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index a5d299165fc..727905270e1 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -1692,7 +1692,22 @@ def list_authenticated_providers( # Ollama servers) — the /models endpoint often works without # auth. The CLI's _model_flow_named_custom always probes, so # the Telegram/Discord picker should do the same for parity. - if api_url: + # Live-discovery policy: + # - With an api_key, the user has explicitly opted into the + # endpoint and live /models is the source of truth — replace + # the (possibly partial) ``models:`` subset configured for + # context-length overrides with the full live catalog. + # This is the Bifrost / aggregator-gateway case. + # - Without an api_key but with an explicit ``models:`` list + # (or top-level ``model:``), the user is narrowing a public + # endpoint to a specific subset (e.g. ollama.com /v1/models + # returns 35 models but the user only wants 4). Preserve the + # explicit list and skip live discovery. + # - Without an api_key AND no explicit models, fall through to + # live discovery so bare-endpoint custom providers (local + # llama.cpp / Ollama servers) still appear populated. + should_probe = bool(api_url) and (bool(api_key) or not grp["models"]) + if should_probe: try: from hermes_cli.models import fetch_api_models From af7b38d78e6f3c37fc7e4a7b3a867b7c8b7ec96d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:49:38 -0700 Subject: [PATCH 125/142] =?UTF-8?q?test(voice=5Fcli):=20drop=20stale=20?= =?UTF-8?q?=E2=89=A51=20requirement=20for=20force=3DTrue=20error=20=5Fvpri?= =?UTF-8?q?nt=20calls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/tools/test_voice_cli_integration.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/tools/test_voice_cli_integration.py b/tests/tools/test_voice_cli_integration.py index 93dffa649a7..a6cf5e36627 100644 --- a/tests/tools/test_voice_cli_integration.py +++ b/tests/tools/test_voice_cli_integration.py @@ -482,8 +482,11 @@ class TestVprintForceParameter: else: unforced_error_count += 1 - assert forced_error_count > 0, \ - "Expected at least one _vprint with force=True for error messages" + # Invariant: no critical-error _vprint call may silently drop under + # streaming suppression — every ❌-prefixed _vprint must pass force=True. + # The codebase may legitimately have zero such calls if errors are + # routed through print() or higher-level Rich panels; what matters is + # that none are quietly suppressed. assert unforced_error_count == 0, \ f"Found {unforced_error_count} critical error _vprint calls without force=True" From 532b209f01b8c70a8dbb75b580b4fc673488ec5d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:49:38 -0700 Subject: [PATCH 126/142] fix(run_agent): scope kimi tool-reasoning trigger to host, not model name substring --- run_agent.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/run_agent.py b/run_agent.py index f25c94f17a9..6e9877a1182 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3601,15 +3601,17 @@ class AIAgent: ``reasoning_content`` on every assistant tool-call message; omitting it causes the next replay to fail with HTTP 400. - Also detects Kimi models served through third-party providers (e.g. - ollama-cloud) by matching ``kimi`` in the model name. + Detection is host-driven, not model-name-driven: aggregators like + OpenRouter that re-export Kimi/Moonshot models speak their own + protocol and reject ``reasoning_content`` echoes. We only enable the + kimi-reasoning replay when the request actually targets a + kimi/moonshot endpoint or the dedicated kimi-coding provider. """ return ( self.provider in {"kimi-coding", "kimi-coding-cn"} or base_url_host_matches(self.base_url, "api.kimi.com") or base_url_host_matches(self.base_url, "moonshot.ai") or base_url_host_matches(self.base_url, "moonshot.cn") - or "kimi" in (self.model or "").lower() ) def _needs_deepseek_tool_reasoning(self) -> bool: From 2551f0813097e2251a19e9281c0f13de898c3798 Mon Sep 17 00:00:00 2001 From: zccyman Date: Sun, 17 May 2026 12:42:06 -0700 Subject: [PATCH 127/142] fix(schema_sanitizer): strip pattern/format from Responses-format tools for xAI compatibility xAI's /responses endpoint rejects pattern and format JSON Schema keywords in tool schemas with HTTP 400 'Invalid arguments passed to the model'. The existing strip_pattern_and_format() only walked OpenAI-format tools ({'function': {'parameters': ...}}), missing Responses-format shapes ({'name': ..., 'parameters': ...}) used by codex_responses API mode. This shows up most often with MCP-derived tools that carry validation keywords (e.g. domain pattern regex in firecrawl, format: date-time) through to the wire. Extends the walk to handle both shapes. Auto-strip wiring is applied separately in chat_completion_helpers (post-refactor location). Closes #27197 --- tests/tools/test_schema_sanitizer.py | 131 +++++++++++++++++++++++++++ tools/schema_sanitizer.py | 14 ++- 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_schema_sanitizer.py b/tests/tools/test_schema_sanitizer.py index 89fbcd91d2b..8c865e87b8d 100644 --- a/tests/tools/test_schema_sanitizer.py +++ b/tests/tools/test_schema_sanitizer.py @@ -304,6 +304,30 @@ def test_strip_none_returns_zero(): assert stripped == 0 + +def test_strip_responses_format_strips_format_keyword(): + """Responses-format: keyword should be stripped.""" + from tools.schema_sanitizer import strip_pattern_and_format + + tools = [ + { + "name": "get_event", + "parameters": { + "type": "object", + "properties": { + "ts": {"type": "string", "format": "date-time"}, + } + }, + "type": "function" + } + ] + + result, stripped = strip_pattern_and_format(tools) + assert stripped == 1, f"Expected 1 format stripped, got {stripped}" + assert "format" not in result[0]["parameters"]["properties"]["ts"], "format should be stripped" + assert result[0]["parameters"]["properties"]["ts"]["type"] == "string", "type should be preserved" + + def test_top_level_allof_stripped_for_codex_backend_compat(): """OpenAI Codex backend rejects top-level allOf/oneOf/anyOf/enum/not.""" tools = [_tool("memory", { @@ -360,3 +384,110 @@ def test_nested_allof_preserved(): nested = out[0]["function"]["parameters"]["properties"]["config"] assert "allOf" in nested assert nested["allOf"] == [{"required": ["mode"]}] + + +def test_strip_responses_format_tools(): + """strip_pattern_and_format should handle Responses-format tools (no function wrapper).""" + from tools.schema_sanitizer import strip_pattern_and_format + + # Responses-format: {"name": "...", "parameters": {...}, "type": "function"} + tools = [ + { + "name": "mcp_firecrawl_search", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "includeDomains": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$" + } + } + } + }, + "type": "function" + } + ] + + result, stripped = strip_pattern_and_format(tools) + assert stripped == 1, f"Expected 1 pattern stripped, got {stripped}" + + # Verify pattern keyword was removed from includeDomains + domains = result[0]["parameters"]["properties"]["includeDomains"]["items"] + assert "pattern" not in domains, f"pattern should be stripped: {domains}" + assert domains["type"] == "string", "type should be preserved" + + +def test_strip_responses_idempotent(): + """Second call on already-stripped Responses-format tools should return 0.""" + from tools.schema_sanitizer import strip_pattern_and_format + + tools = [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string"} # This is a property named pattern, NOT schema keyword + } + } + } + ] + + # Pass 1 - property named 'pattern' should NOT be stripped + result, first = strip_pattern_and_format(tools) + assert first == 0, f"Expected 0 stripped (property pattern preserved), got {first}" + assert "pattern" in result[0]["parameters"]["properties"], "property named pattern should survive" + + # Pass 2 - idempotent + _, second = strip_pattern_and_format(tools) + assert second == 0, f"Expected 0 on second pass, got {second}" + + +def test_strip_responses_mixed_formats(): + """Mixed list of OpenAI-format and Responses-format tools should both be sanitized.""" + from tools.schema_sanitizer import strip_pattern_and_format + + tools = [ + # OpenAI-format: {"function": {"parameters": {...}}} + { + "type": "function", + "function": { + "name": "search", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "pattern": "^[a-z]+$"} + } + } + } + }, + # Responses-format: {"name": "...", "parameters": {...}} + { + "name": "get_time", + "parameters": { + "type": "object", + "properties": { + "tz": {"type": "string", "format": "date-time"} + } + }, + "type": "function" + } + ] + + result, stripped = strip_pattern_and_format(tools) + assert stripped == 2, f"Expected 2 stripped (1 pattern + 1 format), got {stripped}" + + # OpenAI-format tool: pattern stripped from parameters + openai_params = result[0]["function"]["parameters"]["properties"]["query"] + assert "pattern" not in openai_params, f"pattern should be stripped: {openai_params}" + + # Responses-format tool: format stripped + resp_params = result[1]["parameters"]["properties"]["tz"] + assert "format" not in resp_params, f"format should be stripped: {resp_params}" + + # Verify structure preserved + assert result[0]["function"]["parameters"]["type"] == "object" + assert result[1]["parameters"]["type"] == "object" diff --git a/tools/schema_sanitizer.py b/tools/schema_sanitizer.py index 87587c7fed5..0d03998d366 100644 --- a/tools/schema_sanitizer.py +++ b/tools/schema_sanitizer.py @@ -355,11 +355,23 @@ def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]: _walk(item) for tool in tools: - fn = tool.get("function") if isinstance(tool, dict) else None + if not isinstance(tool, dict): + continue + + # OpenAI-format: {"function": {"parameters": {...}}} + fn = tool.get("function") if isinstance(fn, dict): params = fn.get("parameters") if isinstance(params, dict): _walk(params) + continue + + # Responses-format: {"name": "...", "parameters": {...}} + # (used by codex_responses API mode — xAI, OpenAI Codex, etc.) + params = tool.get("parameters") + if isinstance(params, dict): + _walk(params) + continue if stripped: logger.info( From bdc2113b5cdd37cedc033547f0361acbc326fd34 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 12:42:13 -0700 Subject: [PATCH 128/142] fix(xai): wire schema sanitizer into post-refactor build_api_kwargs Port of the run_agent.py changes from #27219 to current main: the _build_api_kwargs body was extracted into agent/chat_completion_helpers. build_api_kwargs, so wire the xAI tool-schema sanitization there (provider in {'xai', 'xai-oauth'} or base_url=api.x.ai). Logs a warning instead of silently swallowing exceptions, matching the contributor's review-followup fix. Co-authored-by: zccyman --- agent/chat_completion_helpers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e536db95eb1..ee5b957bf2f 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -286,6 +286,21 @@ def build_api_kwargs(agent, api_messages: list) -> dict: ) is_xai_responses = agent.provider in {"xai", "xai-oauth"} or agent._base_url_hostname == "api.x.ai" _msgs_for_codex = agent._prepare_messages_for_non_vision_model(api_messages) + + # xAI's /responses endpoint rejects ``pattern`` and ``format`` keywords + # in tool schemas (HTTP 400 "Invalid arguments passed to the model"). + # Most commonly hit when MCP-derived tools carry JSON Schema validation + # keywords through. Strip them before building kwargs. See #27197. + if is_xai_responses: + try: + from tools.schema_sanitizer import strip_pattern_and_format + tools_for_api, _ = strip_pattern_and_format(tools_for_api) + except Exception as exc: + logger.warning( + "%s⚠️ Failed to sanitize tool schemas for xAI: %s", + getattr(agent, "log_prefix", ""), exc, + ) + return _ct.build_kwargs( model=agent.model, messages=_msgs_for_codex, From 04b4f765cc9fe51a60e0d962e1d41e703453978b Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 13:33:16 -0700 Subject: [PATCH 129/142] fix(mcp): use module-level time so test patches do not race background sleepers --- tests/tools/test_mcp_stability.py | 4 ++-- tools/mcp_tool.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/tools/test_mcp_stability.py b/tests/tools/test_mcp_stability.py index 238696feba2..163a05963e0 100644 --- a/tests/tools/test_mcp_stability.py +++ b/tests/tools/test_mcp_stability.py @@ -135,7 +135,7 @@ class TestStdioPidTracking: # bpo-14484). Return True so the SIGKILL escalation fires. with patch("tools.mcp_tool.os.kill") as mock_kill, \ patch("gateway.status._pid_exists", return_value=True), \ - patch("time.sleep") as mock_sleep: + patch("tools.mcp_tool.time.sleep") as mock_sleep: _kill_orphaned_mcp_children() # SIGTERM then SIGKILL; the alive check no longer touches os.kill. @@ -163,7 +163,7 @@ class TestStdioPidTracking: monkeypatch.delattr(signal, "SIGKILL", raising=False) with patch("tools.mcp_tool.os.kill") as mock_kill, \ - patch("time.sleep") as mock_sleep: + patch("tools.mcp_tool.time.sleep") as mock_sleep: _kill_orphaned_mcp_children() # SIGTERM phase, alive check raises (process gone), no escalation diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index e1d87389d42..e50efc05a0c 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -3518,7 +3518,6 @@ def _kill_orphaned_mcp_children(include_active: bool = False) -> None: sessions can still be in flight. """ import signal as _signal - import time as _time with _lock: pids: Dict[int, str] = {} @@ -3543,7 +3542,7 @@ def _kill_orphaned_mcp_children(include_active: bool = False) -> None: pass # Phase 2: Wait for graceful exit - _time.sleep(2) + time.sleep(2) # Phase 3: SIGKILL any survivors _sigkill = getattr(_signal, "SIGKILL", _signal.SIGTERM) From 1345dda0cf4559a72f2a427e103d1b78e4fc9677 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 13:54:12 -0700 Subject: [PATCH 130/142] feat(kanban): orchestrator-driven auto-decomposition on triage (#27572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kanban): orchestrator-driven auto-decomposition on triage Closes the core gap in the kanban system: dropping a one-liner into Triage now decomposes it into a graph of child tasks routed to specialist profiles by description, matching teknium's original vision ("main orchestrator splits/creates actual tasks, doles them out to each agent"). The build --------- - hermes_cli/profiles.py: new `description` + `description_auto` fields on ProfileInfo, persisted in /profile.yaml. Helpers read_profile_meta / write_profile_meta. `create_profile` accepts optional description. - hermes_cli/profile_describer.py: new module — auto-generate a 1-2 sentence description from a profile's skills + model + name via the auxiliary LLM (`auxiliary.profile_describer`). - hermes_cli/main.py: new `hermes profile create --description ...` flag; new `hermes profile describe [name] [--text ... | --auto | --all --auto]` subcommand. - hermes_cli/kanban_db.py: new `decompose_triage_task` atomic helper — creates N child tasks, links the root as a child of every leaf (root waits for the whole graph), flips root `triage -> todo` with orchestrator assignee, records an audit comment + `decomposed` event in a single write_txn. - hermes_cli/kanban_decompose.py: new module — calls the auxiliary LLM (`auxiliary.kanban_decomposer`) with the profile roster + descriptions to produce a JSON task graph, then invokes the DB helper. Rewrites unknown assignees to the configured `kanban.default_assignee` (or the active default profile) so a task NEVER lands with assignee=None. Falls back to specify-style single-task promotion when the LLM returns `fanout: false`. - hermes_cli/kanban.py: new `hermes kanban decompose [task_id | --all]` CLI verb. - hermes_cli/config.py: new DEFAULT_CONFIG keys — kanban.orchestrator_profile, kanban.default_assignee, kanban.auto_decompose (default True), kanban.auto_decompose_per_tick (default 3), auxiliary.kanban_decomposer, auxiliary.profile_describer. - gateway/run.py: kanban dispatcher watcher now runs auto-decompose before each `_tick_once`, capped by `auto_decompose_per_tick` so a bulk-load of triage tasks doesn't burst-spend the aux LLM. - plugins/kanban/dashboard/plugin_api.py: new endpoints — GET /profiles (list roster + descriptions), PATCH /profiles/ (set description, user-authored), POST /profiles//describe-auto (LLM-generate), POST /tasks//decompose (run decomposer), GET/PUT /orchestration (orchestrator/default-assignee/auto-decompose pickers, with resolved fallbacks echoed back). - plugins/kanban/dashboard/dist/index.js: new OrchestrationPanel collapsible — dropdowns for orchestrator profile and default assignee, auto-decompose toggle, per-profile description editor with Save and Auto-generate buttons. New ⚗ Decompose button next to ✨ Specify on triage-column task drawers. Behavior -------- - A task in Triage gets fanned out into a small DAG of child tasks. Children with no internal parents flip to `ready` immediately (parallel dispatch). Children with sibling parents wait. The root stays alive as a parent of every child — when the whole graph finishes, it promotes to `ready` and the orchestrator profile wakes back up to judge completion (the "adds more tasks until done" part of the original vision). - `kanban.orchestrator_profile` unset -> falls back to the default profile (whichever `hermes` launches with no -p flag). - `kanban.default_assignee` unset -> same fallback. Tasks NEVER end up unassigned. - `kanban.auto_decompose=true` (default) runs the decomposer automatically on dispatcher ticks; manual `hermes kanban decompose` is always available. Tests ----- - tests/hermes_cli/test_kanban_decompose_db.py — 7 tests for the atomic DB helper (status transitions, dep graph, audit trail, validation errors). - tests/hermes_cli/test_kanban_decompose.py — 6 tests for the decomposer module (fanout, no-fanout fallback, unknown-assignee rewrite, malformed-JSON resilience, no-aux-client path). - tests/hermes_cli/test_profile_describer.py — 10 tests for profile.yaml r/w + the LLM auto-describer (yaml corrupt tolerance, user-vs-auto description protection, --overwrite, fallback parsing). E2E --- - CLI end-to-end: created profiles with descriptions, dropped a triage task, mocked the aux LLM with a 3-task graph -> verified all three children were created with the right assignees, the dependency edges matched the LLM's graph, root flipped to todo gated by every child, audit comment + `decomposed` event recorded. - Dashboard end-to-end: started the dashboard against an isolated HERMES_HOME, verified all four new endpoints via curl (profile listing, PATCH for description, PUT for orchestration settings, POST for decompose). Opened the UI in the browser, confirmed the OrchestrationPanel renders with all three pickers + the per-profile description editor, typed a description, clicked Save, verified ~/.hermes/profile.yaml was written. Clicked Decompose on the triage card and confirmed the inline error message surfaced as designed ("no auxiliary client configured"). * feat(kanban): surface decompose mode (Auto/Manual) as a one-click pill The auto/manual toggle already existed as kanban.auto_decompose (default true), but it was buried inside the collapsed Orchestration settings panel — users couldn't tell at a glance which mode they were in. This hoists it to a pill at the top of the kanban page so the state is always visible and one click flips it. UX - New "⚗ Decompose: AUTO|MANUAL" pill in the kanban header. Emerald styling when Auto is on (the default), muted/gray when Manual. - Pill is visible both in the collapsed AND expanded Orchestration settings views so context is preserved when the user opens the panel. - Tooltip explains both states + what clicking does. - Renamed the in-panel "Auto-decompose on triage / Enabled" checkbox to "Decompose mode / Auto (default) | Manual" for language parity with the pill. Behavior preserved - Default remains Auto (kanban.auto_decompose=true). - Manual mode restores pre-PR behavior: triage tasks stay in triage until the user clicks ⚗ Decompose on each card (or runs `hermes kanban decompose `). Implementation - plugins/kanban/dashboard/dist/index.js: load /orchestration on mount (not just on expand) so the collapsed pill reflects real state. Render mode pill in both collapsed and expanded headers. Reuses the existing PUT /api/plugins/kanban/orchestration endpoint — no new backend, no new tests required. E2E verified - Pill renders as "⚗ Decompose: AUTO" on page load (default). - One click flips to "⚗ Decompose: MANUAL" with muted styling. - config.yaml on disk shows auto_decompose: false after the flip. - Second click round-trips back to Auto; config.yaml flips to true. * feat(kanban): rename mode pill to "Orchestration: Auto/Manual" Per Teknium feedback — "Decompose" was too implementation-specific. "Orchestration" is the user-facing concept (the whole pitch is the orchestrator profile routing work), and the pill is the front door to it. - Pill text: "Orchestration: Auto" / "Orchestration: Manual" (title case, no ⚗ prefix, no SHOUTY-CAPS for the mode value) - In-panel checkbox label: "Orchestration mode" (was "Decompose mode") - Tooltips updated to match - No behavior change * docs(kanban): document decompose, profile descriptions, orchestration mode Brings the docs site up to parity with the PR. English build verified locally (npx docusaurus build --locale en) — clean, no new broken links or anchors. Pre-existing broken-link warnings (rl-training, llms.txt, step-by-step-checklist, fallback-model) untouched. - website/docs/reference/cli-commands.md + `hermes kanban decompose` action row in the action table, with pointer to the Auto vs Manual orchestration section. - website/docs/reference/profile-commands.md + `--description ""` flag on `hermes profile create`. + Full `hermes profile describe` section: read, --text, --auto, --overwrite, --all flags with examples. - website/docs/user-guide/features/kanban.md (the big one) + Triage column intro rewritten around the Auto-decompose default behavior, with pointer to the new Auto vs Manual section. + Status action row updated to mention both ⚗ Decompose and ✨ Specify on triage cards. + New "Auto vs Manual orchestration" section explaining the two modes, how to flip them (pill, config), how routing-by-description works, the no-None-assignee guarantee, plus a config knob table (auto_decompose, auto_decompose_per_tick, orchestrator_profile, default_assignee) and the two new auxiliary slots (kanban_decomposer, profile_describer). + REST surface table gains 6 new endpoint rows: /tasks/:id/decompose, /profiles (GET), /profiles/:name (PATCH), /profiles/:name/describe-auto, /orchestration (GET + PUT). - website/docs/user-guide/features/kanban-tutorial.md + Triage column blurb updated for Auto by default + Manual via the pill, with cross-link to the Auto vs Manual orchestration section. - website/docs/user-guide/profiles.md + Blank-profile flow now mentions --description and points to the kanban routing model for context. - website/docs/user-guide/configuration.md + `kanban_decomposer` and `profile_describer` added to the `hermes model -> Configure auxiliary models` menu listing. --- gateway/run.py | 95 ++++ hermes_cli/config.py | 44 ++ hermes_cli/kanban.py | 119 +++++ hermes_cli/kanban_db.py | 174 +++++++ hermes_cli/kanban_decompose.py | 440 ++++++++++++++++++ hermes_cli/main.py | 143 ++++++ hermes_cli/profile_describer.py | 299 ++++++++++++ hermes_cli/profiles.py | 107 +++++ plugins/kanban/dashboard/dist/index.js | 359 ++++++++++++++ plugins/kanban/dashboard/plugin_api.py | 273 +++++++++++ tests/hermes_cli/test_kanban_decompose.py | 242 ++++++++++ tests/hermes_cli/test_kanban_decompose_db.py | 152 ++++++ tests/hermes_cli/test_profile_describer.py | 168 +++++++ website/docs/reference/cli-commands.md | 1 + website/docs/reference/profile-commands.md | 35 ++ website/docs/user-guide/configuration.md | 2 + .../user-guide/features/kanban-tutorial.md | 2 +- website/docs/user-guide/features/kanban.md | 38 +- website/docs/user-guide/profiles.md | 8 + 19 files changed, 2698 insertions(+), 3 deletions(-) create mode 100644 hermes_cli/kanban_decompose.py create mode 100644 hermes_cli/profile_describer.py create mode 100644 tests/hermes_cli/test_kanban_decompose.py create mode 100644 tests/hermes_cli/test_kanban_decompose_db.py create mode 100644 tests/hermes_cli/test_profile_describer.py diff --git a/gateway/run.py b/gateway/run.py index a0ab84e850d..818bd282ddb 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4763,11 +4763,106 @@ class GatewayRunner: pass return False + # Auto-decompose: turn fresh triage tasks into ready workgraphs + # before the dispatcher fans out workers. Gated by + # ``kanban.auto_decompose`` (default True). Capped by + # ``kanban.auto_decompose_per_tick`` (default 3) so a bulk-load + # of triage tasks doesn't burst-spend the aux LLM in one tick; + # remainder defers to subsequent ticks. + auto_decompose_enabled = bool(kanban_cfg.get("auto_decompose", True)) + try: + auto_decompose_per_tick = int( + kanban_cfg.get("auto_decompose_per_tick", 3) or 3 + ) + except (TypeError, ValueError): + auto_decompose_per_tick = 3 + if auto_decompose_per_tick < 1: + auto_decompose_per_tick = 1 + + def _auto_decompose_tick() -> int: + """Run the auto-decomposer for up to N triage tasks across all + boards. Returns the number of triage tasks that were + successfully decomposed or specified this tick. + """ + try: + from hermes_cli import kanban_decompose as _decomp + except Exception as exc: # pragma: no cover + logger.warning( + "kanban auto-decompose: import failed (%s); skipping", exc, + ) + return 0 + try: + boards = _kb.list_boards(include_archived=False) + except Exception: + boards = [_kb.read_board_metadata(_kb.DEFAULT_BOARD)] + attempted = 0 + successes = 0 + for b in boards: + slug = b.get("slug") or _kb.DEFAULT_BOARD + if attempted >= auto_decompose_per_tick: + break + # Pin this board for the duration of the call — same + # pattern as the dashboard specify endpoint. The + # decomposer module connects with no board kwarg and + # relies on the env var. + prev_env = os.environ.get("HERMES_KANBAN_BOARD") + try: + os.environ["HERMES_KANBAN_BOARD"] = slug + try: + triage_ids = _decomp.list_triage_ids() + except Exception as exc: + logger.debug( + "kanban auto-decompose: list_triage_ids failed on board %s (%s)", + slug, exc, + ) + triage_ids = [] + for tid in triage_ids: + if attempted >= auto_decompose_per_tick: + break + attempted += 1 + try: + outcome = _decomp.decompose_task( + tid, author="auto-decomposer", + ) + except Exception: + logger.exception( + "kanban auto-decompose: decompose_task crashed on %s", + tid, + ) + continue + if outcome.ok: + successes += 1 + if outcome.fanout and outcome.child_ids: + logger.info( + "kanban auto-decompose [%s]: %s → %d children", + slug, tid, len(outcome.child_ids), + ) + else: + logger.info( + "kanban auto-decompose [%s]: %s → single task (no fanout)", + slug, tid, + ) + else: + # Common no-op reasons (no aux client configured) shouldn't + # spam logs every tick. Log at debug. + logger.debug( + "kanban auto-decompose [%s]: %s skipped: %s", + slug, tid, outcome.reason, + ) + finally: + if prev_env is None: + os.environ.pop("HERMES_KANBAN_BOARD", None) + else: + os.environ["HERMES_KANBAN_BOARD"] = prev_env + return successes + logger.info( "kanban dispatcher: embedded in gateway (interval=%.1fs)", interval ) while self._running: try: + if auto_decompose_enabled: + await asyncio.to_thread(_auto_decompose_tick) results = await asyncio.to_thread(_tick_once) any_spawned = False for slug, res in (results or []): diff --git a/hermes_cli/config.py b/hermes_cli/config.py index e4447183746..3f9bdd69ed4 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -925,6 +925,31 @@ DEFAULT_CONFIG = { "timeout": 120, "extra_body": {}, }, + # Kanban decomposer — decomposes a triage task into a graph of + # child tasks routed to specialist profiles by description. + # Invoked by ``hermes kanban decompose`` and the kanban + # auto-decompose dispatcher tick. Returns a JSON task graph; + # uses more tokens than the specifier so allow more headroom. + "kanban_decomposer": { + "provider": "auto", + "model": "", + "base_url": "", + "api_key": "", + "timeout": 180, + "extra_body": {}, + }, + # Profile describer — auto-generates a 1-2 sentence description + # of what a profile is good at. Invoked by + # ``hermes profile describe --auto`` and the dashboard's + # auto-generate button. Short, cheap call. + "profile_describer": { + "provider": "auto", + "model": "", + "base_url": "", + "api_key": "", + "timeout": 60, + "extra_body": {}, + }, # Curator — skill-usage review fork. Timeout is generous because the # review pass can take several minutes on reasoning models (umbrella # building over hundreds of candidate skills). "auto" = use main chat @@ -1466,6 +1491,25 @@ DEFAULT_CONFIG = { # same task/profile (spawn_failed, timed_out, or crashed). Reassignment # resets the streak for the new profile. "failure_limit": 2, + # Profile that decomposes tasks in the Triage column. When unset, + # falls back to the default profile (the one `hermes` launches with + # no -p flag). Set this to a dedicated 'orchestrator' profile if you + # want decomposition to use a different model/skills from your main + # working profile. + "orchestrator_profile": "", + # Where a child task lands if the orchestrator can't match an + # assignee to any installed profile. When unset, falls back to the + # default profile. A task never ends up with assignee=None. + "default_assignee": "", + # When true, the kanban dispatcher auto-runs the decomposer on + # tasks that land in Triage (every dispatcher tick). When false, + # decomposition is manual via `hermes kanban decompose ` or + # the dashboard's Decompose button. + "auto_decompose": True, + # Max triage tasks to decompose per dispatcher tick. Prevents a + # large bulk-load of triage tasks from spending a burst of aux + # LLM calls in one tick. Excess tasks defer to the next tick. + "auto_decompose_per_tick": 3, }, # execute_code settings — controls the tool used for programmatic tool calls. diff --git a/hermes_cli/kanban.py b/hermes_cli/kanban.py index b4024e2e70e..55b1d4125a2 100644 --- a/hermes_cli/kanban.py +++ b/hermes_cli/kanban.py @@ -610,6 +610,43 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu help="Emit one JSON object per task on stdout", ) + # --- decompose --- (triage → fan-out via auxiliary LLM + orchestrator) + p_decompose = sub.add_parser( + "decompose", + help="Decompose a triage-column task into a graph of child tasks " + "routed to specialist profiles by description. Falls back to " + "specify-style single-task promotion when the task doesn't " + "benefit from fan-out. Uses auxiliary.kanban_decomposer.", + ) + p_decompose.add_argument( + "task_id", + nargs="?", + default=None, + help="Task id to decompose (required unless --all is given)", + ) + p_decompose.add_argument( + "--all", + dest="all_triage", + action="store_true", + help="Decompose every task currently in the triage column", + ) + p_decompose.add_argument( + "--tenant", + default=None, + help="When used with --all, restrict the sweep to this tenant", + ) + p_decompose.add_argument( + "--author", + default=None, + help="Author name recorded on the audit comment " + "(default: $HERMES_PROFILE or 'decomposer')", + ) + p_decompose.add_argument( + "--json", + action="store_true", + help="Emit one JSON object per task on stdout", + ) + # --- gc --- p_gc = sub.add_parser( "gc", help="Garbage-collect archived-task workspaces, old events, and old logs", @@ -740,6 +777,7 @@ def kanban_command(args: argparse.Namespace) -> int: "notify-unsubscribe": _cmd_notify_unsubscribe, "context": _cmd_context, "specify": _cmd_specify, + "decompose": _cmd_decompose, "gc": _cmd_gc, } handler = handlers.get(action) @@ -2115,6 +2153,87 @@ def _cmd_specify(args: argparse.Namespace) -> int: return 0 if (ok_count > 0 or not ids) else 1 +def _cmd_decompose(args: argparse.Namespace) -> int: + """Fan a triage task (or all of them) out into a graph of child + tasks via the auxiliary LLM, routed to specialist profiles by + description. Thin wrapper over ``kanban_decompose``.""" + from hermes_cli import kanban_decompose as decomp + + all_flag = bool(getattr(args, "all_triage", False)) + tenant = getattr(args, "tenant", None) + author = getattr(args, "author", None) or _profile_author() + want_json = bool(getattr(args, "json", False)) + + if args.task_id and all_flag: + print( + "kanban: pass either a task id OR --all, not both", + file=sys.stderr, + ) + return 2 + + if all_flag: + ids = decomp.list_triage_ids(tenant=tenant) + if not ids: + msg = ( + "No triage tasks" + + (f" for tenant {tenant!r}" if tenant else "") + + "." + ) + if want_json: + print(json.dumps({"decomposed": 0, "total": 0})) + else: + print(msg) + return 0 + elif args.task_id: + ids = [args.task_id] + else: + print( + "kanban: decompose requires a task id or --all", + file=sys.stderr, + ) + return 2 + + ok_count = 0 + for tid in ids: + outcome = decomp.decompose_task(tid, author=author) + if outcome.ok: + ok_count += 1 + if want_json: + print(json.dumps({ + "task_id": outcome.task_id, + "ok": outcome.ok, + "reason": outcome.reason, + "fanout": outcome.fanout, + "child_ids": outcome.child_ids, + "new_title": outcome.new_title, + })) + elif outcome.ok: + if outcome.fanout and outcome.child_ids: + child_summary = ", ".join(outcome.child_ids) + print( + f"Decomposed {outcome.task_id} → {len(outcome.child_ids)} " + f"children ({child_summary}); root promoted to todo" + ) + else: + title_suffix = ( + f" — retitled: {outcome.new_title!r}" + if outcome.new_title + else "" + ) + print( + f"Specified {outcome.task_id} → todo " + f"(no fanout){title_suffix}" + ) + else: + print( + f"kanban: decompose {outcome.task_id}: {outcome.reason}", + file=sys.stderr, + ) + if not all_flag: + return 0 if ok_count == 1 else 1 + return 0 if (ok_count > 0 or not ids) else 1 + + def _cmd_gc(args: argparse.Namespace) -> int: """Remove scratch workspaces of archived tasks, prune old events, and delete old worker logs.""" diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 9d5ddad6ed0..4bd4827e386 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -2777,6 +2777,180 @@ def specify_triage_task( return True +def decompose_triage_task( + conn: sqlite3.Connection, + task_id: str, + *, + root_assignee: Optional[str], + children: list[dict], + author: Optional[str] = None, +) -> Optional[list[str]]: + """Fan a triage task out into child tasks and promote the root to ``todo``. + + The root task stays alive and becomes the parent of every child — + when all children reach ``done``, the root promotes to ``ready`` and + its assignee (typically the orchestrator profile) wakes back up to + judge completion or spawn more work. + + ``children`` is a list of dicts, each shaped like:: + + { + "title": "...", + "body": "...", # optional + "assignee": "profile-name", # optional, None -> default fallback + "parents": [0, 2], # indices into this same children list + } + + Returns the list of created child task ids (in input order) on + success. Returns ``None`` when: + - The root task does not exist + - The root task is not in ``triage`` + - A cycle would result (caller built a bad graph) + + Validation of titles/assignees happens inside the same write_txn as + the inserts so a malformed entry aborts the whole decomposition + cleanly (no orphan children). + """ + if not children: + return None + if root_assignee is not None: + root_assignee = _canonical_assignee(root_assignee) + + # Pre-validate the children list shape outside the txn. Cheap checks + # that don't need DB access. Bad input aborts before we touch the DB. + for idx, child in enumerate(children): + if not isinstance(child, dict): + raise ValueError(f"child[{idx}] is not a dict") + title = child.get("title") + if not isinstance(title, str) or not title.strip(): + raise ValueError(f"child[{idx}].title is required") + parents_idx = child.get("parents") or [] + if not isinstance(parents_idx, list): + raise ValueError(f"child[{idx}].parents must be a list") + for p in parents_idx: + if not isinstance(p, int) or p < 0 or p >= len(children): + raise ValueError( + f"child[{idx}].parents[{p}] is not a valid index into children" + ) + if p == idx: + raise ValueError(f"child[{idx}] cannot list itself as a parent") + + # We do the full decomposition in a SINGLE write_txn so it's + # atomic: either every child is created AND the root flips to + # ``todo``, or nothing changes. We deliberately do NOT call any + # kb helper that opens its own write_txn (create_task, link_tasks, + # add_comment) from inside this block — see architecture.md + # write_txn pitfalls. Instead we inline the INSERTs and + # _append_event calls. + now = int(time.time()) + child_ids: list[str] = [] + with write_txn(conn): + root_row = conn.execute( + "SELECT id, status, tenant FROM tasks WHERE id = ?", (task_id,) + ).fetchone() + if root_row is None: + return None + if root_row["status"] != "triage": + return None + tenant = root_row["tenant"] + + # Create children. Status is 'todo' regardless of parents — we + # link them under the root AFTER creation so the dispatcher + # sees a coherent state, and recompute_ready() at the end + # promotes parent-free children to 'ready'. + for idx, child in enumerate(children): + new_id = _new_task_id() + title = child["title"].strip() + body = child.get("body") + assignee = _canonical_assignee(child.get("assignee")) + conn.execute( + "INSERT INTO tasks " + "(id, title, body, assignee, status, workspace_kind, " + " tenant, created_at, created_by) " + "VALUES (?, ?, ?, ?, 'todo', 'scratch', ?, ?, ?)", + ( + new_id, + title, + body if isinstance(body, str) else None, + assignee, + tenant, + now, + (author or "decomposer"), + ), + ) + _append_event( + conn, new_id, "created", + {"by": author or "decomposer", "from_decompose_of": task_id}, + ) + child_ids.append(new_id) + + # Link children to their sibling parents (within the decomposed graph). + for idx, child in enumerate(children): + for p_idx in child.get("parents") or []: + parent_id = child_ids[p_idx] + child_id = child_ids[idx] + conn.execute( + "INSERT OR IGNORE INTO task_links (parent_id, child_id) " + "VALUES (?, ?)", + (parent_id, child_id), + ) + _append_event( + conn, child_id, "linked", + {"parent": parent_id, "child": child_id}, + ) + + # Link the ROOT task as a child of every leaf child — i.e. the + # root waits for the whole graph. Simpler than computing leaves: + # link root under every child. Cycle-free because the root is + # only ever a child here, never a parent of children. + for cid in child_ids: + conn.execute( + "INSERT OR IGNORE INTO task_links (parent_id, child_id) " + "VALUES (?, ?)", + (cid, task_id), + ) + + # Flip the root: triage -> todo, set assignee to the orchestrator. + sets = ["status = 'todo'"] + params: list[Any] = [] + if root_assignee is not None: + sets.append("assignee = ?") + params.append(root_assignee) + params.append(task_id) + conn.execute( + f"UPDATE tasks SET {', '.join(sets)} WHERE id = ?", + tuple(params), + ) + + # Audit comment + event on the root so the timeline shows the fan-out. + if author and author.strip(): + conn.execute( + "INSERT INTO task_comments (task_id, author, body, created_at) " + "VALUES (?, ?, ?, ?)", + ( + task_id, + author.strip(), + "Decomposed into " + + ", ".join(child_ids) + + ". Root will wake when all children complete.", + now, + ), + ) + _append_event( + conn, task_id, "decomposed", + { + "child_ids": child_ids, + "root_assignee": root_assignee, + }, + ) + + # Outside the write_txn: promote parent-free children to 'ready' + # so the dispatcher picks them up on its next tick. Same pattern + # specify_triage_task uses. + recompute_ready(conn) + return child_ids + + def archive_task(conn: sqlite3.Connection, task_id: str) -> bool: with write_txn(conn): cur = conn.execute( diff --git a/hermes_cli/kanban_decompose.py b/hermes_cli/kanban_decompose.py new file mode 100644 index 00000000000..2ebe3f04c6e --- /dev/null +++ b/hermes_cli/kanban_decompose.py @@ -0,0 +1,440 @@ +"""Kanban decomposer — fan a triage task out into a graph of child tasks. + +Invoked by ``hermes kanban decompose [task_id | --all]`` and the +auto-decompose path in the gateway dispatcher loop. Reads the user's +profile roster (with descriptions) and asks the auxiliary LLM to +return a task graph in JSON. Then atomically creates the children, +links them under the root, and flips the root ``triage -> todo``. + +The root task stays alive and becomes the parent of every leaf child, +so when the whole graph completes the root wakes back up — its +assignee (the orchestrator profile) gets a chance to judge completion +and add more tasks if the work isn't done yet. + +Design notes +------------ + +* Mirrors the shape of ``hermes_cli/kanban_specify.py``: lazy aux + client import inside the function, lenient response parse, never + raises on expected failure modes. + +* The system prompt sees the *configured* profile roster — names plus + descriptions plus the default fallback. Profiles without a + description are still listed (with a note) so the orchestrator can + match on name as a fallback, but the user has an obvious incentive + to describe them. + +* ``fanout=false`` collapses to the same effect as ``kanban specify``: + we tighten the body and flip ``triage -> todo`` as a single task, + no children created. This makes ``decompose`` a strict superset of + ``specify`` from the user's perspective. + +* If the LLM picks an assignee that doesn't exist as a profile, we + rewrite it to the configured ``default_assignee`` (or the default + profile if unset). A child task NEVER ends up with ``assignee=None``. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from dataclasses import dataclass +from typing import Optional + +from hermes_cli import kanban_db as kb +from hermes_cli import profiles as profiles_mod + +logger = logging.getLogger(__name__) + + +_SYSTEM_PROMPT = """You are the Kanban decomposer for the Hermes Agent board. + +A user dropped a rough idea into the Triage column. Your job is to break it +into a small graph of concrete child tasks and route each one to the best- +matching profile from the available roster. + +You will be given: + - The original task title and body + - The list of available profiles (each with name + description) + - The fallback "default_assignee" used when no profile fits + +Output a single JSON object with this exact shape: + + { + "fanout": true, + "rationale": "", + "tasks": [ + { + "title": "", + "body": "", + "assignee": "", + "parents": [, ...] + }, + ... + ] + } + +Rules: + - "parents" is a list of INDICES (0-based) into this same "tasks" list, + expressing actual data dependencies. Tasks with no parents run in + PARALLEL. Tasks with parents wait until every parent completes. + - Prefer parallelism. If two tasks can be done independently, give + them no parents so the dispatcher fans them out at once. + - Use 2-6 tasks for normal work. Don't create 20 tiny tasks. Don't + cram everything into 1 task. + - Pick assignees from the roster by matching the task to the profile's + DESCRIPTION (not just the name). When nothing matches well, use null + and the system will route to the default_assignee. + - Each child task body is what a fresh worker will read with no other + context — be specific about goal, approach, and acceptance criteria. + +When the task is genuinely a single unit of work (no useful decomposition), +return: + + { + "fanout": false, + "rationale": "", + "title": "", + "body": "" + } + +In that case the task stays as one work item, just with a tightened spec. + +No preamble, no closing remarks, no code fences. Output only the JSON object. +""" + + +_USER_TEMPLATE = """Task id: {task_id} +Title: {title} +Body: +{body} + +Available profiles (assignees you may pick from): +{roster} + +Default assignee (used when no profile fits a task): {default_assignee} +""" + + +_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) + + +@dataclass +class DecomposeOutcome: + """Result of decomposing a single triage task.""" + + task_id: str + ok: bool + reason: str = "" + fanout: bool = False + child_ids: list[str] | None = None + new_title: Optional[str] = None + + +def _truncate(text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[: limit - 1] + "…" + + +def _extract_json_blob(raw: str) -> Optional[dict]: + if not raw: + return None + stripped = _FENCE_RE.sub("", raw.strip()) + first = stripped.find("{") + last = stripped.rfind("}") + if first == -1 or last == -1 or last <= first: + return None + candidate = stripped[first : last + 1] + try: + val = json.loads(candidate) + except (ValueError, json.JSONDecodeError): + return None + if not isinstance(val, dict): + return None + return val + + +def _profile_author() -> str: + """Mirror of ``hermes_cli.kanban._profile_author``.""" + return ( + os.environ.get("HERMES_PROFILE") + or os.environ.get("USER") + or "decomposer" + ) + + +def _load_config() -> dict: + try: + from hermes_cli.config import load_config + return load_config() or {} + except Exception: + return {} + + +def _resolve_orchestrator_profile(cfg: dict) -> str: + """Resolve which profile owns decomposition. + + Falls back to the active default profile when ``kanban.orchestrator_profile`` + is unset, so a task is never stranded for lack of an orchestrator. + """ + kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} + explicit = (kanban_cfg.get("orchestrator_profile") or "").strip() + if explicit: + try: + if profiles_mod.profile_exists(explicit): + return explicit + except Exception: + pass + # Fall back to the active default profile. + try: + return profiles_mod.get_active_profile_name() or "default" + except Exception: + return "default" + + +def _resolve_default_assignee(cfg: dict) -> str: + """Resolve which profile catches child tasks the orchestrator can't route.""" + kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} + explicit = (kanban_cfg.get("default_assignee") or "").strip() + if explicit: + try: + if profiles_mod.profile_exists(explicit): + return explicit + except Exception: + pass + try: + return profiles_mod.get_active_profile_name() or "default" + except Exception: + return "default" + + +def _build_roster() -> tuple[list[dict], set[str]]: + """Return (roster_for_prompt, valid_assignee_names). + + Each roster entry is ``{name, description, has_description}``. The + valid-set is used after the LLM responds to rewrite invalid + assignees to the default fallback. + """ + roster: list[dict] = [] + valid: set[str] = set() + try: + all_profiles = profiles_mod.list_profiles() + except Exception as exc: + logger.warning("decompose: failed to list profiles: %s", exc) + return roster, valid + for p in all_profiles: + desc = (p.description or "").strip() + roster.append({ + "name": p.name, + "description": desc or f"(no description; profile named {p.name!r})", + "has_description": bool(desc), + }) + valid.add(p.name) + return roster, valid + + +def _format_roster(roster: list[dict]) -> str: + if not roster: + return " (no profiles installed — decomposer cannot route work)" + lines = [] + for entry in roster: + tag = "" if entry["has_description"] else " ⚠ undescribed" + lines.append(f" - {entry['name']}{tag}: {entry['description']}") + return "\n".join(lines) + + +def decompose_task( + task_id: str, + *, + author: Optional[str] = None, + timeout: Optional[int] = None, +) -> DecomposeOutcome: + """Decompose a triage task into a graph of child tasks. + + Returns an outcome describing what happened. Never raises for + expected failure modes (task not in triage, no aux client + configured, API error, malformed response, decomposer returned + fanout=true with empty task list) — those surface via ``ok=False``. + """ + with kb.connect() as conn: + task = kb.get_task(conn, task_id) + if task is None: + return DecomposeOutcome(task_id, False, "unknown task id") + if task.status != "triage": + return DecomposeOutcome( + task_id, False, f"task is not in triage (status={task.status!r})" + ) + + cfg = _load_config() + orchestrator = _resolve_orchestrator_profile(cfg) + default_assignee = _resolve_default_assignee(cfg) + roster, valid_names = _build_roster() + + try: + from agent.auxiliary_client import ( # type: ignore + get_auxiliary_extra_body, + get_text_auxiliary_client, + ) + except Exception as exc: + logger.debug("decompose: auxiliary client import failed: %s", exc) + return DecomposeOutcome(task_id, False, "auxiliary client unavailable") + + try: + client, model = get_text_auxiliary_client("kanban_decomposer") + except Exception as exc: + logger.debug("decompose: get_text_auxiliary_client failed: %s", exc) + return DecomposeOutcome(task_id, False, "auxiliary client unavailable") + + if client is None or not model: + return DecomposeOutcome(task_id, False, "no auxiliary client configured") + + user_msg = _USER_TEMPLATE.format( + task_id=task.id, + title=_truncate(task.title or "", 400), + body=_truncate(task.body or "(no body)", 4000), + roster=_format_roster(roster), + default_assignee=default_assignee, + ) + + try: + resp = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": user_msg}, + ], + temperature=0.3, + max_tokens=4000, + timeout=timeout or 180, + extra_body=get_auxiliary_extra_body() or None, + ) + except Exception as exc: + logger.info( + "decompose: API call failed for %s (%s)", task_id, exc, + ) + return DecomposeOutcome(task_id, False, f"LLM error: {type(exc).__name__}") + + try: + raw = resp.choices[0].message.content or "" + except Exception: + raw = "" + + parsed = _extract_json_blob(raw) + if parsed is None: + return DecomposeOutcome(task_id, False, "LLM returned malformed JSON") + + fanout = bool(parsed.get("fanout")) + audit_author = author or _profile_author() + + if not fanout: + # Fall back to single-task spec promotion (same effect as specify). + new_title = parsed.get("title") + new_body = parsed.get("body") + title_val = new_title.strip() if isinstance(new_title, str) and new_title.strip() else None + body_val = new_body if isinstance(new_body, str) and new_body.strip() else None + if title_val is None and body_val is None: + return DecomposeOutcome( + task_id, False, "decomposer returned fanout=false with no title/body", + ) + with kb.connect() as conn: + ok = kb.specify_triage_task( + conn, + task_id, + title=title_val, + body=body_val, + author=audit_author, + ) + if not ok: + return DecomposeOutcome( + task_id, False, "task moved out of triage before promotion", + ) + return DecomposeOutcome( + task_id, True, "single task (no fanout)", + fanout=False, new_title=title_val, + ) + + raw_tasks = parsed.get("tasks") or [] + if not isinstance(raw_tasks, list) or not raw_tasks: + return DecomposeOutcome( + task_id, False, "decomposer returned fanout=true with empty tasks list", + ) + + # Rewrite invalid assignees to the default fallback. Never leave a + # task with assignee=None — the user explicitly does not want that. + children: list[dict] = [] + for idx, entry in enumerate(raw_tasks): + if not isinstance(entry, dict): + return DecomposeOutcome( + task_id, False, f"tasks[{idx}] is not an object", + ) + title = entry.get("title") + if not isinstance(title, str) or not title.strip(): + return DecomposeOutcome( + task_id, False, f"tasks[{idx}].title is missing or empty", + ) + body = entry.get("body") + if not isinstance(body, str): + body = "" + assignee = entry.get("assignee") + if not isinstance(assignee, str) or not assignee.strip(): + chosen = default_assignee + elif assignee not in valid_names: + logger.info( + "decompose: task %s child %d picked unknown assignee %r — " + "routing to default_assignee %r", + task_id, idx, assignee, default_assignee, + ) + chosen = default_assignee + else: + chosen = assignee + parents = entry.get("parents") or [] + if not isinstance(parents, list): + parents = [] + # Clean parent indices: drop non-int and out-of-range. + clean_parents = [p for p in parents if isinstance(p, int) and 0 <= p < len(raw_tasks) and p != idx] + children.append({ + "title": title.strip()[:200], + "body": body.strip(), + "assignee": chosen, + "parents": clean_parents, + }) + + try: + with kb.connect() as conn: + child_ids = kb.decompose_triage_task( + conn, + task_id, + root_assignee=orchestrator, + children=children, + author=audit_author, + ) + except ValueError as exc: + return DecomposeOutcome(task_id, False, f"DB rejected graph: {exc}") + except Exception as exc: + logger.exception("decompose: DB error on task %s", task_id) + return DecomposeOutcome(task_id, False, f"DB error: {type(exc).__name__}") + + if child_ids is None: + return DecomposeOutcome( + task_id, False, "task moved out of triage before decomposition", + ) + + return DecomposeOutcome( + task_id, True, f"decomposed into {len(child_ids)} children", + fanout=True, child_ids=child_ids, + ) + + +def list_triage_ids(*, tenant: Optional[str] = None) -> list[str]: + """Return task ids currently in the triage column.""" + with kb.connect() as conn: + rows = kb.list_tasks( + conn, + status="triage", + tenant=tenant, + limit=1000, + ) + return [row.id for row in rows] diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 6ea8dd122fc..575835b2c7d 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -9043,6 +9043,7 @@ def cmd_profile(args): clone_config=clone, no_alias=no_alias, no_skills=no_skills, + description=getattr(args, "description", None), ) print(f"\nProfile '{name}' created at {profile_dir}") @@ -9142,6 +9143,107 @@ def cmd_profile(args): print(f"Error: {e}") sys.exit(1) + elif action == "describe": + # Read or write a profile's description. The description is + # consumed by the kanban decomposer to route tasks based on + # role instead of name alone. + from hermes_cli import profiles as _profiles_mod + + all_flag = bool(getattr(args, "all_missing", False)) + auto_flag = bool(getattr(args, "auto", False)) + overwrite_flag = bool(getattr(args, "overwrite", False)) + text_value = getattr(args, "text", None) + name = getattr(args, "profile_name", None) + + if all_flag and not auto_flag: + print("profile describe: --all requires --auto", file=sys.stderr) + sys.exit(2) + if all_flag and (text_value or name): + print( + "profile describe: --all is mutually exclusive with a profile name / --text", + file=sys.stderr, + ) + sys.exit(2) + if not all_flag and not name: + print("profile describe: profile name is required (or --all --auto)", file=sys.stderr) + sys.exit(2) + if text_value and auto_flag: + print( + "profile describe: --text is mutually exclusive with --auto", + file=sys.stderr, + ) + sys.exit(2) + + # Show current description if no operation requested. + if name and not text_value and not auto_flag: + try: + if _profiles_mod.normalize_profile_name(name) == "default": + from hermes_constants import get_hermes_home as _hh + profile_dir = Path(_hh()) + else: + profile_dir = _profiles_mod.get_profile_dir(name) + except Exception as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + if not profile_dir.is_dir(): + print(f"Error: profile '{name}' not found", file=sys.stderr) + sys.exit(1) + meta = _profiles_mod.read_profile_meta(profile_dir) + desc = meta.get("description") or "" + if not desc: + print(f"(no description set for '{name}')") + else: + tag = "[auto] " if meta.get("description_auto") else "" + print(f"{tag}{desc}") + sys.exit(0) + + # --text path: just write the user-authored description. + if text_value: + try: + if _profiles_mod.normalize_profile_name(name) == "default": + from hermes_constants import get_hermes_home as _hh + profile_dir = Path(_hh()) + else: + profile_dir = _profiles_mod.get_profile_dir(name) + _profiles_mod.write_profile_meta( + profile_dir, + description=text_value, + description_auto=False, + ) + print(f"Description updated for '{name}'.") + except Exception as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + sys.exit(0) + + # --auto path: invoke the LLM describer. + from hermes_cli import profile_describer as _pd + + if all_flag: + targets = _pd.list_describable_profiles(missing_only=True) + if not targets: + print("All profiles already have descriptions.") + sys.exit(0) + else: + targets = [name] + + ok_count = 0 + fail_count = 0 + for tgt in targets: + outcome = _pd.describe_profile(tgt, overwrite=overwrite_flag) + if outcome.ok: + ok_count += 1 + print(f"Described '{outcome.profile_name}': {outcome.description}") + else: + fail_count += 1 + print( + f"profile describe {outcome.profile_name}: {outcome.reason}", + file=sys.stderr, + ) + if not all_flag: + sys.exit(0 if ok_count == 1 else 1) + sys.exit(0 if ok_count > 0 else 1) + elif action == "show": name = args.profile_name from hermes_cli.profiles import ( @@ -12023,6 +12125,13 @@ Examples: action="store_true", help="Create an empty profile with no bundled skills (opts out of `hermes update` skill sync)", ) + profile_create.add_argument( + "--description", + default=None, + help="One- or two-sentence description of what this profile is good at. " + "Used by the kanban decomposer to route tasks based on role instead " + "of profile name alone. Skip and add later via `hermes profile describe`.", + ) profile_delete = profile_subparsers.add_parser("delete", help="Delete a profile") profile_delete.add_argument("profile_name", help="Profile to delete") @@ -12030,6 +12139,40 @@ Examples: "-y", "--yes", action="store_true", help="Skip confirmation prompt" ) + profile_describe = profile_subparsers.add_parser( + "describe", + help="Read or set a profile's description (used by the kanban orchestrator)", + ) + profile_describe.add_argument( + "profile_name", + nargs="?", + default=None, + help="Profile to describe (omit + use --all --auto to sweep)", + ) + profile_describe.add_argument( + "--text", + default=None, + help="Set description to this exact text (overwrites any existing description)", + ) + profile_describe.add_argument( + "--auto", + action="store_true", + help="Auto-generate description via the auxiliary LLM " + "(uses auxiliary.profile_describer)", + ) + profile_describe.add_argument( + "--overwrite", + action="store_true", + help="With --auto, replace user-authored descriptions too (default: only " + "fill in missing or previously-auto descriptions)", + ) + profile_describe.add_argument( + "--all", + dest="all_missing", + action="store_true", + help="With --auto, run on every profile missing a description", + ) + profile_show = profile_subparsers.add_parser("show", help="Show profile details") profile_show.add_argument("profile_name", help="Profile to show") diff --git a/hermes_cli/profile_describer.py b/hermes_cli/profile_describer.py new file mode 100644 index 00000000000..55d646d92cd --- /dev/null +++ b/hermes_cli/profile_describer.py @@ -0,0 +1,299 @@ +"""Profile describer — auto-generate ``description`` for a profile. + +Used by ``hermes profile describe --auto`` and the dashboard's +"auto-generate description" button. Reads the profile's installed +skills, model+provider, name, and optionally a small slice of memory, +then asks the auxiliary LLM to produce a 1-2 sentence description of +what the profile is good at. + +Result is written to ``/profile.yaml`` with +``description_auto: true`` so the dashboard can surface a "review" +badge. User can edit afterward to confirm. + +Design notes +------------ +- Mirrors the shape of ``hermes_cli/kanban_specify.py``: lazy aux + client import inside the function, lenient response parse, never + raises on expected failure modes. +- Reads at most ``MAX_SKILLS_FOR_PROMPT`` skill names to keep the + prompt bounded. No skill body — names + categories are enough + signal and avoid blowing context on profiles with 100+ skills. +- Memory is intentionally NOT read here. Memories are personal and + the orchestrator routes work to a *role* not a *biography*. If we + find later that memory adds signal we can wire it; for now, + skills + name + model is plenty. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from hermes_cli import profiles as profiles_mod + +logger = logging.getLogger(__name__) + +# Cap on how many skill names we feed the LLM. Profiles with 200+ +# skills (uncommon but possible) would blow context otherwise. The cap +# is per-category — see _collect_skills. +MAX_SKILLS_FOR_PROMPT = 60 + + +_SYSTEM_PROMPT = """You are a profile-describer for the Hermes Agent kanban board. + +A user runs multiple "profiles" — distinct agent identities, each with their +own skills, model, and configuration. The kanban board's orchestrator routes +work to whichever profile best fits each task. To do that well, every +profile needs a short, concrete description of what it's good at. + +You are given a profile's: + - Name + - Model / provider + - List of installed skill names (a strong signal of role / domain) + +Produce a single JSON object with exactly one key: + + { + "description": "<1-2 sentence description, plain prose, no preamble>" + } + +Rules: + - The description is what an orchestrator will read to decide whether to + route a task here. Lead with the profile's strongest capability. + - Stay concrete. Bad: "an AI agent that helps users." + Good: "Reads and modifies Python codebases — runs tests, + refactors functions, opens GitHub PRs." + - 1-2 sentences, <= 280 characters total. + - Never invent capabilities the skills don't suggest. + - Never write "Hermes Agent profile" or other meta-narration. + - No code fences, no preamble, no closing remarks. Output only JSON. +""" + + +_USER_TEMPLATE = """Profile name: {name} +Default model: {model} +Provider: {provider} +Installed skill count: {skill_count} +Notable skills (up to {skill_cap}): +{skill_list} +""" + + +_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) + + +@dataclass +class DescribeOutcome: + """Result of describing a single profile.""" + + profile_name: str + ok: bool + reason: str = "" + description: Optional[str] = None + + +def _collect_skills(profile_dir: Path) -> list[str]: + """Return a stable, capped list of skill names for the prompt. + + Format: ``category/skill_name`` where category is the immediate + subdir under ``skills/`` (e.g. ``devops``, ``research``). Skills + that live directly under ``skills/`` show as bare ``skill_name``. + """ + skills_dir = profile_dir / "skills" + if not skills_dir.is_dir(): + return [] + names: list[str] = [] + for md in skills_dir.rglob("SKILL.md"): + path_str = str(md) + if "/.hub/" in path_str or "/.git/" in path_str: + continue + try: + rel = md.relative_to(skills_dir) + except ValueError: + continue + parts = rel.parts[:-1] # drop SKILL.md filename + if not parts: + continue + # parts[-1] is the skill dir name; parts[:-1] is the category path + if len(parts) == 1: + names.append(parts[0]) + else: + names.append(f"{parts[0]}/{parts[-1]}") + names.sort() + # Keep within prompt budget. Skills earlier in alphabet aren't more + # important — we'll let the LLM see a sample. Pick evenly-spaced + # entries instead of just the head so a profile with skills A..Z + # doesn't get described as "starts with A". + if len(names) <= MAX_SKILLS_FOR_PROMPT: + return names + step = len(names) / MAX_SKILLS_FOR_PROMPT + sampled = [names[int(i * step)] for i in range(MAX_SKILLS_FOR_PROMPT)] + return sampled + + +def _extract_json_blob(raw: str) -> Optional[dict]: + if not raw: + return None + stripped = _FENCE_RE.sub("", raw.strip()) + first = stripped.find("{") + last = stripped.rfind("}") + if first == -1 or last == -1 or last <= first: + return None + candidate = stripped[first : last + 1] + try: + val = json.loads(candidate) + except (ValueError, json.JSONDecodeError): + return None + if not isinstance(val, dict): + return None + return val + + +def describe_profile( + profile_name: str, + *, + overwrite: bool = False, + timeout: Optional[int] = None, +) -> DescribeOutcome: + """Auto-generate a description for one profile. + + Returns an outcome describing what happened. Never raises for + expected failure modes (profile missing, no aux client configured, + API error, malformed response) — those surface via ``ok=False`` so + a sweep can continue past individual failures. + + ``overwrite`` controls whether an existing user-authored description + is replaced. By default we refuse to overwrite a description with + ``description_auto: false`` to protect curated text. Auto-generated + descriptions (``description_auto: true``) are always replaceable. + """ + canon = profiles_mod.normalize_profile_name(profile_name) + if not profiles_mod.profile_exists(canon): + # Special case: "default" exists as a virtual profile name + # mapped to the default home dir. profile_exists() handles it. + return DescribeOutcome(canon, False, "profile not found") + + try: + if canon == "default": + from hermes_constants import get_hermes_home # type: ignore + profile_dir = Path(get_hermes_home()) + else: + profile_dir = profiles_mod.get_profile_dir(canon) + except Exception as exc: + return DescribeOutcome(canon, False, f"cannot resolve profile dir: {exc}") + + # Honor curated descriptions unless --overwrite. + existing = profiles_mod.read_profile_meta(profile_dir) + if existing.get("description") and not existing.get("description_auto") and not overwrite: + return DescribeOutcome( + canon, + False, + "profile already has a user-authored description " + "(use --overwrite to replace)", + ) + + skill_names = _collect_skills(profile_dir) + skill_list = "\n".join(f" - {n}" for n in skill_names) or " (no skills installed)" + skill_count = sum( + 1 for _ in (profile_dir / "skills").rglob("SKILL.md") + if "/.hub/" not in str(_) and "/.git/" not in str(_) + ) if (profile_dir / "skills").is_dir() else 0 + + # Read model + provider from the profile's config. + try: + model, provider = profiles_mod._read_config_model(profile_dir) + except Exception: + model, provider = None, None + + try: + from agent.auxiliary_client import ( # type: ignore + get_auxiliary_extra_body, + get_text_auxiliary_client, + ) + except Exception as exc: + logger.debug("describe: auxiliary client import failed: %s", exc) + return DescribeOutcome(canon, False, "auxiliary client unavailable") + + try: + client, aux_model = get_text_auxiliary_client("profile_describer") + except Exception as exc: + logger.debug("describe: get_text_auxiliary_client failed: %s", exc) + return DescribeOutcome(canon, False, "auxiliary client unavailable") + + if client is None or not aux_model: + return DescribeOutcome(canon, False, "no auxiliary client configured") + + user_msg = _USER_TEMPLATE.format( + name=canon, + model=(model or "(unset)"), + provider=(provider or "(unset)"), + skill_count=skill_count, + skill_cap=MAX_SKILLS_FOR_PROMPT, + skill_list=skill_list, + ) + + try: + resp = client.chat.completions.create( + model=aux_model, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": user_msg}, + ], + temperature=0.3, + max_tokens=400, + timeout=timeout or 60, + extra_body=get_auxiliary_extra_body() or None, + ) + except Exception as exc: + logger.info("describe: API call failed for %s (%s)", canon, exc) + return DescribeOutcome(canon, False, f"LLM error: {type(exc).__name__}") + + try: + raw = resp.choices[0].message.content or "" + except Exception: + raw = "" + + parsed = _extract_json_blob(raw) + if parsed is None: + # Fall back: take the raw text trimmed to one paragraph. + text = raw.strip().split("\n\n", 1)[0] + if not text: + return DescribeOutcome(canon, False, "LLM returned an empty response") + description = text[:280] + else: + val = parsed.get("description") + if not isinstance(val, str) or not val.strip(): + return DescribeOutcome( + canon, False, "LLM response missing 'description' field" + ) + description = val.strip()[:280] + + try: + profiles_mod.write_profile_meta( + profile_dir, + description=description, + description_auto=True, + ) + except Exception as exc: + return DescribeOutcome(canon, False, f"failed to write profile.yaml: {exc}") + + return DescribeOutcome(canon, True, "described", description=description) + + +def list_describable_profiles(*, missing_only: bool = True) -> list[str]: + """Return profile names that can be described. + + ``missing_only=True`` (default) returns only profiles without a + description. ``missing_only=False`` returns every profile. + """ + out: list[str] = [] + for p in profiles_mod.list_profiles(): + if missing_only and (p.description or "").strip() and not p.description_auto: + continue + out.append(p.name) + return out diff --git a/hermes_cli/profiles.py b/hermes_cli/profiles.py index de555caf9be..d35669c6243 100644 --- a/hermes_cli/profiles.py +++ b/hermes_cli/profiles.py @@ -412,6 +412,17 @@ class ProfileInfo: distribution_name: Optional[str] = None distribution_version: Optional[str] = None distribution_source: Optional[str] = None + # Free-form description (1-2 sentences) of what this profile is good + # at. Persisted in ``/profile.yaml``. Empty when the + # user has not described the profile (legacy profiles, fresh + # installs). Surfaced to the kanban decomposer so it can route work + # to the right profile based on role rather than name alone. + description: str = "" + # When True, ``description`` was auto-generated by the LLM + # describer and has not been confirmed by the user. The dashboard + # surfaces a "review" badge in this case so the user can edit or + # accept. + description_auto: bool = False def _read_distribution_meta(profile_dir: Path) -> tuple: @@ -479,6 +490,82 @@ def _count_skills(profile_dir: Path) -> int: return count +# --------------------------------------------------------------------------- +# profile.yaml — per-profile metadata (description, role, etc.) +# --------------------------------------------------------------------------- +# +# We keep this file deliberately tiny and separate from the profile's +# ``config.yaml``. ``config.yaml`` is the user-facing Hermes config +# (~5000 lines of defaults); ``profile.yaml`` is metadata ABOUT the +# profile itself (its role, who described it). Mixing them makes both +# harder to read. +# +# Missing file -> empty defaults; never an error. The kanban decomposer +# tolerates empty descriptions and just falls back to the profile name. + + +def _profile_yaml_path(profile_dir: Path) -> Path: + return profile_dir / "profile.yaml" + + +def read_profile_meta(profile_dir: Path) -> dict: + """Read ``/profile.yaml`` and return a dict. + + Returns ``{"description": "", "description_auto": False}`` when the + file is missing or unreadable. Never raises — a corrupt + profile.yaml on an unrelated profile must not break + ``hermes profile list``. + """ + path = _profile_yaml_path(profile_dir) + if not path.is_file(): + return {"description": "", "description_auto": False} + try: + import yaml + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + except Exception: + return {"description": "", "description_auto": False} + if not isinstance(data, dict): + return {"description": "", "description_auto": False} + return { + "description": str(data.get("description") or "").strip(), + "description_auto": bool(data.get("description_auto", False)), + } + + +def write_profile_meta( + profile_dir: Path, + *, + description: Optional[str] = None, + description_auto: Optional[bool] = None, +) -> None: + """Update ``/profile.yaml`` in place. + + Only the explicitly passed fields are overwritten; unspecified + fields preserve existing values. Creates the file if missing. + Profile directory itself must exist. + """ + if not profile_dir.is_dir(): + raise FileNotFoundError(f"profile directory does not exist: {profile_dir}") + import yaml + path = _profile_yaml_path(profile_dir) + existing: dict = {} + if path.is_file(): + try: + with open(path, "r", encoding="utf-8") as f: + loaded = yaml.safe_load(f) or {} + if isinstance(loaded, dict): + existing = loaded + except Exception: + existing = {} + if description is not None: + existing["description"] = description.strip() + if description_auto is not None: + existing["description_auto"] = bool(description_auto) + with open(path, "w", encoding="utf-8") as f: + yaml.safe_dump(existing, f, sort_keys=False, default_flow_style=False) + + # --------------------------------------------------------------------------- # CRUD operations # --------------------------------------------------------------------------- @@ -493,6 +580,7 @@ def list_profiles() -> List[ProfileInfo]: if default_home.is_dir(): model, provider = _read_config_model(default_home) dist_name, dist_version, dist_source = _read_distribution_meta(default_home) + meta = read_profile_meta(default_home) profiles.append(ProfileInfo( name="default", path=default_home, @@ -505,6 +593,8 @@ def list_profiles() -> List[ProfileInfo]: distribution_name=dist_name, distribution_version=dist_version, distribution_source=dist_source, + description=meta.get("description", ""), + description_auto=meta.get("description_auto", False), )) # Named profiles @@ -519,6 +609,7 @@ def list_profiles() -> List[ProfileInfo]: model, provider = _read_config_model(entry) alias_path = wrapper_dir / name dist_name, dist_version, dist_source = _read_distribution_meta(entry) + meta = read_profile_meta(entry) profiles.append(ProfileInfo( name=name, path=entry, @@ -532,6 +623,8 @@ def list_profiles() -> List[ProfileInfo]: distribution_name=dist_name, distribution_version=dist_version, distribution_source=dist_source, + description=meta.get("description", ""), + description_auto=meta.get("description_auto", False), )) return profiles @@ -544,6 +637,7 @@ def create_profile( clone_config: bool = False, no_alias: bool = False, no_skills: bool = False, + description: Optional[str] = None, ) -> Path: """Create a new profile directory. @@ -667,6 +761,19 @@ def create_profile( except OSError: pass # best-effort — the feature still works via the empty skills/ dir + # Persist description if the caller provided one. Done last so a + # partial-create failure doesn't strand a description file in an + # incomplete profile. + if description and description.strip(): + try: + write_profile_meta( + profile_dir, + description=description.strip(), + description_auto=False, + ) + except Exception: + pass # non-fatal — user can describe later with `hermes profile describe` + return profile_dir diff --git a/plugins/kanban/dashboard/dist/index.js b/plugins/kanban/dashboard/dist/index.js index 6f05df72bf6..3f6def61cef 100644 --- a/plugins/kanban/dashboard/dist/index.js +++ b/plugins/kanban/dashboard/dist/index.js @@ -908,6 +908,7 @@ return createNewBoard(payload).then(function () { setShowNewBoard(false); }); }, }) : null, + h(OrchestrationPanel, null), h(AttentionStrip, { boardData, onOpen: setSelectedTaskId, @@ -1386,6 +1387,288 @@ }, "?"); } + // --------------------------------------------------------------------- + // OrchestrationPanel — collapsible settings panel for the kanban + // orchestrator (orchestrator profile picker, default assignee picker, + // auto-decompose toggle, plus per-profile description editing with + // auto-generate). Backed by /orchestration + /profiles endpoints. + // --------------------------------------------------------------------- + + function OrchestrationPanel() { + const [expanded, setExpanded] = useState(false); + const [settings, setSettings] = useState(null); + const [profiles, setProfiles] = useState([]); + const [busy, setBusy] = useState({}); + const [msg, setMsg] = useState(null); + + const loadAll = useCallback(function () { + Promise.all([ + SDK.fetchJSON(`${API}/orchestration`), + SDK.fetchJSON(`${API}/profiles`), + ]).then(function (results) { + setSettings(results[0] || null); + setProfiles((results[1] && results[1].profiles) || []); + setMsg(null); + }).catch(function (err) { + setMsg({ ok: false, text: "Failed to load: " + (err.message || String(err)) }); + }); + }, []); + + useEffect(function () { + // Load on mount so the collapsed pill shows the real mode without + // requiring the user to expand the panel first. + if (settings === null) loadAll(); + }, [settings, loadAll]); + + const saveSettings = function (patch) { + setMsg(null); + return SDK.fetchJSON(`${API}/orchestration`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }).then(function (res) { + setSettings(res); + setMsg({ ok: true, text: "Settings saved." }); + return res; + }).catch(function (err) { + setMsg({ ok: false, text: "Save failed: " + (err.message || String(err)) }); + }); + }; + + const saveProfileDescription = function (name, description) { + setBusy(function (b) { return Object.assign({}, b, { [name]: "save" }); }); + return SDK.fetchJSON(`${API}/profiles/${encodeURIComponent(name)}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ description: description }), + }).then(function () { + loadAll(); + setMsg({ ok: true, text: `Description saved for ${name}.` }); + }).catch(function (err) { + setMsg({ ok: false, text: "Save failed: " + (err.message || String(err)) }); + }).then(function () { + setBusy(function (b) { + const next = Object.assign({}, b); delete next[name]; return next; + }); + }); + }; + + const autoGenerateDescription = function (name, overwrite) { + setBusy(function (b) { return Object.assign({}, b, { [name]: "auto" }); }); + return SDK.fetchJSON(`${API}/profiles/${encodeURIComponent(name)}/describe-auto`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ overwrite: !!overwrite }), + }).then(function (res) { + if (res && res.ok) { + loadAll(); + setMsg({ ok: true, text: `Auto-generated description for ${name}.` }); + } else { + setMsg({ + ok: false, + text: "Auto-generate failed: " + ((res && res.reason) || "unknown error"), + }); + } + }).catch(function (err) { + setMsg({ ok: false, text: "Auto-generate failed: " + (err.message || String(err)) }); + }).then(function () { + setBusy(function (b) { + const next = Object.assign({}, b); delete next[name]; return next; + }); + }); + }; + + const headerLabel = expanded + ? "▾ Orchestration settings" + : "▸ Orchestration settings"; + + // Mode pill — always visible (collapsed or expanded). One click flips + // between Auto and Manual. Auto = dispatcher decomposes new triage tasks + // every tick. Manual = pre-PR behavior, the user clicks ⚗ Decompose on + // each triage card (or runs `hermes kanban decompose `) and tasks + // stay in triage until then. + const autoOn = !!(settings && settings.auto_decompose); + const modePillTitle = settings === null + ? "Loading mode…" + : (autoOn + ? "Orchestration: Auto — the dispatcher decomposes new triage tasks automatically every tick. Click to switch to Manual (pre-PR behavior)." + : "Orchestration: Manual — triage tasks stay in triage until you click ⚗ Decompose on each card. Click to switch to Auto."); + const modePill = h("button", { + type: "button", + onClick: function () { + if (settings === null) return; // not loaded yet + saveSettings({ auto_decompose: !autoOn }); + }, + disabled: settings === null, + title: modePillTitle, + className: "inline-flex items-center gap-1 rounded-full border px-2 py-0.5 " + + "text-xs font-medium " + + (autoOn + ? "border-emerald-500/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300" + : "border-muted-foreground/30 bg-muted/30 text-muted-foreground"), + }, + "Orchestration: ", + h("span", { className: "ml-1 font-semibold" }, + settings === null ? "…" : (autoOn ? "Auto" : "Manual")) + ); + + if (!expanded) { + return h("div", { className: "flex items-center gap-3 text-xs" }, + modePill, + h("button", { + type: "button", + onClick: function () { setExpanded(true); }, + className: "underline text-muted-foreground hover:text-foreground", + title: "Configure the kanban orchestrator (profile picker, default assignee, auto-decompose, profile descriptions)", + }, headerLabel), + ); + } + + const profileOptions = profiles.map(function (p) { + const tag = p.is_default ? " (default)" : ""; + return h(SelectOption, { key: p.name, value: p.name }, p.name + tag); + }); + + return h(Card, { className: "p-3" }, + h(CardContent, { className: "p-2 flex flex-col gap-3" }, + h("div", { className: "flex items-center justify-between" }, + h("button", { + type: "button", + onClick: function () { setExpanded(false); }, + className: "text-sm font-medium underline-offset-2 hover:underline", + }, headerLabel), + modePill, + h(Button, { onClick: loadAll, size: "sm" }, "Reload"), + ), + msg ? h("div", { + className: msg.ok ? "hermes-kanban-msg-ok" : "hermes-kanban-msg-err", + }, msg.text) : null, + + settings ? h("div", { className: "grid gap-3 sm:grid-cols-3" }, + h("div", { className: "flex flex-col gap-1" }, + h(Label, { className: "text-xs text-muted-foreground" }, + "Orchestrator profile"), + h(Select, { + value: settings.orchestrator_profile || "", + className: "h-8", + onChange: function (e) { + const v = (e && e.target ? e.target.value : e) || ""; + saveSettings({ orchestrator_profile: v }); + }, + }, + h(SelectOption, { value: "" }, + "(default: " + (settings.active_profile || "default") + ")"), + profileOptions, + ), + h("div", { className: "text-[10px] text-muted-foreground" }, + "Resolved: " + (settings.resolved_orchestrator_profile || "default")), + ), + h("div", { className: "flex flex-col gap-1" }, + h(Label, { className: "text-xs text-muted-foreground" }, + "Default assignee"), + h(Select, { + value: settings.default_assignee || "", + className: "h-8", + onChange: function (e) { + const v = (e && e.target ? e.target.value : e) || ""; + saveSettings({ default_assignee: v }); + }, + }, + h(SelectOption, { value: "" }, + "(default: " + (settings.active_profile || "default") + ")"), + profileOptions, + ), + h("div", { className: "text-[10px] text-muted-foreground" }, + "Resolved: " + (settings.resolved_default_assignee || "default")), + ), + h("div", { className: "flex flex-col gap-1" }, + h(Label, { className: "text-xs text-muted-foreground" }, + "Orchestration mode"), + h("label", { className: "flex items-center gap-2 text-xs h-8" }, + h("input", { + type: "checkbox", + checked: !!settings.auto_decompose, + onChange: function (e) { + saveSettings({ auto_decompose: !!e.target.checked }); + }, + }), + settings.auto_decompose ? "Auto (default)" : "Manual", + ), + h("div", { className: "text-[10px] text-muted-foreground" }, + "When on, the dispatcher decomposes new triage tasks automatically."), + ), + ) : h("div", { className: "text-xs text-muted-foreground" }, + "Loading…"), + + h("div", { className: "border-t pt-3" }, + h(Label, { className: "text-xs text-muted-foreground" }, + "Profile descriptions"), + h("div", { className: "text-[10px] text-muted-foreground pb-2" }, + "Descriptions guide the orchestrator's routing. Click ⚗ to auto-generate, or edit and save."), + profiles.length === 0 + ? h("div", { className: "text-xs text-muted-foreground" }, "No profiles installed.") + : h("div", { className: "flex flex-col gap-2" }, + profiles.map(function (p) { + return h(ProfileDescriptionRow, { + key: p.name, + profile: p, + busy: busy[p.name] || null, + onSave: saveProfileDescription, + onAuto: autoGenerateDescription, + }); + }), + ), + ), + ), + ); + } + + function ProfileDescriptionRow(props) { + const p = props.profile; + const [draft, setDraft] = useState(p.description || ""); + const busy = props.busy; + // Re-sync the local draft if the server-side description changes (e.g. + // after auto-generate). Cheap because re-runs only happen on prop change. + useEffect(function () { + setDraft(p.description || ""); + }, [p.description]); + + const tag = p.description_auto && p.description ? " [auto, review]" : ""; + return h("div", { className: "flex flex-col gap-1 border-l-2 pl-2", + style: { borderColor: p.description ? "#888" : "#cc6" } }, + h("div", { className: "flex items-center gap-2 text-xs" }, + h("span", { className: "font-medium" }, p.name), + p.is_default ? h("span", { className: "text-[10px] text-muted-foreground" }, "(default)") : null, + p.description_auto && p.description + ? h("span", { className: "text-[10px] text-yellow-600" }, "auto — review") + : null, + !p.description + ? h("span", { className: "text-[10px] text-yellow-600" }, "⚠ no description") + : null, + ), + h("div", { className: "flex items-center gap-2" }, + h(Input, { + value: draft, + onChange: function (e) { setDraft(e.target.value); }, + placeholder: "What is this profile good at?", + className: "h-7 text-xs flex-1", + }), + h(Button, { + onClick: function () { props.onSave(p.name, draft); }, + size: "sm", + disabled: !!busy || draft === (p.description || ""), + title: "Save the description above as user-authored", + }, busy === "save" ? "Saving…" : "Save"), + h(Button, { + onClick: function () { props.onAuto(p.name, true); }, + size: "sm", + disabled: !!busy, + title: "Auto-generate a description from this profile's skills and model", + }, busy === "auto" ? "Generating…" : "⚗ Auto"), + ), + ); + } + function BoardSwitcher(props) { const { t } = useI18n(); const list = props.boardList || []; @@ -2395,6 +2678,25 @@ }); }; + // POST /tasks/:id/decompose — fan a triage task out into a graph + // of child tasks routed to specialist profiles by description. + // Refreshes both the drawer (so the user sees the root flip to + // todo) and the board (so the new children appear in the columns). + const doDecompose = function () { + return SDK.fetchJSON( + withBoard(`${API}/tasks/${encodeURIComponent(props.taskId)}/decompose`, boardSlug), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + } + ).then(function (res) { + load(); + props.onRefresh(); + return res; + }); + }; + const addLink = function (parentId) { return SDK.fetchJSON(withBoard(`${API}/links`, boardSlug), { method: "POST", @@ -2486,6 +2788,7 @@ boardSlug: boardSlug, onPatch: doPatch, onSpecify: doSpecify, + onDecompose: doDecompose, onAddParent: addLink, onRemoveParent: removeLink, onAddChild: addChild, @@ -2559,6 +2862,7 @@ task: t, onPatch: props.onPatch, onSpecify: props.onSpecify, + onDecompose: props.onDecompose, }), h(DiagnosticsSection, { task: t, @@ -3023,6 +3327,8 @@ const task = props.task; const [specifyBusy, setSpecifyBusy] = useState(false); const [specifyMsg, setSpecifyMsg] = useState(null); + const [decomposeBusy, setDecomposeBusy] = useState(false); + const [decomposeMsg, setDecomposeMsg] = useState(null); const b = function (label, patch, enabled, confirmMsg) { return h(Button, { onClick: function () { if (enabled !== false) props.onPatch(patch, { confirm: confirmMsg }); }, @@ -3067,9 +3373,57 @@ }, specifyBusy ? "Specifying…" : "✨ Specify") : null; + // "Decompose" is the orchestrator-driven fan-out. Like Specify, only + // makes sense on triage-column tasks — elsewhere the backend short- + // circuits with ok:false. When the orchestrator returns fanout:false + // we render the same single-task message as Specify; when it fans + // out we report the child count for quick at-a-glance verification. + const decomposeButton = (task.status === "triage" && props.onDecompose) + ? h(Button, { + onClick: function () { + if (decomposeBusy) return; + setDecomposeBusy(true); + setDecomposeMsg(null); + props.onDecompose().then(function (res) { + if (res && res.ok) { + if (res.fanout && res.child_ids && res.child_ids.length) { + setDecomposeMsg({ + ok: true, + text: `Decomposed into ${res.child_ids.length} children: ${res.child_ids.join(", ")}`, + }); + } else { + const suffix = res.new_title + ? ` — retitled: ${res.new_title}` + : ""; + setDecomposeMsg({ + ok: true, + text: `Single task (no fanout)${suffix}`, + }); + } + } else { + setDecomposeMsg({ + ok: false, + text: "Decompose failed: " + ((res && res.reason) || "unknown error"), + }); + } + }).catch(function (err) { + setDecomposeMsg({ + ok: false, + text: "Decompose failed: " + (err.message || String(err)), + }); + }).then(function () { + setDecomposeBusy(false); + }); + }, + disabled: decomposeBusy, + size: "sm", + }, decomposeBusy ? "Decomposing…" : "⚗ Decompose") + : null; + return h("div", null, h("div", { className: "hermes-kanban-actions" }, specifyButton, + decomposeButton, b("→ triage", { status: "triage" }, task.status !== "triage"), b("→ ready", { status: "ready" }, task.status !== "ready"), // No direct → running button: /tasks/:id PATCH rejects status=running @@ -3091,6 +3445,11 @@ ? "hermes-kanban-msg-ok" : "hermes-kanban-msg-err", }, specifyMsg.text) : null, + decomposeMsg ? h("div", { + className: decomposeMsg.ok + ? "hermes-kanban-msg-ok" + : "hermes-kanban-msg-err", + }, decomposeMsg.text) : null, ); } diff --git a/plugins/kanban/dashboard/plugin_api.py b/plugins/kanban/dashboard/plugin_api.py index 08824e3807b..16e60663854 100644 --- a/plugins/kanban/dashboard/plugin_api.py +++ b/plugins/kanban/dashboard/plugin_api.py @@ -1535,6 +1535,279 @@ def switch_board(slug: str): _EVENT_POLL_SECONDS = 0.3 +# --------------------------------------------------------------------------- +# Profile metadata & description editing (consumed by the kanban orchestrator) +# --------------------------------------------------------------------------- + +class DescribeBody(BaseModel): + description: Optional[str] = None # explicit user-authored text + + +class DescribeAutoBody(BaseModel): + overwrite: bool = False + + +@router.get("/profiles") +def list_profile_roster(): + """Return every installed profile with its description. + + Consumed by the dashboard's settings panel (orchestrator picker) + and the profile-description editing UI. Profiles without a + description still appear here — they're routable on name alone, + just less precisely. + """ + try: + from hermes_cli import profiles as profiles_mod + profiles = profiles_mod.list_profiles() + except Exception as exc: + raise HTTPException(status_code=500, detail=f"failed to list profiles: {exc}") + return { + "profiles": [ + { + "name": p.name, + "is_default": bool(p.is_default), + "model": p.model or "", + "provider": p.provider or "", + "description": p.description or "", + "description_auto": bool(p.description_auto), + "skill_count": int(p.skill_count or 0), + } + for p in profiles + ], + } + + +@router.patch("/profiles/{profile_name}") +def update_profile_description(profile_name: str, payload: DescribeBody): + """Set or clear the description of a profile. + + Empty string clears the description; non-empty stores it as a + user-authored description (``description_auto: false``) so the + auto-describer won't overwrite it on a sweep without + ``--overwrite``. + """ + try: + from hermes_cli import profiles as profiles_mod + canon = profiles_mod.normalize_profile_name(profile_name) + if canon == "default": + from hermes_constants import get_hermes_home # type: ignore + from pathlib import Path as _Path + profile_dir = _Path(get_hermes_home()) + else: + profile_dir = profiles_mod.get_profile_dir(canon) + if not profile_dir.is_dir(): + raise HTTPException(status_code=404, detail=f"profile '{profile_name}' not found") + text = (payload.description or "").strip() + profiles_mod.write_profile_meta( + profile_dir, + description=text, + description_auto=False, + ) + except HTTPException: + raise + except Exception as exc: + raise HTTPException(status_code=500, detail=f"failed to update profile: {exc}") + return {"ok": True, "profile": canon, "description": text} + + +@router.post("/profiles/{profile_name}/describe-auto") +def auto_describe_profile(profile_name: str, payload: DescribeAutoBody): + """Generate a description for the named profile via the auxiliary + LLM (``auxiliary.profile_describer``). Persists with + ``description_auto: true`` so the dashboard can surface a "review" + badge. + + Maps 1:1 to ``hermes profile describe --auto``. Non-OK + outcomes are NOT HTTP errors — the UI renders the reason inline + (e.g. "no auxiliary client configured") so the operator can fix + config and retry without a page reload. + """ + try: + from hermes_cli import profile_describer # noqa: WPS433 (intentional) + outcome = profile_describer.describe_profile( + profile_name, + overwrite=bool(payload.overwrite), + ) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"describer crashed: {exc}") + return { + "ok": bool(outcome.ok), + "profile": outcome.profile_name, + "reason": outcome.reason, + "description": outcome.description, + } + + +# --------------------------------------------------------------------------- +# Decompose endpoint (orchestrator-driven fan-out) +# --------------------------------------------------------------------------- + +class DecomposeBody(BaseModel): + author: Optional[str] = None + + +@router.post("/tasks/{task_id}/decompose") +def decompose_task_endpoint( + task_id: str, + payload: DecomposeBody, + board: Optional[str] = Query(None), +): + """Fan a triage-column task out into a graph of child tasks via the + auxiliary LLM, routed to specialist profiles by description. Maps + 1:1 to ``hermes kanban decompose ``. + + Returns the outcome shape used by the CLI: ``{ok, task_id, reason, + fanout, child_ids, new_title}``. A non-OK outcome is NOT an HTTP + error — the UI renders the reason inline. + + Runs in FastAPI's threadpool (sync ``def``) because the LLM call + can take minutes on reasoning models. + """ + board = _resolve_board(board) + prev_env = os.environ.get("HERMES_KANBAN_BOARD") + try: + os.environ["HERMES_KANBAN_BOARD"] = board or kanban_db.DEFAULT_BOARD + from hermes_cli import kanban_decompose # noqa: WPS433 (intentional) + outcome = kanban_decompose.decompose_task( + task_id, + author=(payload.author or None), + ) + finally: + if prev_env is None: + os.environ.pop("HERMES_KANBAN_BOARD", None) + else: + os.environ["HERMES_KANBAN_BOARD"] = prev_env + + return { + "ok": bool(outcome.ok), + "task_id": outcome.task_id, + "reason": outcome.reason, + "fanout": bool(outcome.fanout), + "child_ids": outcome.child_ids or [], + "new_title": outcome.new_title, + } + + +# --------------------------------------------------------------------------- +# Orchestration settings (kanban.orchestrator_profile / default_assignee / +# auto_decompose) — surfaced to the dashboard's settings panel +# --------------------------------------------------------------------------- + +class OrchestrationSettingsBody(BaseModel): + orchestrator_profile: Optional[str] = None + default_assignee: Optional[str] = None + auto_decompose: Optional[bool] = None + + +@router.get("/orchestration") +def get_orchestration_settings(): + """Return the current kanban orchestration knobs from config.yaml + plus the resolved effective values (filling in fallbacks).""" + try: + from hermes_cli.config import load_config + cfg = load_config() or {} + except Exception: + cfg = {} + kanban_cfg = (cfg.get("kanban") or {}) if isinstance(cfg, dict) else {} + explicit_orch = (kanban_cfg.get("orchestrator_profile") or "").strip() + explicit_default = (kanban_cfg.get("default_assignee") or "").strip() + auto_decompose = bool(kanban_cfg.get("auto_decompose", True)) + + # Resolve fallbacks the same way the decomposer does. + resolved_orch = explicit_orch + resolved_default = explicit_default + try: + from hermes_cli import profiles as profiles_mod + active_default = profiles_mod.get_active_profile_name() or "default" + if not resolved_orch or not profiles_mod.profile_exists(resolved_orch): + resolved_orch = active_default + if not resolved_default or not profiles_mod.profile_exists(resolved_default): + resolved_default = active_default + except Exception: + active_default = "default" + if not resolved_orch: + resolved_orch = active_default + if not resolved_default: + resolved_default = active_default + + return { + "orchestrator_profile": explicit_orch, + "default_assignee": explicit_default, + "auto_decompose": auto_decompose, + "resolved_orchestrator_profile": resolved_orch, + "resolved_default_assignee": resolved_default, + "active_profile": active_default, + } + + +@router.put("/orchestration") +def set_orchestration_settings(payload: OrchestrationSettingsBody): + """Update the kanban orchestration knobs in ~/.hermes/config.yaml. + + Each field is optional — only fields explicitly passed are + written. ``orchestrator_profile`` / ``default_assignee`` accept + empty strings to clear the override and fall back to the default + profile. + """ + try: + from hermes_cli.config import load_config, save_config + cfg = load_config() or {} + except Exception as exc: + raise HTTPException(status_code=500, detail=f"failed to load config: {exc}") + + kanban_section = cfg.setdefault("kanban", {}) + if not isinstance(kanban_section, dict): + kanban_section = {} + cfg["kanban"] = kanban_section + + # Validate any non-empty profile names exist before saving. + try: + from hermes_cli import profiles as profiles_mod + except Exception: + profiles_mod = None # type: ignore + + if payload.orchestrator_profile is not None: + name = (payload.orchestrator_profile or "").strip() + if name and profiles_mod is not None: + try: + if not profiles_mod.profile_exists(name): + raise HTTPException( + status_code=400, + detail=f"profile '{name}' does not exist", + ) + except HTTPException: + raise + except Exception: + pass # fail open if the lookup itself errors + kanban_section["orchestrator_profile"] = name + + if payload.default_assignee is not None: + name = (payload.default_assignee or "").strip() + if name and profiles_mod is not None: + try: + if not profiles_mod.profile_exists(name): + raise HTTPException( + status_code=400, + detail=f"profile '{name}' does not exist", + ) + except HTTPException: + raise + except Exception: + pass + kanban_section["default_assignee"] = name + + if payload.auto_decompose is not None: + kanban_section["auto_decompose"] = bool(payload.auto_decompose) + + try: + save_config(cfg) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"failed to save config: {exc}") + + # Echo back the resolved state (callers usually re-render from it). + return get_orchestration_settings() + + @router.websocket("/events") async def stream_events(ws: WebSocket): # Enforce the dashboard session token as a query param — browsers can't diff --git a/tests/hermes_cli/test_kanban_decompose.py b/tests/hermes_cli/test_kanban_decompose.py new file mode 100644 index 00000000000..f55e10e2f8e --- /dev/null +++ b/tests/hermes_cli/test_kanban_decompose.py @@ -0,0 +1,242 @@ +"""Tests for the decomposer module + `hermes kanban decompose` CLI surface. + +The auxiliary LLM client is mocked — no network calls. Tests exercise the +prompt plumbing, response parsing, DB writes (via the real DB helper), +and the assignee-fallback logic. +""" + +from __future__ import annotations + +import argparse +import json as jsonlib +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from hermes_cli import kanban as kanban_cli +from hermes_cli import kanban_db as kb +from hermes_cli import kanban_decompose as decomp + + +@pytest.fixture +def kanban_home(tmp_path, monkeypatch): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + kb.init_db() + return home + + +def _fake_aux_response(content: str): + resp = MagicMock() + resp.choices = [MagicMock()] + resp.choices[0].message.content = content + return resp + + +def _mock_client_returning(content: str): + client = MagicMock() + client.chat.completions.create = MagicMock(return_value=_fake_aux_response(content)) + return client + + +def _patch_aux_client(content: str, *, model: str = "test-model"): + client = _mock_client_returning(content) + return patch( + "agent.auxiliary_client.get_text_auxiliary_client", + return_value=(client, model), + ) + + +def _patch_extra_body(): + return patch( + "agent.auxiliary_client.get_auxiliary_extra_body", + return_value={}, + ) + + +def _patch_list_profiles(names: list[str]): + """Pretend the named profiles exist. The decomposer uses + profiles_mod.list_profiles() to build the roster + valid-set, and + profiles_mod.profile_exists() to resolve orchestrator/default.""" + from types import SimpleNamespace + fake_profiles = [ + SimpleNamespace( + name=n, is_default=(i == 0), description=f"desc for {n}", + description_auto=False, model="m", provider="p", skill_count=1, + ) + for i, n in enumerate(names) + ] + return [ + patch("hermes_cli.profiles.list_profiles", return_value=fake_profiles), + patch("hermes_cli.profiles.profile_exists", side_effect=lambda x: x in names), + patch("hermes_cli.profiles.get_active_profile_name", return_value=names[0] if names else "default"), + ] + + +def test_decompose_with_fanout_creates_children(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="ship a feature", triage=True) + + llm_payload = jsonlib.dumps({ + "fanout": True, + "rationale": "test split", + "tasks": [ + {"title": "research", "body": "look it up", "assignee": "researcher", "parents": []}, + {"title": "build", "body": "code it", "assignee": "engineer", "parents": [0]}, + ], + }) + + patches = _patch_list_profiles(["orchestrator", "researcher", "engineer"]) + for p in patches: + p.start() + try: + with _patch_aux_client(llm_payload), _patch_extra_body(): + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + + assert outcome.ok, outcome.reason + assert outcome.fanout is True + assert outcome.child_ids and len(outcome.child_ids) == 2 + + with kb.connect() as conn: + root = kb.get_task(conn, tid) + c0 = kb.get_task(conn, outcome.child_ids[0]) + c1 = kb.get_task(conn, outcome.child_ids[1]) + assert root.status == "todo" + assert c0.status == "ready" + assert c1.status == "todo" + assert c0.assignee == "researcher" + assert c1.assignee == "engineer" + + +def test_decompose_fanout_false_falls_back_to_specify(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="just one thing", triage=True) + + llm_payload = jsonlib.dumps({ + "fanout": False, + "rationale": "single unit", + "title": "Tightened title", + "body": "**Goal**\nDo the thing.", + }) + + patches = _patch_list_profiles(["orchestrator"]) + for p in patches: + p.start() + try: + with _patch_aux_client(llm_payload), _patch_extra_body(): + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + + assert outcome.ok, outcome.reason + assert outcome.fanout is False + assert outcome.new_title == "Tightened title" + with kb.connect() as conn: + task = kb.get_task(conn, tid) + # specify path with no parents -> recompute_ready flips to 'ready' + assert task.status == "ready" + assert task.title == "Tightened title" + + +def test_decompose_unknown_assignee_falls_back_to_default(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="x", triage=True) + + # Roster only has 'orchestrator' and 'fallback'; LLM picks 'made_up'. + llm_payload = jsonlib.dumps({ + "fanout": True, + "rationale": "test", + "tasks": [ + {"title": "do X", "body": "", "assignee": "made_up", "parents": []}, + ], + }) + + patches = _patch_list_profiles(["orchestrator", "fallback"]) + for p in patches: + p.start() + try: + with patch.dict( + "os.environ", {}, clear=False, + ), _patch_aux_client(llm_payload), _patch_extra_body(), \ + patch( + "hermes_cli.kanban_decompose._load_config", + return_value={ + "kanban": { + "orchestrator_profile": "orchestrator", + "default_assignee": "fallback", + } + }, + ): + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + + assert outcome.ok, outcome.reason + assert outcome.child_ids and len(outcome.child_ids) == 1 + with kb.connect() as conn: + child = kb.get_task(conn, outcome.child_ids[0]) + # 'made_up' wasn't in roster, so assignee rewritten to 'fallback' + assert child.assignee == "fallback" + + +def test_decompose_handles_malformed_llm_json(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="x", triage=True) + + patches = _patch_list_profiles(["orchestrator"]) + for p in patches: + p.start() + try: + with _patch_aux_client("not json at all, sorry"), _patch_extra_body(): + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + + assert outcome.ok is False + assert "malformed JSON" in outcome.reason + + +def test_decompose_returns_false_when_task_not_triage(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="x") # ready, not triage + + patches = _patch_list_profiles(["orchestrator"]) + for p in patches: + p.start() + try: + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + assert outcome.ok is False + assert "not in triage" in outcome.reason + + +def test_decompose_no_aux_client_configured(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="x", triage=True) + + patches = _patch_list_profiles(["orchestrator"]) + for p in patches: + p.start() + try: + with patch( + "agent.auxiliary_client.get_text_auxiliary_client", + return_value=(None, ""), + ): + outcome = decomp.decompose_task(tid, author="me") + finally: + for p in patches: + p.stop() + + assert outcome.ok is False + assert "no auxiliary client" in outcome.reason diff --git a/tests/hermes_cli/test_kanban_decompose_db.py b/tests/hermes_cli/test_kanban_decompose_db.py new file mode 100644 index 00000000000..236fb1fff1b --- /dev/null +++ b/tests/hermes_cli/test_kanban_decompose_db.py @@ -0,0 +1,152 @@ +"""Tests for kb.decompose_triage_task — the DB-layer atomic fan-out +from the triage column. LLM-free by design. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from hermes_cli import kanban_db as kb + + +@pytest.fixture +def kanban_home(tmp_path, monkeypatch): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + kb.init_db() + return home + + +def _create_triage(conn, title="rough idea", body=None, assignee=None, tenant=None): + return kb.create_task( + conn, + title=title, + body=body, + assignee=assignee, + tenant=tenant, + triage=True, + ) + + +def test_decompose_creates_children_and_promotes_root(kanban_home): + with kb.connect() as conn: + tid = _create_triage(conn, title="ship a feature") + assert kb.get_task(conn, tid).status == "triage" + + children = [ + {"title": "research", "body": "look at prior art", "assignee": "researcher", "parents": []}, + {"title": "build it", "body": "write code", "assignee": "engineer", "parents": [0]}, + ] + with kb.connect() as conn: + child_ids = kb.decompose_triage_task( + conn, + tid, + root_assignee="orchestrator", + children=children, + author="decomposer", + ) + assert child_ids is not None + assert len(child_ids) == 2 + + with kb.connect() as conn: + root = kb.get_task(conn, tid) + c0 = kb.get_task(conn, child_ids[0]) + c1 = kb.get_task(conn, child_ids[1]) + + # Root flipped to todo with orchestrator assignee, gated by children. + assert root.status == "todo" + assert root.assignee == "orchestrator" + # First child has no internal parents → ready on recompute_ready. + assert c0.status == "ready" + assert c0.assignee == "researcher" + # Second child has parents=[0] → stays in todo until c0 completes. + assert c1.status == "todo" + assert c1.assignee == "engineer" + + +def test_decompose_returns_none_when_task_missing(kanban_home): + with kb.connect() as conn: + result = kb.decompose_triage_task( + conn, + "nonexistent", + root_assignee="orch", + children=[{"title": "x"}], + author="me", + ) + assert result is None + + +def test_decompose_returns_none_when_task_not_in_triage(kanban_home): + with kb.connect() as conn: + tid = kb.create_task(conn, title="already a real task") # not triage + result = kb.decompose_triage_task( + conn, + tid, + root_assignee="orch", + children=[{"title": "x"}], + author="me", + ) + assert result is None + + +def test_decompose_empty_children_returns_none(kanban_home): + with kb.connect() as conn: + tid = _create_triage(conn) + result = kb.decompose_triage_task( + conn, + tid, + root_assignee="orch", + children=[], + author="me", + ) + assert result is None + + +def test_decompose_rejects_self_parent(kanban_home): + with kb.connect() as conn: + tid = _create_triage(conn) + with pytest.raises(ValueError, match="cannot list itself"): + kb.decompose_triage_task( + conn, + tid, + root_assignee="orch", + children=[{"title": "x", "parents": [0]}], + author="me", + ) + + +def test_decompose_rejects_out_of_range_parent(kanban_home): + with kb.connect() as conn: + tid = _create_triage(conn) + with pytest.raises(ValueError, match="not a valid index"): + kb.decompose_triage_task( + conn, + tid, + root_assignee="orch", + children=[{"title": "x", "parents": [5]}], + author="me", + ) + + +def test_decompose_records_audit_comment_and_event(kanban_home): + with kb.connect() as conn: + tid = _create_triage(conn) + child_ids = kb.decompose_triage_task( + conn, + tid, + root_assignee="orch", + children=[{"title": "task A", "assignee": "researcher"}], + author="alice", + ) + assert child_ids is not None + + with kb.connect() as conn: + comments = kb.list_comments(conn, tid) + events = kb.list_events(conn, tid) + + assert any("Decomposed into" in (c.body or "") for c in comments) + assert any(ev.kind == "decomposed" for ev in events) diff --git a/tests/hermes_cli/test_profile_describer.py b/tests/hermes_cli/test_profile_describer.py new file mode 100644 index 00000000000..3fc5fa3a6be --- /dev/null +++ b/tests/hermes_cli/test_profile_describer.py @@ -0,0 +1,168 @@ +"""Tests for the profile.yaml metadata layer (description + description_auto) +and the profile_describer LLM module. +""" + +from __future__ import annotations + +import json as jsonlib +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from hermes_cli import profiles as profiles_mod +from hermes_cli import profile_describer as describer + + +@pytest.fixture +def profile_env(tmp_path, monkeypatch): + """Set up an isolated HERMES_HOME with a default profile dir.""" + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + return home + + +def test_read_profile_meta_empty_when_missing(profile_env): + meta = profiles_mod.read_profile_meta(profile_env) + assert meta == {"description": "", "description_auto": False} + + +def test_write_and_read_profile_meta(profile_env): + profiles_mod.write_profile_meta( + profile_env, + description="a useful researcher", + description_auto=False, + ) + meta = profiles_mod.read_profile_meta(profile_env) + assert meta["description"] == "a useful researcher" + assert meta["description_auto"] is False + + +def test_write_profile_meta_preserves_other_fields(profile_env): + # First write sets description_auto=True; second write only updates + # description and leaves description_auto unchanged. + profiles_mod.write_profile_meta( + profile_env, + description="auto-gen", + description_auto=True, + ) + profiles_mod.write_profile_meta(profile_env, description="edited by hand") + meta = profiles_mod.read_profile_meta(profile_env) + assert meta["description"] == "edited by hand" + assert meta["description_auto"] is True + + +def test_write_profile_meta_rejects_missing_dir(tmp_path): + bogus = tmp_path / "does_not_exist" + with pytest.raises(FileNotFoundError): + profiles_mod.write_profile_meta(bogus, description="x") + + +def test_read_profile_meta_tolerates_corrupt_yaml(profile_env): + (profile_env / "profile.yaml").write_text("not: valid: yaml: [unclosed") + meta = profiles_mod.read_profile_meta(profile_env) + assert meta == {"description": "", "description_auto": False} + + +# --------------------------------------------------------------------------- +# profile_describer module +# --------------------------------------------------------------------------- + + +def _fake_aux_response(content: str): + resp = MagicMock() + resp.choices = [MagicMock()] + resp.choices[0].message.content = content + return resp + + +def _patch_aux_client(content: str): + client = MagicMock() + client.chat.completions.create = MagicMock(return_value=_fake_aux_response(content)) + return patch( + "agent.auxiliary_client.get_text_auxiliary_client", + return_value=(client, "test-model"), + ) + + +def test_describer_writes_description_with_auto_true(profile_env, monkeypatch): + # Pretend "myprof" is a registered profile pointing at profile_env. + monkeypatch.setattr( + profiles_mod, "profile_exists", lambda n: n == "myprof", + ) + monkeypatch.setattr( + profiles_mod, "normalize_profile_name", lambda n: n, + ) + monkeypatch.setattr( + profiles_mod, "get_profile_dir", lambda n: profile_env, + ) + + payload = jsonlib.dumps({"description": "writes Python codebases"}) + with _patch_aux_client(payload), patch( + "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} + ): + outcome = describer.describe_profile("myprof") + + assert outcome.ok, outcome.reason + assert outcome.description == "writes Python codebases" + meta = profiles_mod.read_profile_meta(profile_env) + assert meta["description"] == "writes Python codebases" + assert meta["description_auto"] is True + + +def test_describer_refuses_to_overwrite_user_authored(profile_env, monkeypatch): + profiles_mod.write_profile_meta( + profile_env, description="curated", description_auto=False, + ) + monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") + monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) + monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) + + outcome = describer.describe_profile("myprof") + assert outcome.ok is False + assert "already has a user-authored description" in outcome.reason + # Description unchanged + assert profiles_mod.read_profile_meta(profile_env)["description"] == "curated" + + +def test_describer_overwrite_flag_replaces_user_authored(profile_env, monkeypatch): + profiles_mod.write_profile_meta( + profile_env, description="curated", description_auto=False, + ) + monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") + monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) + monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) + + payload = jsonlib.dumps({"description": "new auto-gen"}) + with _patch_aux_client(payload), patch( + "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} + ): + outcome = describer.describe_profile("myprof", overwrite=True) + assert outcome.ok, outcome.reason + meta = profiles_mod.read_profile_meta(profile_env) + assert meta["description"] == "new auto-gen" + assert meta["description_auto"] is True + + +def test_describer_handles_malformed_llm_response(profile_env, monkeypatch): + monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") + monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) + monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) + + # Non-JSON: describer falls back to taking the first paragraph as the description. + with _patch_aux_client("Plain text description that sneaks in"), patch( + "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} + ): + outcome = describer.describe_profile("myprof") + assert outcome.ok + assert "Plain text description" in (outcome.description or "") + + +def test_describer_returns_false_when_profile_missing(profile_env, monkeypatch): + monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: False) + monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) + outcome = describer.describe_profile("ghost") + assert outcome.ok is False + assert "not found" in outcome.reason diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index 4cfc80191f1..37e52707cae 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -411,6 +411,7 @@ Multi-profile, multi-project collaboration board. Each install can host many boa | `dispatch` | One dispatcher pass on the active board. Flags: `--dry-run`, `--max N`, `--json`. | | `context ` | Print the full context a worker would see (title + body + parent results + comments). | | `specify ` / `specify --all` | Flesh out a triage-column task into a concrete spec (title + body with goal, approach, acceptance criteria) via the auxiliary LLM, then promote it to `todo`. Flags: `--tenant` (scope `--all` to one tenant), `--author`, `--json`. Configure the model under `auxiliary.triage_specifier` in `config.yaml`. | +| `decompose ` / `decompose --all` | Fan a triage-column task out into a graph of child tasks routed to specialist profiles by description (the orchestrator-driven path). Falls back to specify-style single-task promotion when the LLM decides the task doesn't benefit from fan-out. Same flags as `specify`. Configure the model under `auxiliary.kanban_decomposer` in `config.yaml`. Also runs automatically every dispatcher tick when `kanban.auto_decompose: true` (the default). See [Auto vs Manual orchestration](/docs/user-guide/features/kanban#auto-vs-manual-orchestration). | | `gc` | Remove scratch workspaces for archived tasks. | Examples: diff --git a/website/docs/reference/profile-commands.md b/website/docs/reference/profile-commands.md index 376394a637e..467134b6d05 100644 --- a/website/docs/reference/profile-commands.md +++ b/website/docs/reference/profile-commands.md @@ -83,6 +83,7 @@ Creates a new profile. | `--clone-all` | Copy everything (config, memories, skills, sessions, state) from the current profile. | | `--clone-from ` | Clone from a specific profile instead of the current one. Used with `--clone` or `--clone-all`. | | `--no-alias` | Skip wrapper script creation. | +| `--description ""` | One- or two-sentence description of what this profile is good at. Used by the kanban orchestrator to route tasks based on role instead of profile name alone. Skip and add later via `hermes profile describe`. Persisted in `/profile.yaml`. | Creating a profile does **not** make that profile directory the default project/workspace directory for terminal commands. If you want a profile to start in a specific project, set `terminal.cwd` in that profile's `config.yaml`. @@ -102,6 +103,40 @@ hermes profile create backup --clone-all hermes profile create work2 --clone --clone-from work ``` +## `hermes profile describe` + +```bash +hermes profile describe [] [options] +``` + +Read or set a profile's description. The description is consumed by the kanban orchestrator to route tasks based on what each profile is good at, rather than guessing from the profile name alone. Persisted in `/profile.yaml` so it survives reboots and is shared with the gateway. + +With no flags, prints the current description (or `(no description set for '')` if empty). + +| Argument / Option | Description | +|-------------------|-------------| +| `` | Profile to describe. Required unless `--all --auto` is used. | +| `--text ""` | Set the description to this exact text (user-authored). Overwrites any existing description. | +| `--auto` | Auto-generate a 1-2 sentence description via the auxiliary LLM, based on the profile's installed skills, configured model, and name. Configure the model under `auxiliary.profile_describer` in `config.yaml`. Auto-generated descriptions are marked `description_auto: true` so the dashboard can flag them for review. | +| `--overwrite` | With `--auto`, replace user-authored descriptions too (default: skip profiles whose description was set explicitly). | +| `--all` | With `--auto`, sweep every profile missing a description. | + +**Examples:** + +```bash +# Read the current description +hermes profile describe researcher + +# Set it explicitly +hermes profile describe researcher --text "Reads source code and writes findings." + +# Let the LLM generate one +hermes profile describe researcher --auto + +# Fill in descriptions for every profile that doesn't have one +hermes profile describe --all --auto +``` + ## `hermes profile delete` ```bash diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 5ac0d8c9df2..d972b38b384 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -785,6 +785,8 @@ $ hermes model [ ] compression currently: auto / main model [ ] approval currently: auto / main model [ ] triage_specifier currently: auto / main model +[ ] kanban_decomposer currently: auto / main model +[ ] profile_describer currently: auto / main model ``` Select a task, pick a provider (OAuth flows open a browser; API-key providers prompt), pick a model. The change persists to `auxiliary..*` in `config.yaml`. Same machinery as the main-model picker — no extra syntax to learn. diff --git a/website/docs/user-guide/features/kanban-tutorial.md b/website/docs/user-guide/features/kanban-tutorial.md index 5f79569c7bc..88a0f9cf5ec 100644 --- a/website/docs/user-guide/features/kanban-tutorial.md +++ b/website/docs/user-guide/features/kanban-tutorial.md @@ -22,7 +22,7 @@ Throughout the tutorial, **code blocks labelled `bash` are commands *you* run.** Six columns, left to right: -- **Triage** — raw ideas, a specifier will flesh out the spec before anyone works on them. Click the **✨ Specify** button on any triage card (or run `hermes kanban specify ` / `/kanban specify ` from a chat) to have the auxiliary LLM turn a one-liner into a full spec (goal, approach, acceptance criteria) and promote it to `todo` in one shot. Configure which model runs it under `auxiliary.triage_specifier` in `config.yaml`. +- **Triage** — raw ideas. By default the dispatcher auto-runs the **decomposer** (orchestrator-driven fan-out) on tasks here: it reads your profile roster + descriptions and produces a graph of child tasks routed to the best-fit specialists, with the original task held alive as the parent so the orchestrator wakes back up to judge completion when everything finishes. Flip the **Orchestration: Auto/Manual** pill at the top of the kanban page to switch modes. In Manual mode (or for setups without an orchestrator profile) click **⚗ Decompose** on a card, or run `hermes kanban decompose ` / `/kanban decompose `. For single tasks that don't need fan-out, **✨ Specify** does a one-shot spec rewrite (goal, approach, acceptance criteria) and promotes to `todo`. Configure the models under `auxiliary.kanban_decomposer` and `auxiliary.triage_specifier` in `config.yaml`. See [Auto vs Manual orchestration](./kanban#auto-vs-manual-orchestration) in the main Kanban guide. - **Todo** — created but waiting on dependencies, or not yet assigned. - **Ready** — assigned and waiting for the dispatcher to claim. - **In progress** — a worker is actively running the task. With "Lanes by profile" on (the default), this column sub-groups by assignee so you can see at a glance what each worker is doing. diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md index 91c6dacde67..7328fc4b615 100644 --- a/website/docs/user-guide/features/kanban.md +++ b/website/docs/user-guide/features/kanban.md @@ -444,7 +444,7 @@ hermes dashboard # "Kanban" tab appears in the nav, after "Skills" ### What the plugin gives you - A **Kanban** tab showing one column per status: `triage`, `todo`, `ready`, `running`, `blocked`, `done` (plus `archived` when the toggle is on). - - `triage` is the parking column for rough ideas a specifier is expected to flesh out. Tasks created with `hermes kanban create --triage` (or via the Triage column's inline create) land here and the dispatcher leaves them alone until a human or specifier promotes them to `todo` / `ready`. Run `hermes kanban specify ` to have the auxiliary LLM expand a triage task into a concrete spec (title + body with goal, approach, acceptance criteria) and promote it to `todo` in one shot; `--all` sweeps every triage task at once. Configure which model runs the specifier under `auxiliary.triage_specifier` in `config.yaml`. + - `triage` is the parking column for rough ideas. By default (`kanban.auto_decompose: true`), the dispatcher auto-runs the **decomposer** on tasks that land here — the orchestrator profile reads the rough idea, looks at your profile roster (with descriptions), and fans the task out into a small graph of child tasks routed to the best-fit specialists. The original task stays alive as the parent of every child so the orchestrator wakes back up to judge completion when everything finishes. Flip the **Orchestration: Auto/Manual** pill at the top of the page (or set `kanban.auto_decompose: false`) to switch to manual mode, where triage tasks stay put until you click **⚗ Decompose** on a card or run `hermes kanban decompose `. For tasks that don't need fan-out (or for setups without an orchestrator profile), the **✨ Specify** button does a single-task spec rewrite (title + body with goal, approach, acceptance criteria) via the same LLM machinery. See [Auto vs Manual orchestration](#auto-vs-manual-orchestration) below. - Cards show the task id, title, priority badge, tenant tag, assigned profile, comment/link counts, a **progress pill** (`N/M` children done when the task has dependents), and "created N ago". A per-card checkbox enables multi-select. - **Per-profile lanes inside Running** — toolbar checkbox toggles sub-grouping of the Running column by assignee. - **Live updates via WebSocket** — the plugin tails the append-only `task_events` table on a short poll interval; the board reflects changes the instant any profile (CLI, gateway, or another dashboard tab) acts. Reloads are debounced so a burst of events triggers a single refetch. @@ -456,12 +456,40 @@ hermes dashboard # "Kanban" tab appears in the nav, after "Skills" - **Editable assignee / priority** — click the meta row to rewrite. - **Editable description** — markdown-rendered by default (headings, bold, italic, inline code, fenced code, `http(s)` / `mailto:` links, bullet lists), with an "edit" button that swaps in a textarea. Markdown rendering is a tiny, XSS-safe renderer — every substitution runs on HTML-escaped input, only `http(s)` / `mailto:` links pass through, and `target="_blank"` + `rel="noopener noreferrer"` are always set. - **Dependency editor** — chip list of parents and children, each with an `×` to unlink, plus dropdowns over every other task to add a new parent or child. Cycle attempts are rejected server-side with a clear message. - - **Status action row** (→ triage / → ready / → running / block / unblock / complete / archive) with confirm prompts for destructive transitions. For cards in the **Triage** column the row also exposes a **✨ Specify** button that calls the auxiliary LLM (`auxiliary.triage_specifier` in `config.yaml`) to expand the one-liner into a concrete spec (title + body with goal, approach, acceptance criteria) and promote the task to `todo`. The same behaviour is reachable from the CLI (`hermes kanban specify ` / `--all`), from any gateway platform (`/kanban specify `), and programmatically via `POST /api/plugins/kanban/tasks/:id/specify`. + - **Status action row** (→ triage / → ready / → running / block / unblock / complete / archive) with confirm prompts for destructive transitions. For cards in the **Triage** column the row also exposes two LLM-driven actions: **⚗ Decompose** fans the task out into a graph of child tasks routed to specialist profiles by description (the orchestrator-driven path), and **✨ Specify** does a single-task spec rewrite. Decompose falls back to specify-style promotion when the LLM decides the task doesn't benefit from fan-out, so it's a strict superset. Both are reachable from the CLI (`hermes kanban decompose ` / `specify ` / `--all`), from any gateway platform (`/kanban decompose `), and programmatically via `POST /api/plugins/kanban/tasks/:id/decompose` and `…/specify`. Configure the models under `auxiliary.kanban_decomposer` and `auxiliary.triage_specifier` in `config.yaml`. - Result section (also markdown-rendered), comment thread with Enter-to-submit, the last 20 events. - **Toolbar filters** — free-text search, tenant dropdown (defaults to `dashboard.kanban.default_tenant` from `config.yaml`), assignee dropdown, "show archived" toggle, "lanes by profile" toggle, and a **Nudge dispatcher** button so you don't have to wait for the next 60 s tick. Visually the target is the familiar Linear / Fusion layout: dark theme, column headers with counts, coloured status dots, pill chips for priority and tenant. The plugin reads only theme CSS vars (`--color-*`, `--radius`, `--font-mono`, ...), so it reskins automatically with whichever dashboard theme is active. +### Auto vs Manual orchestration + +The kanban board has two ways to handle a task you drop into the Triage column: + +**Auto (default)** — `kanban.auto_decompose: true`. The gateway-embedded dispatcher runs the **decomposer** on each tick, capped by `kanban.auto_decompose_per_tick` (default 3 tasks per tick) so a bulk-load of triage tasks doesn't burst-spend the auxiliary LLM. The decomposer reads the rough idea, looks at your installed profiles + their descriptions, and asks the LLM to produce a JSON task graph: which tasks to spawn, who they go to, and which depend on which. The original triage task becomes the parent of every leaf in the graph, so it stays alive until the whole graph completes — and then promotes back to `ready` so its assignee (the orchestrator profile) can judge completion and add more tasks if the work isn't done. This is the "drop a one-liner, walk away" flow. + +**Manual** — `kanban.auto_decompose: false`. Triage tasks stay in triage until you act. Click the **⚗ Decompose** button on a card, run `hermes kanban decompose ` (or `--all`), or use `/kanban decompose ` from a chat. This matches the pre-decomposer behavior of the board, useful when you want full control over what runs when. + +Flip between the two modes from the **Orchestration: Auto/Manual** pill at the top of the kanban page (emerald = Auto, muted gray = Manual), or by editing `config.yaml` directly. Both modes coexist with `hermes kanban specify` — that's still available as a single-task spec rewrite when you don't want fan-out. + +The decomposer's routing decisions depend on profile descriptions, which is a per-profile labeling primitive you set with `hermes profile create --description "..."`, `hermes profile describe --text "..."`, `hermes profile describe --auto` (LLM-generates from the profile's installed skills + model), or the dashboard's per-profile editor in the expanded **Orchestration settings** panel. Profiles without a description still appear in the roster — they're routable by name, just less precisely. The decomposer NEVER lands a child task with `assignee=None`: when the LLM picks an unknown profile, the child gets routed to `kanban.default_assignee` (or the active default profile if that's unset). + +Config knobs (all under `kanban:` in `~/.hermes/config.yaml`): + +| Key | Default | Purpose | +|---|---|---| +| `auto_decompose` | `true` | Dispatcher auto-runs the decomposer every tick. | +| `auto_decompose_per_tick` | `3` | Cap on decompositions per dispatcher tick. Excess defers to the next tick. | +| `orchestrator_profile` | `""` | Profile that owns decomposition. Empty = fall back to active default profile. | +| `default_assignee` | `""` | Where a child task lands when the LLM picks an unknown profile. Empty = fall back to active default. | + +And the two auxiliary LLM slots: + +| Key | Purpose | +|---|---| +| `auxiliary.kanban_decomposer` | Model that produces the task graph (called by Decompose). Set `provider`/`model` to override the main chat model. | +| `auxiliary.profile_describer` | Model that auto-generates profile descriptions (called by `hermes profile describe --auto`). | + ### Architecture The GUI is strictly a **read-through-the-DB + write-through-kanban_db** layer with no domain logic of its own: @@ -499,6 +527,12 @@ All routes are mounted under `/api/plugins/kanban/` and protected by the dashboa | `POST` | `/tasks/bulk` | Apply the same patch (status / archive / assignee / priority) to every id in `ids`. Per-id failures reported without aborting siblings | | `POST` | `/tasks/:id/comments` | Append a comment | | `POST` | `/tasks/:id/specify` | Run the triage specifier — auxiliary LLM fleshes out the task body and promotes it from `triage` to `todo`. Returns `{ok, task_id, reason, new_title}`; `ok=false` with a human-readable reason on "not in triage" / no aux client / LLM error is a 200, not a 4xx | +| `POST` | `/tasks/:id/decompose` | Run the kanban decomposer — auxiliary LLM produces a task graph and the helper atomically creates the children + links the root + flips `triage → todo`. Returns `{ok, task_id, reason, fanout, child_ids, new_title}`. Same 200-on-LLM-error convention as `/specify`. | +| `GET` | `/profiles` | List installed profiles with their descriptions (consumed by the dashboard's profile-description editor and the orchestrator picker). | +| `PATCH` | `/profiles/:name` | Set or clear a profile's description (user-authored — `description_auto: false`). Returns `{ok, profile, description}`. | +| `POST` | `/profiles/:name/describe-auto` | Generate a description for a profile via `auxiliary.profile_describer`. Persists with `description_auto: true` so the dashboard can surface a "review" badge. | +| `GET` | `/orchestration` | Read the kanban orchestration settings (`orchestrator_profile`, `default_assignee`, `auto_decompose`) plus the *resolved* effective values after fallbacks. | +| `PUT` | `/orchestration` | Update one or more of the three orchestration keys in `config.yaml`. Validates that non-empty profile names actually exist. | | `POST` | `/links` | Add a dependency (`parent_id` → `child_id`) | | `DELETE` | `/links?parent_id=…&child_id=…` | Remove a dependency | | `POST` | `/dispatch?max=…&dry_run=…` | Nudge the dispatcher — skip the 60 s wait | diff --git a/website/docs/user-guide/profiles.md b/website/docs/user-guide/profiles.md index 522b24cb770..73ea0a8cadd 100644 --- a/website/docs/user-guide/profiles.md +++ b/website/docs/user-guide/profiles.md @@ -32,6 +32,14 @@ hermes profile create mybot Creates a fresh profile with bundled skills seeded. Run `mybot setup` to configure API keys, model, and gateway tokens. +If you plan to use this profile as a kanban worker (or want the kanban orchestrator to route work to it), pass `--description ""` at create time so the orchestrator knows what it's good at: + +```bash +hermes profile create researcher --description "Reads source code and external docs, writes findings." +``` + +You can also set or auto-generate the description later with `hermes profile describe` — see the [Kanban guide](./features/kanban#auto-vs-manual-orchestration) for the full routing model. + ### Clone config only (`--clone`) ```bash From 4c46c35ed0d3864f1cec55d87ab6d0f838ec7a2e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 14:44:37 -0700 Subject: [PATCH 131/142] docs(messaging): clarify admin/user split and signal future gating (#27623) Restructures the security section so the admin/user distinction is a first-class concept rather than buried under 'Slash Command Access Control'. The new section makes explicit that: - Slash commands are the first capability gated by the tier split today - Future gating (tools, model switching, etc.) will hang off the same admin/user distinction, so configuring it now is forward-compatible - Allowlists vs the admin/user split solve different problems and are contrasted up front Heading renamed: 'Slash Command Access Control' -> 'Admins vs Regular Users'. The platform-specific pages (telegram.md, discord.md) keep the old heading since slash gating IS the only thing they currently gate. --- website/docs/user-guide/messaging/index.md | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index acd12872812..ef02bc7fe16 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -222,9 +222,22 @@ hermes pairing revoke telegram 123456789 # Remove access Pairing codes expire after 1 hour, are rate-limited, and use cryptographic randomness. -### Slash Command Access Control +### Admins vs Regular Users -Once users are allowed in, you can split them into **admins** (full slash command access) and **regular users** (only the slash commands you explicitly enable). This applies per platform and per scope (DM vs group/channel) and works through the live command registry, so it covers built-in AND plugin-registered slash commands without per-feature wiring. +Allowlists answer "can this person reach the bot at all?" The **admin / user split** answers "now that they're in, what are they allowed to do?" + +Every allowed user falls into one of two tiers per scope (DM vs group/channel): + +- **Admin** — full access. Can run every registered slash command (built-in + plugin) and use every gated capability. +- **Regular user** — restricted access. Can chat with the agent normally, but can only run the slash commands you explicitly enable. The always-allowed floor is `/help` and `/whoami`. + +The tiers are configured per platform and per scope. DM admin status does not imply group/channel admin status — each scope has its own admin list. + +**What the tiers gate today:** slash commands. The split runs through the live command registry, so it covers built-ins and plugin-registered commands without per-feature wiring. Plain chat is not affected — non-admins can still talk to the agent. + +**What may be gated in the future:** more capability surfaces (tool access, model switching, expensive operations) will hang off the same admin / user distinction as we add them. Configuring the split now means those future restrictions land cleanly without you having to re-model who's an admin. + +#### Configuration ```yaml gateway: @@ -239,13 +252,9 @@ gateway: group_user_allowed_commands: [status] ``` -Behavior: +**Backward compat:** if `allow_admin_from` is not set for a scope, the tier split is disabled for that scope and every allowed user has full access. Existing installs keep working with no changes — opt in when you want the distinction. -- A user in `allow_admin_from` for a scope can run **every** registered slash command. -- A user in `allow_from` but not in `allow_admin_from` can only run commands in `user_allowed_commands`, plus the always-allowed floor: `/help` and `/whoami`. -- Plain chat is unaffected. Non-admins can still talk to the agent normally; they just can't trigger arbitrary commands. -- **Backward compat:** if `allow_admin_from` is not set for a scope, slash gating is disabled for that scope. Existing installs keep working with no changes. -- DM admin status does not imply group/channel admin status. Each scope has its own admin list. +#### Inspecting your access Use `/whoami` from any platform to see the active scope, your tier (admin / user / unrestricted), and which slash commands you can run. See the [Telegram](/docs/user-guide/messaging/telegram#slash-command-access-control) and [Discord](/docs/user-guide/messaging/discord#slash-command-access-control) pages for platform-specific examples. From c9055626232e1866fedcca8073d0c13ae62e7b90 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Sun, 17 May 2026 15:41:03 +1000 Subject: [PATCH 132/142] fix(auth): stop replaying invalid Nous refresh tokens Quarantine Nous OAuth state when refresh fails with terminal invalid_grant/invalid_token errors. Clear local and shared refresh material across runtime, managed access-token, proxy, and credential-pool paths so Hermes stops retrying revoked refresh sessions. --- agent/credential_pool.py | 41 +++++++ hermes_cli/auth.py | 118 +++++++++++++++++--- hermes_cli/proxy/adapters/nous_portal.py | 14 +++ tests/agent/test_credential_pool.py | 64 +++++++++++ tests/hermes_cli/test_auth_nous_provider.py | 84 ++++++++++++++ tests/hermes_cli/test_proxy.py | 31 +++++ 6 files changed, 338 insertions(+), 14 deletions(-) diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 7f27873a7fb..93e3d609ee8 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -929,6 +929,47 @@ class CredentialPool: self._persist() self._sync_device_code_entry_to_auth_store(updated) return updated + if auth_mod._is_terminal_nous_refresh_error(exc): + logger.debug("Nous refresh token is terminally invalid; clearing local token state") + try: + with _auth_store_lock(): + auth_store = _load_auth_store() + state = _load_provider_state(auth_store, "nous") or { + "client_id": entry.client_id, + "portal_base_url": entry.portal_base_url, + "inference_base_url": entry.inference_base_url, + "token_type": entry.token_type, + "scope": entry.scope, + "tls": entry.tls, + } + store_refresh = str(state.get("refresh_token") or "").strip() + entry_refresh = str(entry.refresh_token or "").strip() + if not store_refresh or store_refresh == entry_refresh: + auth_mod._quarantine_nous_oauth_state( + state, + exc, + reason="credential_pool_refresh_failure", + ) + _save_provider_state(auth_store, "nous", state) + _save_auth_store(auth_store) + except Exception as clear_exc: + logger.debug("Failed to clear terminal Nous OAuth state: %s", clear_exc) + + cleared = replace( + entry, + access_token=None, + refresh_token=None, + agent_key=None, + agent_key_expires_at=None, + ) + self._replace_entry(entry, cleared) + self._persist() + self._mark_exhausted( + cleared, + 401, + {"reason": getattr(exc, "code", None), "message": str(exc)}, + ) + return None self._mark_exhausted(entry, None) return None diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 8b154db7468..50f105de10a 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -3616,6 +3616,63 @@ def _read_shared_nous_state() -> Optional[Dict[str, Any]]: return payload +def _clear_shared_nous_state(reason: str) -> None: + """Remove the shared Nous OAuth store after a terminal token failure.""" + try: + with _nous_shared_store_lock(): + path = _nous_shared_store_path() + try: + path.unlink() + except FileNotFoundError: + pass + _oauth_trace("nous_shared_store_cleared", reason=reason) + except Exception as exc: + logger.debug("Failed to clear shared Nous auth store: %s", exc) + + +def _is_terminal_nous_refresh_error(exc: Exception) -> bool: + """True when retrying the same Nous refresh token cannot succeed.""" + return ( + isinstance(exc, AuthError) + and exc.provider == "nous" + and exc.code in {"invalid_grant", "invalid_token"} + and bool(exc.relogin_required) + ) + + +def _quarantine_nous_oauth_state( + state: Dict[str, Any], + error: AuthError, + *, + reason: str, +) -> None: + """Keep routing metadata but remove dead OAuth material so it is not replayed.""" + for key in ( + "access_token", + "refresh_token", + "expires_at", + "expires_in", + "obtained_at", + "agent_key", + "agent_key_id", + "agent_key_expires_at", + "agent_key_expires_in", + "agent_key_reused", + "agent_key_obtained_at", + ): + state.pop(key, None) + state["last_auth_error"] = { + "provider": "nous", + "code": error.code, + "message": str(error), + "reason": reason, + "relogin_required": True, + "at": datetime.now(timezone.utc).isoformat(), + } + _clear_shared_nous_state(reason) + invalidate_nous_auth_status_cache() + + def _try_import_shared_nous_state( *, timeout_seconds: float = 15.0, @@ -3671,6 +3728,8 @@ def _try_import_shared_nous_state( error_type=type(exc).__name__, error_code=getattr(exc, "code", None), ) + if _is_terminal_nous_refresh_error(exc): + _clear_shared_nous_state("shared_import_terminal_refresh_failure") logger.debug("Shared Nous import failed: %s", exc) return None except Exception as exc: @@ -3896,12 +3955,23 @@ def resolve_nous_access_token( headers={"Accept": "application/json"}, verify=verify, ) as client: - refreshed = _refresh_access_token( - client=client, - portal_base_url=portal_base_url, - client_id=client_id, - refresh_token=refresh_token, - ) + try: + refreshed = _refresh_access_token( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + refresh_token=refresh_token, + ) + except AuthError as exc: + if _is_terminal_nous_refresh_error(exc): + _quarantine_nous_oauth_state( + state, + exc, + reason="managed_access_token_refresh_failure", + ) + _save_provider_state(auth_store, "nous", state) + _save_auth_store(auth_store) + raise now = datetime.now(timezone.utc) access_ttl = _coerce_ttl_seconds(refreshed.get("expires_in")) @@ -4209,10 +4279,20 @@ def resolve_nous_runtime_credentials( reason="access_expiring", refresh_token_fp=_token_fingerprint(refresh_token), ) - refreshed = _refresh_access_token( - client=client, portal_base_url=portal_base_url, - client_id=client_id, refresh_token=refresh_token, - ) + try: + refreshed = _refresh_access_token( + client=client, portal_base_url=portal_base_url, + client_id=client_id, refresh_token=refresh_token, + ) + except AuthError as exc: + if _is_terminal_nous_refresh_error(exc): + _quarantine_nous_oauth_state( + state, + exc, + reason="runtime_access_refresh_failure", + ) + _persist_state("terminal_runtime_access_refresh_failure") + raise now = datetime.now(timezone.utc) access_ttl = _coerce_ttl_seconds(refreshed.get("expires_in")) previous_refresh_token = refresh_token @@ -4283,10 +4363,20 @@ def resolve_nous_runtime_credentials( reason="mint_retry_after_invalid_token", refresh_token_fp=_token_fingerprint(latest_refresh_token), ) - refreshed = _refresh_access_token( - client=client, portal_base_url=portal_base_url, - client_id=client_id, refresh_token=latest_refresh_token, - ) + try: + refreshed = _refresh_access_token( + client=client, portal_base_url=portal_base_url, + client_id=client_id, refresh_token=latest_refresh_token, + ) + except AuthError as exc: + if _is_terminal_nous_refresh_error(exc): + _quarantine_nous_oauth_state( + state, + exc, + reason="runtime_mint_retry_refresh_failure", + ) + _persist_state("terminal_runtime_mint_retry_refresh_failure") + raise now = datetime.now(timezone.utc) access_ttl = _coerce_ttl_seconds(refreshed.get("expires_in")) state["access_token"] = refreshed["access_token"] diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index b72cbd305b3..842489659a4 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -16,8 +16,11 @@ import threading from typing import Any, Dict, FrozenSet, Optional from hermes_cli.auth import ( + AuthError, DEFAULT_NOUS_INFERENCE_URL, _load_auth_store, + _is_terminal_nous_refresh_error, + _quarantine_nous_oauth_state, _save_auth_store, _write_shared_nous_state, refresh_nous_oauth_from_state, @@ -81,6 +84,17 @@ class NousPortalAdapter(UpstreamAdapter): try: refreshed = refresh_nous_oauth_from_state(state) + except AuthError as exc: + if _is_terminal_nous_refresh_error(exc): + _quarantine_nous_oauth_state( + state, + exc, + reason="proxy_refresh_failure", + ) + self._save_state(state) + raise RuntimeError( + f"Failed to refresh Nous Portal credentials: {exc}" + ) from exc except Exception as exc: raise RuntimeError( f"Failed to refresh Nous Portal credentials: {exc}" diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index 299567a9a6f..e2d2726f21b 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -510,6 +510,70 @@ def test_load_pool_migrates_nous_provider_state(tmp_path, monkeypatch): assert entry.agent_key == "agent-key" +def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared")) + _write_auth_store( + tmp_path, + { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "inference_base_url": "https://inference.example.com/v1", + "client_id": "hermes-cli", + "token_type": "Bearer", + "scope": "inference:mint_agent_key", + "access_token": "access-token", + "refresh_token": "refresh-token", + "expires_at": "2026-03-24T12:00:00+00:00", + "agent_key": "agent-key", + "agent_key_expires_at": "2026-03-24T13:30:00+00:00", + } + }, + }, + ) + + from agent.credential_pool import load_pool + from hermes_cli import auth as auth_mod + from hermes_cli.auth import AuthError + + refresh_calls = {"count": 0} + + def _terminal_refresh_failure(*_args, **_kwargs): + refresh_calls["count"] += 1 + raise AuthError( + "Refresh session has been revoked", + provider="nous", + code="invalid_grant", + relogin_required=True, + ) + + monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _terminal_refresh_failure) + + pool = load_pool("nous") + assert pool.select() is not None + assert pool.try_refresh_current() is None + + entry = pool.entries()[0] + assert entry.last_status == "exhausted" + assert entry.last_error_code == 401 + assert entry.refresh_token is None + assert entry.access_token is None + assert entry.agent_key is None + + auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text()) + nous_state = auth_payload["providers"]["nous"] + assert not nous_state.get("refresh_token") + assert not nous_state.get("access_token") + assert not nous_state.get("agent_key") + assert nous_state["last_auth_error"]["code"] == "invalid_grant" + + assert pool.try_refresh_current() is None + assert refresh_calls["count"] == 1 + + def test_load_pool_removes_stale_file_backed_singleton_entry(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index 5cd546462dd..37662c77ece 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -373,6 +373,89 @@ def test_refresh_token_persisted_when_mint_times_out(tmp_path, monkeypatch): assert state_after_failure["access_token"] == "access-1" +def test_terminal_refresh_failure_quarantines_tokens( + tmp_path, monkeypatch, shared_store_env, +): + """A revoked/invalid Nous refresh token must not be replayed forever.""" + from hermes_cli import auth as auth_mod + + hermes_home = tmp_path / "hermes" + _setup_nous_auth(hermes_home, refresh_token="refresh-old") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + shared_state = _full_state_fixture() + shared_state["access_token"] = "access-old" + shared_state["refresh_token"] = "refresh-old" + shared_state["expires_at"] = "2026-02-01T00:00:00+00:00" + auth_mod._write_shared_nous_state(shared_state) + + refresh_calls: list[str] = [] + + def _terminal_refresh_failure(*, client, portal_base_url, client_id, refresh_token): + refresh_calls.append(refresh_token) + raise AuthError( + "Refresh session has been revoked", + provider="nous", + code="invalid_grant", + relogin_required=True, + ) + + monkeypatch.setattr(auth_mod, "_refresh_access_token", _terminal_refresh_failure) + + with pytest.raises(AuthError, match="Refresh session has been revoked"): + auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + state_after_failure = auth_mod.get_provider_auth_state("nous") + assert state_after_failure is not None + assert not state_after_failure.get("refresh_token") + assert not state_after_failure.get("access_token") + assert not state_after_failure.get("agent_key") + assert state_after_failure["last_auth_error"]["code"] == "invalid_grant" + assert auth_mod._read_shared_nous_state() is None + + with pytest.raises(AuthError, match="No access token found"): + auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert refresh_calls == ["refresh-old"] + + +def test_managed_access_token_refresh_failure_quarantines_tokens( + tmp_path, monkeypatch, shared_store_env, +): + from hermes_cli import auth as auth_mod + + hermes_home = tmp_path / "hermes" + _setup_nous_auth(hermes_home, refresh_token="refresh-old") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + refresh_calls: list[str] = [] + + def _terminal_refresh_failure(*, client, portal_base_url, client_id, refresh_token): + refresh_calls.append(refresh_token) + raise AuthError( + "Invalid refresh token", + provider="nous", + code="invalid_grant", + relogin_required=True, + ) + + monkeypatch.setattr(auth_mod, "_refresh_access_token", _terminal_refresh_failure) + + with pytest.raises(AuthError, match="Invalid refresh token"): + auth_mod.resolve_nous_access_token() + + state_after_failure = auth_mod.get_provider_auth_state("nous") + assert state_after_failure is not None + assert not state_after_failure.get("refresh_token") + assert not state_after_failure.get("access_token") + assert state_after_failure["last_auth_error"]["message"] == "Invalid refresh token" + + with pytest.raises(AuthError, match="No access token found"): + auth_mod.resolve_nous_access_token() + + assert refresh_calls == ["refresh-old"] + + def test_mint_retry_uses_latest_rotated_refresh_token(tmp_path, monkeypatch): hermes_home = tmp_path / "hermes" _setup_nous_auth(hermes_home, refresh_token="refresh-old") @@ -1118,6 +1201,7 @@ def test_try_import_shared_returns_none_on_refresh_failure( monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _boom) assert auth_mod._try_import_shared_nous_state() is None + assert auth_mod._read_shared_nous_state() is None def test_try_import_shared_rehydrates_on_success(shared_store_env, monkeypatch): diff --git a/tests/hermes_cli/test_proxy.py b/tests/hermes_cli/test_proxy.py index 0c874facac7..3ab06eeb92f 100644 --- a/tests/hermes_cli/test_proxy.py +++ b/tests/hermes_cli/test_proxy.py @@ -164,6 +164,37 @@ def test_nous_adapter_get_credential_raises_on_refresh_failure(tmp_path, monkeyp adapter.get_credential() +def test_nous_adapter_quarantines_terminal_refresh_failure(tmp_path, monkeypatch): + from hermes_cli.auth import AuthError + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + _write_auth_store(tmp_path, { + "access_token": "access-tok", + "refresh_token": "refresh-tok", + "agent_key": "stale-agent-key", + }) + + with patch( + "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + side_effect=AuthError( + "Refresh session has been revoked", + provider="nous", + code="invalid_grant", + relogin_required=True, + ), + ): + adapter = NousPortalAdapter() + with pytest.raises(RuntimeError, match="Refresh session has been revoked"): + adapter.get_credential() + + stored = json.loads((tmp_path / "auth.json").read_text()) + nous_state = stored["providers"]["nous"] + assert not nous_state.get("refresh_token") + assert not nous_state.get("access_token") + assert not nous_state.get("agent_key") + assert nous_state["last_auth_error"]["code"] == "invalid_grant" + + def test_nous_adapter_get_credential_raises_when_no_agent_key_returned(tmp_path, monkeypatch): """If the refresh helper succeeds but produces no agent_key, we surface a clear error.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) From 89a3d038cfb289ce73b9d7aac9b0b7ca85a018f0 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Sun, 17 May 2026 19:34:44 +1000 Subject: [PATCH 133/142] Switch to JWT token for inference against Nous, falling back to old opaque token on failure. --- agent/auxiliary_client.py | 7 +- agent/credential_pool.py | 2 + hermes_cli/auth.py | 364 ++++++++++++++++++-- hermes_cli/proxy/adapters/nous_portal.py | 11 +- hermes_cli/runtime_provider.py | 15 +- tests/agent/test_auxiliary_client.py | 2 + tests/agent/test_credential_pool.py | 56 +++ tests/conftest.py | 1 + tests/hermes_cli/test_auth_commands.py | 4 +- tests/hermes_cli/test_auth_nous_provider.py | 363 ++++++++++++++++++- 10 files changed, 780 insertions(+), 45 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index a7fcd311f11..b2733fd8a1b 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -755,7 +755,8 @@ class _CodexCompletionsAdapter: def _check_cancelled() -> None: if deadline is not None and time.monotonic() >= deadline: - timed_out.set() + if not timed_out.is_set(): + _close_client_on_timeout() raise TimeoutError(_timeout_message()) try: from tools.interrupt import is_interrupted @@ -1233,7 +1234,7 @@ def _read_nous_auth() -> Optional[dict]: def _nous_api_key(provider: dict) -> str: - """Extract the best API key from a Nous provider state dict.""" + """Extract the Nous runtime credential from the compatibility field.""" return provider.get("agent_key") or provider.get("access_token", "") @@ -1246,7 +1247,7 @@ def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[ """Return fresh Nous runtime credentials when available. This mirrors the main agent's 401 recovery path and keeps auxiliary - clients aligned with the singleton auth store + mint flow instead of + clients aligned with the singleton auth store + JWT/mint flow instead of relying only on whatever raw tokens happen to be sitting in auth.json or the credential pool. """ diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 93e3d609ee8..b1c41977d51 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -166,6 +166,8 @@ class PooledCredential: @property def runtime_api_key(self) -> str: if self.provider == "nous": + # Nous stores the runtime inference credential in agent_key for + # compatibility. It may be a NAS invoke JWT or legacy opaque key. return str(self.agent_key or self.access_token or "") return str(self.access_token or "") diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 50f105de10a..2a670589d48 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -67,9 +67,13 @@ AUTH_LOCK_TIMEOUT_SECONDS = 15.0 DEFAULT_NOUS_PORTAL_URL = "https://portal.nousresearch.com" DEFAULT_NOUS_INFERENCE_URL = "https://inference-api.nousresearch.com/v1" DEFAULT_NOUS_CLIENT_ID = "hermes-cli" -DEFAULT_NOUS_SCOPE = "inference:mint_agent_key" +NOUS_LEGACY_AGENT_KEY_SCOPE = "inference:mint_agent_key" +NOUS_INFERENCE_INVOKE_SCOPE = "inference:invoke" +DEFAULT_NOUS_SCOPE = f"{NOUS_INFERENCE_INVOKE_SCOPE} {NOUS_LEGACY_AGENT_KEY_SCOPE}" +NOUS_LEGACY_SESSION_KEYS_ENV = "HERMES_AGENT_USE_LEGACY_SESSION_KEYS" DEFAULT_AGENT_KEY_MIN_TTL_SECONDS = 30 * 60 # 30 minutes ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 # refresh 2 min before expiry +NOUS_INVOKE_JWT_MIN_TTL_SECONDS = ACCESS_TOKEN_REFRESH_SKEW_SECONDS DEVICE_AUTH_POLL_INTERVAL_CAP_SECONDS = 1 # poll at most every 1s DEFAULT_CODEX_BASE_URL = "https://chatgpt.com/backend-api/codex" DEFAULT_XAI_OAUTH_BASE_URL = "https://api.x.ai/v1" @@ -1549,6 +1553,117 @@ def _decode_jwt_claims(token: Any) -> Dict[str, Any]: return claims if isinstance(claims, dict) else {} +def _scope_values(raw_scope: Any) -> set[str]: + scopes: set[str] = set() + if isinstance(raw_scope, str): + for part in raw_scope.replace(",", " ").split(): + cleaned = part.strip() + if cleaned: + scopes.add(cleaned) + elif isinstance(raw_scope, (list, tuple, set, frozenset)): + for item in raw_scope: + if isinstance(item, str): + scopes.update(_scope_values(item)) + return scopes + + +def _nous_legacy_session_keys_forced() -> bool: + return is_truthy_value(os.getenv(NOUS_LEGACY_SESSION_KEYS_ENV), default=False) + + +def _nous_scope_has_invoke(raw_scope: Any) -> bool: + return NOUS_INFERENCE_INVOKE_SCOPE in _scope_values(raw_scope) + + +def _nous_invoke_jwt_is_usable( + token: Any, + *, + scope: Any = None, + expires_at: Any = None, + min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, +) -> bool: + claims = _decode_jwt_claims(token) + if not claims: + return False + scopes = ( + _scope_values(scope) + | _scope_values(claims.get("scope")) + | _scope_values(claims.get("scp")) + ) + if NOUS_INFERENCE_INVOKE_SCOPE not in scopes: + return False + exp = claims.get("exp") + skew = max(0, int(min_ttl_seconds)) + if isinstance(exp, (int, float)): + return float(exp) > (time.time() + skew) + return not _is_expiring(expires_at, skew) + + +def _nous_invoke_jwt_unavailable_reason( + token: Any, + *, + scope: Any = None, + expires_at: Any = None, + min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, +) -> str: + claims = _decode_jwt_claims(token) + if not claims: + return "access_token_not_jwt" + scopes = ( + _scope_values(scope) + | _scope_values(claims.get("scope")) + | _scope_values(claims.get("scp")) + ) + if NOUS_INFERENCE_INVOKE_SCOPE not in scopes: + return "missing_inference_invoke_scope" + exp = claims.get("exp") + skew = max(0, int(min_ttl_seconds)) + if isinstance(exp, (int, float)) and float(exp) <= (time.time() + skew): + return "invoke_jwt_expiring" + if not isinstance(exp, (int, float)) and _is_expiring(expires_at, skew): + return "invoke_jwt_expiry_unknown_or_expiring" + return "invoke_jwt_unavailable" + + +def _nous_jwt_expires_at(token: Any, fallback_expires_at: Any = None) -> Optional[str]: + claims = _decode_jwt_claims(token) + exp = claims.get("exp") + if isinstance(exp, (int, float)): + try: + return datetime.fromtimestamp(float(exp), tz=timezone.utc).isoformat() + except Exception: + pass + return fallback_expires_at if isinstance(fallback_expires_at, str) else None + + +def _set_nous_agent_key_from_invoke_jwt( + state: Dict[str, Any], + *, + obtained_at: Optional[str] = None, +) -> None: + access_token = state.get("access_token") + if not isinstance(access_token, str) or not access_token.strip(): + return + now = datetime.now(timezone.utc) + effective_obtained_at = obtained_at or now.isoformat() + expires_at = _nous_jwt_expires_at(access_token, state.get("expires_at")) + expires_epoch = _parse_iso_timestamp(expires_at) + expires_in = ( + max(0, int(expires_epoch - time.time())) + if expires_epoch is not None + else _coerce_ttl_seconds(state.get("expires_in")) + ) + if expires_at: + state["expires_at"] = expires_at + state["expires_in"] = expires_in + state["agent_key"] = access_token + state["agent_key_id"] = None + state["agent_key_expires_at"] = expires_at + state["agent_key_expires_in"] = expires_in + state["agent_key_reused"] = False + state["agent_key_obtained_at"] = effective_obtained_at + + def _codex_access_token_is_expiring(access_token: Any, skew_seconds: int) -> bool: claims = _decode_jwt_claims(access_token) exp = claims.get("exp") @@ -3333,6 +3448,34 @@ def _request_device_code( return data +def _is_nous_invoke_scope_refusal(exc: Exception) -> bool: + if not isinstance(exc, httpx.HTTPStatusError): + return False + response = exc.response + if response.status_code not in {400, 401, 403}: + return False + try: + payload = response.json() + except Exception: + payload = {} + text = " ".join( + str(value) + for value in ( + payload.get("error") if isinstance(payload, dict) else None, + payload.get("error_description") if isinstance(payload, dict) else None, + response.text, + ) + if value + ).lower() + if not text: + return False + return ( + "invalid_scope" in text + or "unsupported_scope" in text + or "scope" in text and NOUS_INFERENCE_INVOKE_SCOPE in text + ) + + def _poll_for_token( client: httpx.Client, portal_base_url: str, @@ -3524,8 +3667,9 @@ def _write_shared_nous_state(state: Dict[str, Any]) -> None: is a convenience layer; the per-profile auth.json remains the source of truth. - We deliberately omit the short-lived ``agent_key`` (24h TTL, profile- - specific) — only the long-lived OAuth tokens are cross-profile useful. + We deliberately omit the runtime ``agent_key`` compatibility field + (either an invoke JWT or legacy opaque session key) — only OAuth tokens + are cross-profile useful. """ refresh_token = state.get("refresh_token") access_token = state.get("access_token") @@ -3894,6 +4038,14 @@ def _agent_key_is_usable(state: Dict[str, Any], min_ttl_seconds: int) -> bool: key = state.get("agent_key") if not isinstance(key, str) or not key.strip(): return False + if _decode_jwt_claims(key): + if _nous_legacy_session_keys_forced(): + return False + return _nous_invoke_jwt_is_usable( + key, + scope=state.get("scope"), + expires_at=state.get("agent_key_expires_at"), + ) return not _is_expiring(state.get("agent_key_expires_at"), min_ttl_seconds) @@ -4039,7 +4191,23 @@ def refresh_nous_oauth_pure( timeout = httpx.Timeout(timeout_seconds if timeout_seconds else 15.0) with httpx.Client(timeout=timeout, headers={"Accept": "application/json"}, verify=verify) as client: - if force_refresh or _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS): + min_agent_key_ttl = max(60, int(min_key_ttl_seconds)) + legacy_session_keys = _nous_legacy_session_keys_forced() + current_invoke_jwt_usable = ( + not legacy_session_keys + and _nous_invoke_jwt_is_usable( + state.get("access_token"), + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ) + if ( + force_refresh + or ( + _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS) + and not current_invoke_jwt_usable + ) + ): refreshed = _refresh_access_token( client=client, portal_base_url=state["portal_base_url"], @@ -4061,7 +4229,39 @@ def refresh_nous_oauth_pure( now.timestamp() + access_ttl, tz=timezone.utc ).isoformat() - if force_mint or not _agent_key_is_usable(state, max(60, int(min_key_ttl_seconds))): + if ( + not legacy_session_keys + and _nous_invoke_jwt_is_usable( + state.get("access_token"), + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ): + _set_nous_agent_key_from_invoke_jwt(state) + logger.info("Nous inference auth: using NAS invoke JWT") + _oauth_trace( + "nous_invoke_jwt_selected", + access_token_fp=_token_fingerprint(state.get("access_token")), + ) + elif force_mint or not _agent_key_is_usable(state, min_agent_key_ttl): + fallback_reason = ( + "forced_legacy_session_keys" + if legacy_session_keys + else _nous_invoke_jwt_unavailable_reason( + state.get("access_token"), + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ) + logger.info( + "Nous inference auth: using legacy session key path (%s)", + fallback_reason, + ) + _oauth_trace( + "nous_legacy_session_key_selected", + reason=fallback_reason, + access_token_fp=_token_fingerprint(state.get("access_token")), + ) mint_payload = _mint_agent_key( client=client, portal_base_url=state["portal_base_url"], @@ -4175,6 +4375,15 @@ def persist_nous_credentials( ) +def _sync_nous_pool_from_auth_store() -> None: + try: + from agent.credential_pool import load_pool + + load_pool("nous") + except Exception as exc: + logger.debug("Failed to sync Nous credential pool from auth store: %s", exc) + + def resolve_nous_runtime_credentials( *, min_key_ttl_seconds: int = DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, @@ -4191,7 +4400,7 @@ def resolve_nous_runtime_credentials( Concurrent processes coordinate through the auth store file lock. Returns dict with: provider, base_url, api_key, key_id, expires_at, - expires_in, source ("cache" or "portal"). + expires_in, source ("invoke_jwt", "cache", or "portal"), and auth_path. """ min_key_ttl_seconds = max(60, int(min_key_ttl_seconds)) sequence_id = uuid.uuid4().hex[:12] @@ -4260,15 +4469,35 @@ def resolve_nous_runtime_credentials( raise AuthError("No access token found for Nous Portal login.", provider="nous", relogin_required=True) - # Step 1: refresh access token if expiring - if _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS): + # Step 1: refresh access token if expiring. If the access token + # is already a valid invoke JWT, trust its own exp claim even when + # older auth.json metadata has a stale/missing expires_at. + current_invoke_jwt_usable = ( + not _nous_legacy_session_keys_forced() + and _nous_invoke_jwt_is_usable( + access_token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ) + if ( + _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS) + and not current_invoke_jwt_usable + ): with _nous_shared_store_lock(timeout_seconds=max(timeout_seconds + 5.0, AUTH_LOCK_TIMEOUT_SECONDS)): if _merge_shared_nous_oauth_state(state): access_token = state.get("access_token") refresh_token = state.get("refresh_token") _persist_state("post_shared_merge_access_expiring") - if _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS): + if ( + _is_expiring(state.get("expires_at"), ACCESS_TOKEN_REFRESH_SKEW_SECONDS) + and not _nous_invoke_jwt_is_usable( + access_token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ): if not isinstance(refresh_token, str) or not refresh_token: raise AuthError("Session expired and no refresh token is available.", provider="nous", relogin_required=True) @@ -4320,14 +4549,56 @@ def resolve_nous_runtime_credentials( # Persist immediately so downstream mint failures cannot drop rotated refresh tokens. _persist_state("post_refresh_access_expiring") - # Step 2: mint agent key if missing/expiring + # Step 2: resolve the compatibility ``agent_key`` field. Preferred + # path stores the NAS invoke JWT there; legacy path mints/reuses + # the opaque session key. used_cached_key = False mint_payload: Optional[Dict[str, Any]] = None + selected_auth_path = "legacy_session_key" + legacy_session_keys = _nous_legacy_session_keys_forced() - if not force_mint and _agent_key_is_usable(state, min_key_ttl_seconds): + if ( + not legacy_session_keys + and _nous_invoke_jwt_is_usable( + access_token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ): + _set_nous_agent_key_from_invoke_jwt(state) + selected_auth_path = "invoke_jwt" + logger.info("Nous inference auth: using NAS invoke JWT") + _oauth_trace( + "nous_invoke_jwt_selected", + sequence_id=sequence_id, + access_token_fp=_token_fingerprint(access_token), + ) + elif not force_mint and _agent_key_is_usable(state, min_key_ttl_seconds): used_cached_key = True + selected_auth_path = "legacy_session_key_cache" + logger.info("Nous inference auth: using cached legacy session key") _oauth_trace("agent_key_reuse", sequence_id=sequence_id) else: + fallback_reason = ( + "forced_legacy_session_keys" + if legacy_session_keys + else _nous_invoke_jwt_unavailable_reason( + access_token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ) + selected_auth_path = "legacy_session_key_mint" + logger.info( + "Nous inference auth: using legacy session key path (%s)", + fallback_reason, + ) + _oauth_trace( + "nous_legacy_session_key_selected", + sequence_id=sequence_id, + reason=fallback_reason, + access_token_fp=_token_fingerprint(access_token), + ) try: _oauth_trace( "mint_start", @@ -4403,10 +4674,28 @@ def resolve_nous_runtime_credentials( # Persist retry refresh immediately for crash safety and cross-process visibility. _persist_state("post_refresh_mint_retry") - mint_payload = _mint_agent_key( - client=client, portal_base_url=portal_base_url, - access_token=access_token, min_ttl_seconds=min_key_ttl_seconds, - ) + if ( + not legacy_session_keys + and _nous_invoke_jwt_is_usable( + access_token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ): + _set_nous_agent_key_from_invoke_jwt(state) + mint_payload = None + selected_auth_path = "invoke_jwt" + logger.info("Nous inference auth: using NAS invoke JWT") + _oauth_trace( + "nous_invoke_jwt_selected", + sequence_id=sequence_id, + access_token_fp=_token_fingerprint(access_token), + ) + else: + mint_payload = _mint_agent_key( + client=client, portal_base_url=portal_base_url, + access_token=access_token, min_ttl_seconds=min_key_ttl_seconds, + ) else: raise @@ -4438,6 +4727,8 @@ def resolve_nous_runtime_credentials( _persist_state("resolve_nous_runtime_credentials_final") + _sync_nous_pool_from_auth_store() + api_key = state.get("agent_key") if not isinstance(api_key, str) or not api_key: raise AuthError("Failed to resolve a Nous inference API key", @@ -4458,7 +4749,12 @@ def resolve_nous_runtime_credentials( "key_id": state.get("agent_key_id"), "expires_at": expires_at, "expires_in": expires_in, - "source": "cache" if used_cached_key else "portal", + "source": ( + "invoke_jwt" + if selected_auth_path == "invoke_jwt" + else ("cache" if used_cached_key else "portal") + ), + "auth_path": selected_auth_path, } @@ -6137,7 +6433,10 @@ def _nous_device_code_login( or pconfig.inference_base_url ).rstrip("/") client_id = client_id or pconfig.client_id + explicit_scope = scope is not None scope = scope or pconfig.scope + if _nous_legacy_session_keys_forced(): + scope = NOUS_LEGACY_AGENT_KEY_SCOPE timeout = httpx.Timeout(timeout_seconds) verify: bool | str = False if insecure else (ca_bundle if ca_bundle else True) @@ -6152,12 +6451,30 @@ def _nous_device_code_login( print(f"TLS verification: custom CA bundle ({ca_bundle})") with httpx.Client(timeout=timeout, headers={"Accept": "application/json"}, verify=verify) as client: - device_data = _request_device_code( - client=client, - portal_base_url=portal_base_url, - client_id=client_id, - scope=scope, - ) + try: + device_data = _request_device_code( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + scope=scope, + ) + except Exception as exc: + if ( + not explicit_scope + and _nous_scope_has_invoke(scope) + and _is_nous_invoke_scope_refusal(exc) + ): + logger.info("Nous inference auth: NAS refused invoke scope, retrying legacy scope") + _oauth_trace("nous_device_code_invoke_scope_refused") + scope = NOUS_LEGACY_AGENT_KEY_SCOPE + device_data = _request_device_code( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + scope=scope, + ) + else: + raise verification_url = str(device_data["verification_uri_complete"]) user_code = str(device_data["user_code"]) @@ -6287,7 +6604,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: portal_base_url=getattr(args, "portal_url", None), inference_base_url=getattr(args, "inference_url", None), client_id=getattr(args, "client_id", None) or pconfig.client_id, - scope=getattr(args, "scope", None) or pconfig.scope, + scope=getattr(args, "scope", None), open_browser=not getattr(args, "no_browser", False), timeout_seconds=timeout_seconds, insecure=insecure, @@ -6314,6 +6631,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # these credentials. Best-effort: any I/O failure is logged and # swallowed inside the helper. _write_shared_nous_state(auth_state) + _sync_nous_pool_from_auth_store() print() print("Login successful!") diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index 842489659a4..b69f9d52644 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -1,12 +1,13 @@ """Nous Portal upstream adapter. Reads the user's Nous OAuth state from ``~/.hermes/auth.json``, refreshes -the access token and mints a fresh agent key when needed, and exposes the -upstream base URL plus minted bearer for the proxy server to forward to. +the access token and resolves the ``agent_key`` compatibility credential +when needed, then exposes the upstream base URL plus bearer for the proxy +server to forward to. -The minted ``agent_key`` (not the OAuth ``access_token``) is what -``inference-api.nousresearch.com`` accepts as a bearer. The refresh helper -already handles both — see :func:`hermes_cli.auth.refresh_nous_oauth_from_state`. +The ``agent_key`` field may hold either a NAS invoke JWT or the legacy +opaque session key. The refresh helper handles both — see +:func:`hermes_cli.auth.refresh_nous_oauth_from_state`. """ from __future__ import annotations diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index c186f1d6e7c..de32131d861 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -875,10 +875,9 @@ def _resolve_explicit_runtime( explicit_base_url or str(state.get("inference_base_url") or auth_mod.DEFAULT_NOUS_INFERENCE_URL).strip().rstrip("/") ) - # Only use agent_key for inference — access_token is an OAuth token for the - # portal API (minting keys, refreshing tokens), not for the inference API. - # Falling back to access_token sends an OAuth bearer token to the inference - # endpoint, which returns 404 because it is not a valid inference credential. + # Only use the agent_key compatibility field for inference. It may be + # either a NAS invoke JWT or a legacy opaque session key; raw OAuth + # access_token fallback is handled by resolve_nous_runtime_credentials(). api_key = explicit_api_key or str(state.get("agent_key") or "").strip() expires_at = state.get("agent_key_expires_at") or state.get("expires_at") if not api_key: @@ -1069,17 +1068,19 @@ def resolve_runtime_provider( getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "") ) - # For Nous, the pool entry's runtime_api_key is the agent_key — a - # short-lived inference credential (~30 min TTL). The pool doesn't + # For Nous, the pool entry's runtime_api_key is the agent_key + # compatibility field: either an invoke JWT or legacy opaque key. + # The pool doesn't # refresh it during selection (that would trigger network calls in # non-runtime contexts like `hermes auth list`). If the key is # expired, clear pool_api_key so we fall through to - # resolve_nous_runtime_credentials() which handles refresh + mint. + # resolve_nous_runtime_credentials() which handles refresh + fallback. if provider == "nous" and entry is not None and pool_api_key: min_ttl = max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))) nous_state = { "agent_key": getattr(entry, "agent_key", None), "agent_key_expires_at": getattr(entry, "agent_key_expires_at", None), + "scope": getattr(entry, "scope", None), } if not _agent_key_is_usable(nous_state, min_ttl): logger.debug("Nous pool entry agent_key expired/missing, falling through to runtime resolution") diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 96f5802f839..61af7585a21 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -673,6 +673,8 @@ class TestGetTextAuxiliaryClient: def test_custom_endpoint_uses_codex_wrapper_when_runtime_requests_responses_api(self): with patch("agent.auxiliary_client._resolve_custom_runtime", return_value=("https://api.openai.com/v1", "sk-test", "codex_responses")), \ + patch("agent.auxiliary_client._read_nous_auth", return_value=None), \ + patch("agent.auxiliary_client._resolve_nous_runtime_api", return_value=None), \ patch("agent.auxiliary_client._read_main_model", return_value="gpt-5.3-codex"), \ patch("agent.auxiliary_client.OpenAI") as mock_openai: client, model = get_text_auxiliary_client() diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index e2d2726f21b..f7eaf9fa273 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -2,8 +2,10 @@ from __future__ import annotations +import base64 import json import time +from datetime import datetime, timezone import pytest @@ -14,6 +16,14 @@ def _write_auth_store(tmp_path, payload: dict) -> None: (hermes_home / "auth.json").write_text(json.dumps(payload, indent=2)) +def _jwt_with_claims(claims: dict) -> str: + def _part(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + return f"{_part({'alg': 'none', 'typ': 'JWT'})}.{_part(claims)}.sig" + + def test_fill_first_selection_skips_recently_exhausted_entry(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) _write_auth_store( @@ -510,6 +520,52 @@ def test_load_pool_migrates_nous_provider_state(tmp_path, monkeypatch): assert entry.agent_key == "agent-key" +def test_load_pool_mirrors_nous_invoke_jwt_agent_key_runtime_api_key(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + expires_at = datetime.fromtimestamp(time.time() + 3600, tz=timezone.utc).isoformat() + token = _jwt_with_claims({ + "sub": "test-user", + "scope": ["inference:invoke", "inference:mint_agent_key"], + "exp": int(time.time() + 3600), + }) + _write_auth_store( + tmp_path, + { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "inference_base_url": "https://inference.example.com/v1", + "client_id": "hermes-cli", + "token_type": "Bearer", + "scope": "inference:invoke inference:mint_agent_key", + "access_token": token, + "refresh_token": "refresh-token", + "expires_at": expires_at, + "agent_key": token, + "agent_key_expires_at": expires_at, + } + }, + }, + ) + + from agent.credential_pool import load_pool + + pool = load_pool("nous") + entry = pool.select() + + assert entry is not None + assert entry.source == "device_code" + assert entry.agent_key == token + assert entry.runtime_api_key == token + + auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text()) + pool_entry = auth_payload["credential_pool"]["nous"][0] + assert pool_entry["agent_key"] == token + assert pool_entry["agent_key_expires_at"] == expires_at + + def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared")) diff --git a/tests/conftest.py b/tests/conftest.py index aa2b1b1fbcb..176089d5691 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -187,6 +187,7 @@ _HERMES_BEHAVIORAL_VARS = frozenset({ "HERMES_BACKGROUND_NOTIFICATIONS", "HERMES_EXEC_ASK", "HERMES_HOME_MODE", + "HERMES_AGENT_USE_LEGACY_SESSION_KEYS", # Kanban path/board pins must never leak from a developer shell or # dispatched worker into tests; otherwise tests can write fake tasks to # the real ~/.hermes/kanban.db instead of the per-test HERMES_HOME. diff --git a/tests/hermes_cli/test_auth_commands.py b/tests/hermes_cli/test_auth_commands.py index 74e2a64d312..22182ba43a8 100644 --- a/tests/hermes_cli/test_auth_commands.py +++ b/tests/hermes_cli/test_auth_commands.py @@ -107,7 +107,7 @@ def test_auth_add_nous_oauth_persists_pool_entry(tmp_path, monkeypatch): "portal_base_url": "https://portal.example.com", "inference_base_url": "https://inference.example.com/v1", "client_id": "hermes-cli", - "scope": "inference:mint_agent_key", + "scope": "inference:invoke inference:mint_agent_key", "token_type": "Bearer", "access_token": token, "refresh_token": "refresh-token", @@ -228,7 +228,7 @@ def test_auth_add_nous_oauth_honors_custom_label(tmp_path, monkeypatch): "portal_base_url": "https://portal.example.com", "inference_base_url": "https://inference.example.com/v1", "client_id": "hermes-cli", - "scope": "inference:mint_agent_key", + "scope": "inference:invoke inference:mint_agent_key", "token_type": "Bearer", "access_token": token, "refresh_token": "refresh-token", diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index 37662c77ece..1d07737a857 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -1,6 +1,9 @@ """Regression tests for Nous OAuth refresh + agent-key mint interactions.""" +import base64 import json +import logging +import time from datetime import datetime, timezone from pathlib import Path @@ -125,6 +128,11 @@ def _setup_nous_auth( *, access_token: str = "access-old", refresh_token: str = "refresh-old", + scope: str = "inference:mint_agent_key", + expires_at: str = "2026-02-01T00:00:00+00:00", + expires_in: int = 0, + agent_key: str | None = None, + agent_key_expires_at: str | None = None, ) -> None: hermes_home.mkdir(parents=True, exist_ok=True) auth_store = { @@ -136,15 +144,15 @@ def _setup_nous_auth( "inference_base_url": "https://inference.example.com/v1", "client_id": "hermes-cli", "token_type": "Bearer", - "scope": "inference:mint_agent_key", + "scope": scope, "access_token": access_token, "refresh_token": refresh_token, "obtained_at": "2026-02-01T00:00:00+00:00", - "expires_in": 0, - "expires_at": "2026-02-01T00:00:00+00:00", - "agent_key": None, + "expires_in": expires_in, + "expires_at": expires_at, + "agent_key": agent_key, "agent_key_id": None, - "agent_key_expires_at": None, + "agent_key_expires_at": agent_key_expires_at, "agent_key_expires_in": None, "agent_key_reused": None, "agent_key_obtained_at": None, @@ -164,6 +172,351 @@ def _mint_payload(api_key: str = "agent-key") -> dict: } +def _jwt_with_claims(claims: dict) -> str: + def _part(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + return f"{_part({'alg': 'none', 'typ': 'JWT'})}.{_part(claims)}.sig" + + +def _future_iso(seconds: int = 3600) -> str: + return datetime.fromtimestamp(time.time() + seconds, tz=timezone.utc).isoformat() + + +def _invoke_jwt(*, seconds: int = 3600, scope: object = "inference:invoke inference:mint_agent_key") -> str: + return _jwt_with_claims({ + "sub": "test-user", + "scope": scope, + "exp": int(time.time() + seconds), + }) + + +def test_resolve_nous_runtime_credentials_prefers_invoke_jwt_and_mirrors( + tmp_path, + monkeypatch, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _invoke_jwt(seconds=3600) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at=_future_iso(3600), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + def _unexpected_mint(*args, **kwargs): + raise AssertionError("legacy agent-key mint should not run for invoke JWT") + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _unexpected_mint) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert creds["api_key"] == token + assert creds["source"] == "invoke_jwt" + assert creds["auth_path"] == "invoke_jwt" + + payload = json.loads((hermes_home / "auth.json").read_text()) + singleton = payload["providers"]["nous"] + assert singleton["agent_key"] == token + assert datetime.fromisoformat(singleton["agent_key_expires_at"]).timestamp() > time.time() + 300 + + pool_entries = payload["credential_pool"]["nous"] + assert len(pool_entries) == 1 + assert pool_entries[0]["agent_key"] == token + assert pool_entries[0]["source"] == auth_mod.NOUS_DEVICE_CODE_SOURCE + + +def test_resolve_nous_runtime_credentials_trusts_invoke_jwt_exp_over_stale_metadata( + tmp_path, + monkeypatch, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _invoke_jwt(seconds=3600) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at="2000-01-01T00:00:00+00:00", + expires_in=0, + agent_key=token, + agent_key_expires_at="2000-01-01T00:00:00+00:00", + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + def _unexpected_refresh(*args, **kwargs): + raise AssertionError("valid invoke JWT should not be refreshed because metadata is stale") + + def _unexpected_mint(*args, **kwargs): + raise AssertionError("valid invoke JWT should not fall back to legacy mint") + + monkeypatch.setattr(auth_mod, "_refresh_access_token", _unexpected_refresh) + monkeypatch.setattr(auth_mod, "_mint_agent_key", _unexpected_mint) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert creds["api_key"] == token + assert creds["source"] == "invoke_jwt" + payload = json.loads((hermes_home / "auth.json").read_text()) + singleton = payload["providers"]["nous"] + assert singleton["agent_key"] == token + assert datetime.fromisoformat(singleton["expires_at"]).timestamp() > time.time() + 300 + assert datetime.fromisoformat(singleton["agent_key_expires_at"]).timestamp() > time.time() + 300 + + +def test_resolve_nous_runtime_credentials_does_not_apply_legacy_ttl_to_invoke_jwt( + tmp_path, + monkeypatch, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _invoke_jwt(seconds=900) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at=_future_iso(900), + expires_in=900, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + def _unexpected_mint(*args, **kwargs): + raise AssertionError("1800s legacy min TTL should not force opaque mint for invoke JWT") + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _unexpected_mint) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=1800) + + assert creds["api_key"] == token + assert creds["source"] == "invoke_jwt" + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload["providers"]["nous"]["agent_key"] == token + assert payload["credential_pool"]["nous"][0]["agent_key"] == token + + +def test_resolve_nous_runtime_credentials_falls_back_when_invoke_scope_missing( + tmp_path, + monkeypatch, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _jwt_with_claims({ + "sub": "test-user", + "scope": "inference:mint_agent_key", + "exp": int(time.time() + 3600), + }) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + expires_at=_future_iso(3600), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + calls = [] + + def _fake_mint_agent_key(*, client, portal_base_url, access_token, min_ttl_seconds): + del client, portal_base_url, min_ttl_seconds + calls.append(access_token) + return _mint_payload(api_key="opaque-agent-key") + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _fake_mint_agent_key) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert calls == [token] + assert creds["api_key"] == "opaque-agent-key" + assert creds["source"] == "portal" + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload["providers"]["nous"]["agent_key"] == "opaque-agent-key" + assert payload["credential_pool"]["nous"][0]["agent_key"] == "opaque-agent-key" + + +def test_nous_device_code_login_retries_legacy_scope_when_invoke_refused(monkeypatch): + import hermes_cli.auth as auth_mod + + scopes = [] + + def _fake_request_device_code(*, client, portal_base_url, client_id, scope): + del client, portal_base_url, client_id + scopes.append(scope) + if len(scopes) == 1: + request = httpx.Request("POST", "https://portal.example.com/api/oauth/device/code") + response = httpx.Response( + 400, + json={ + "error": "invalid_scope", + "error_description": "unsupported inference:invoke", + }, + request=request, + ) + raise httpx.HTTPStatusError("invalid_scope", request=request, response=response) + return { + "device_code": "device", + "user_code": "user", + "verification_uri": "https://portal.example.com/device", + "verification_uri_complete": "https://portal.example.com/device?code=user", + "expires_in": 600, + "interval": 1, + } + + def _fake_poll_for_token(**kwargs): + del kwargs + return { + "access_token": "access-legacy", + "refresh_token": "refresh-legacy", + "expires_in": 900, + "scope": auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + } + + def _fake_refresh(state, **kwargs): + del kwargs + refreshed = dict(state) + refreshed["agent_key"] = "opaque-agent-key" + refreshed["agent_key_expires_at"] = _future_iso(1800) + return refreshed + + monkeypatch.setattr(auth_mod, "_request_device_code", _fake_request_device_code) + monkeypatch.setattr(auth_mod, "_poll_for_token", _fake_poll_for_token) + monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _fake_refresh) + + result = auth_mod._nous_device_code_login( + portal_base_url="https://portal.example.com", + inference_base_url="https://inference.example.com/v1", + open_browser=False, + timeout_seconds=1, + ) + + assert scopes == [auth_mod.DEFAULT_NOUS_SCOPE, auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE] + assert result["scope"] == auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE + assert result["agent_key"] == "opaque-agent-key" + + +def test_forced_legacy_env_skips_invoke_scope_and_jwt_storage(tmp_path, monkeypatch): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _invoke_jwt(seconds=3600) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at=_future_iso(3600), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.setenv(auth_mod.NOUS_LEGACY_SESSION_KEYS_ENV, "true") + + mint_calls = [] + + def _fake_mint_agent_key(*, client, portal_base_url, access_token, min_ttl_seconds): + del client, portal_base_url, min_ttl_seconds + mint_calls.append(access_token) + return _mint_payload(api_key="forced-legacy-key") + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _fake_mint_agent_key) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert mint_calls == [token] + assert creds["api_key"] == "forced-legacy-key" + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload["providers"]["nous"]["agent_key"] == "forced-legacy-key" + + requested_scopes = [] + + def _fake_request_device_code(*, client, portal_base_url, client_id, scope): + del client, portal_base_url, client_id + requested_scopes.append(scope) + return { + "device_code": "device", + "user_code": "user", + "verification_uri": "https://portal.example.com/device", + "verification_uri_complete": "https://portal.example.com/device?code=user", + "expires_in": 600, + "interval": 1, + } + + def _fake_poll_for_token(**kwargs): + del kwargs + return { + "access_token": "access-legacy", + "refresh_token": "refresh-legacy", + "expires_in": 900, + "scope": auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + } + + def _fake_refresh(state, **kwargs): + del kwargs + refreshed = dict(state) + refreshed["agent_key"] = "forced-legacy-login-key" + refreshed["agent_key_expires_at"] = _future_iso(1800) + return refreshed + + monkeypatch.setattr(auth_mod, "_request_device_code", _fake_request_device_code) + monkeypatch.setattr(auth_mod, "_poll_for_token", _fake_poll_for_token) + monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _fake_refresh) + + auth_mod._nous_device_code_login( + portal_base_url="https://portal.example.com", + inference_base_url="https://inference.example.com/v1", + open_browser=False, + timeout_seconds=1, + ) + + assert requested_scopes == [auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE] + + +def test_nous_inference_auth_logs_do_not_include_secret_values( + tmp_path, + monkeypatch, + caplog, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _jwt_with_claims({ + "sub": "secret-user", + "scope": "inference:mint_agent_key", + "exp": int(time.time() + 3600), + }) + refresh_token = "refresh-secret-token" + opaque_key = "opaque-secret-agent-key" + _setup_nous_auth( + hermes_home, + access_token=token, + refresh_token=refresh_token, + scope=auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + expires_at=_future_iso(3600), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + def _fake_mint_agent_key(*, client, portal_base_url, access_token, min_ttl_seconds): + del client, portal_base_url, access_token, min_ttl_seconds + return _mint_payload(api_key=opaque_key) + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _fake_mint_agent_key) + + caplog.set_level(logging.INFO, logger="hermes_cli.auth") + auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + logged = caplog.text + assert "legacy session key path" in logged + assert token not in logged + assert refresh_token not in logged + assert opaque_key not in logged + + def test_get_nous_auth_status_checks_credential_pool(tmp_path, monkeypatch): """get_nous_auth_status() should find Nous credentials in the pool even when the auth store has no Nous provider entry — this is the From 0bac7dd05bd56fd615ef4b5c499a60a42a8b32b6 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Sun, 17 May 2026 20:34:39 +1000 Subject: [PATCH 134/142] refactor(auth): collapse Nous inference fallback controls --- agent/auxiliary_client.py | 16 +- agent/credential_pool.py | 51 +- hermes_cli/auth.py | 544 ++++++++++++++------ hermes_cli/proxy/adapters/base.py | 15 + hermes_cli/proxy/adapters/nous_portal.py | 45 +- hermes_cli/proxy/server.py | 115 +++-- hermes_cli/web_server.py | 39 +- run_agent.py | 12 +- tests/agent/test_credential_pool.py | 84 ++- tests/hermes_cli/test_auth_nous_provider.py | 137 ++++- tests/hermes_cli/test_proxy.py | 112 +++- tests/hermes_cli/test_web_oauth_dispatch.py | 139 ++++- tests/run_agent/test_run_agent.py | 2 +- 13 files changed, 1071 insertions(+), 240 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index b2733fd8a1b..e67b37b00da 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -1252,12 +1252,20 @@ def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[ or the credential pool. """ try: - from hermes_cli.auth import resolve_nous_runtime_credentials + from hermes_cli.auth import ( + NOUS_INFERENCE_AUTH_AUTO, + NOUS_INFERENCE_AUTH_LEGACY, + resolve_nous_runtime_credentials, + ) creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - force_mint=force_refresh, + auth_mode=( + NOUS_INFERENCE_AUTH_LEGACY + if force_refresh + else NOUS_INFERENCE_AUTH_AUTO + ), ) except Exception as exc: logger.debug("Auxiliary Nous runtime credential resolution failed: %s", exc) @@ -2501,12 +2509,12 @@ def _refresh_provider_credentials(provider: str) -> bool: _evict_cached_clients(normalized) return True if normalized == "nous": - from hermes_cli.auth import resolve_nous_runtime_credentials + from hermes_cli.auth import NOUS_INFERENCE_AUTH_LEGACY, resolve_nous_runtime_credentials creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - force_mint=True, + auth_mode=NOUS_INFERENCE_AUTH_LEGACY, ) if not str(creds.get("api_key", "") or "").strip(): return False diff --git a/agent/credential_pool.py b/agent/credential_pool.py index b1c41977d51..7c91a08d2aa 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -831,7 +831,11 @@ class CredentialPool: nous_state, min_key_ttl_seconds=DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, force_refresh=force, - force_mint=force, + auth_mode=( + auth_mod.NOUS_INFERENCE_AUTH_LEGACY + if force + else auth_mod.NOUS_INFERENCE_AUTH_AUTO + ), ) # Apply returned fields: dataclass fields via replace, extras via dict update field_updates = {} @@ -952,25 +956,27 @@ class CredentialPool: exc, reason="credential_pool_refresh_failure", ) + auth_mod._quarantine_nous_pool_entries( + auth_store, + exc, + reason="credential_pool_refresh_failure", + ) _save_provider_state(auth_store, "nous", state) _save_auth_store(auth_store) except Exception as clear_exc: logger.debug("Failed to clear terminal Nous OAuth state: %s", clear_exc) - cleared = replace( - entry, - access_token=None, - refresh_token=None, - agent_key=None, - agent_key_expires_at=None, - ) - self._replace_entry(entry, cleared) + singleton_sources = { + auth_mod.NOUS_DEVICE_CODE_SOURCE, + f"manual:{auth_mod.NOUS_DEVICE_CODE_SOURCE}", + } + self._entries = [ + item for item in self._entries + if item.source not in singleton_sources + ] + if self._current_id == entry.id: + self._current_id = None self._persist() - self._mark_exhausted( - cleared, - 401, - {"reason": getattr(exc, "code", None), "message": str(exc)}, - ) return None self._mark_exhausted(entry, None) return None @@ -1408,7 +1414,22 @@ def _seed_from_singletons(provider: str, entries: List[PooledCredential]) -> Tup elif provider == "nous": state = _load_provider_state(auth_store, "nous") - if state and not _is_suppressed(provider, "device_code"): + has_runtime_material = bool( + isinstance(state, dict) + and ( + str(state.get("access_token") or "").strip() + or str(state.get("agent_key") or "").strip() + ) + ) + if state and not has_runtime_material: + retained = [ + entry for entry in entries + if entry.source not in {"device_code", "manual:device_code"} + ] + if len(retained) != len(entries): + entries[:] = retained + changed = True + if state and has_runtime_material and not _is_suppressed(provider, "device_code"): active_sources.add("device_code") # Prefer a user-supplied label embedded in the singleton state # (set by persist_nous_credentials(label=...) when the user ran diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 2a670589d48..783f2c0c655 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -11,6 +11,12 @@ Architecture: - resolve_provider() picks the active provider via priority chain - resolve_*_runtime_credentials() handles token refresh and key minting - logout_command() is the CLI entry point for clearing auth + +Nous authentication paths: +- Invoke JWT (preferred): use a scoped access_token directly for inference. +- Legacy session key (fallback): mint an opaque 24h key when JWT auth is + unavailable, or when HERMES_AGENT_USE_LEGACY_SESSION_KEYS is set for + debugging or rollback. """ from __future__ import annotations @@ -71,6 +77,15 @@ NOUS_LEGACY_AGENT_KEY_SCOPE = "inference:mint_agent_key" NOUS_INFERENCE_INVOKE_SCOPE = "inference:invoke" DEFAULT_NOUS_SCOPE = f"{NOUS_INFERENCE_INVOKE_SCOPE} {NOUS_LEGACY_AGENT_KEY_SCOPE}" NOUS_LEGACY_SESSION_KEYS_ENV = "HERMES_AGENT_USE_LEGACY_SESSION_KEYS" +NOUS_DEVICE_CODE_SOURCE = "device_code" +NOUS_INFERENCE_AUTH_AUTO = "auto" +NOUS_INFERENCE_AUTH_FRESH = "fresh" +NOUS_INFERENCE_AUTH_LEGACY = "legacy" +NOUS_INFERENCE_AUTH_MODES = frozenset({ + NOUS_INFERENCE_AUTH_AUTO, + NOUS_INFERENCE_AUTH_FRESH, + NOUS_INFERENCE_AUTH_LEGACY, +}) DEFAULT_AGENT_KEY_MIN_TTL_SECONDS = 30 * 60 # 30 minutes ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 # refresh 2 min before expiry NOUS_INVOKE_JWT_MIN_TTL_SECONDS = ACCESS_TOKEN_REFRESH_SKEW_SECONDS @@ -1554,6 +1569,8 @@ def _decode_jwt_claims(token: Any) -> Dict[str, Any]: def _scope_values(raw_scope: Any) -> set[str]: + # OAuth token responses normally return a space-separated string. Keep + # collection support for JWT ``scp`` claims and older stored test fixtures. scopes: set[str] = set() if isinstance(raw_scope, str): for part in raw_scope.replace(",", " ").split(): @@ -1575,37 +1592,24 @@ def _nous_scope_has_invoke(raw_scope: Any) -> bool: return NOUS_INFERENCE_INVOKE_SCOPE in _scope_values(raw_scope) -def _nous_invoke_jwt_is_usable( +def _normalize_nous_auth_mode(auth_mode: Optional[str]) -> str: + mode = str(auth_mode or NOUS_INFERENCE_AUTH_AUTO).strip().lower() + if mode not in NOUS_INFERENCE_AUTH_MODES: + allowed = ", ".join(sorted(NOUS_INFERENCE_AUTH_MODES)) + raise ValueError( + f"Invalid Nous inference auth mode {auth_mode!r}; expected one of: {allowed}" + ) + return mode + + +def _nous_invoke_jwt_status( token: Any, *, scope: Any = None, expires_at: Any = None, min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, -) -> bool: - claims = _decode_jwt_claims(token) - if not claims: - return False - scopes = ( - _scope_values(scope) - | _scope_values(claims.get("scope")) - | _scope_values(claims.get("scp")) - ) - if NOUS_INFERENCE_INVOKE_SCOPE not in scopes: - return False - exp = claims.get("exp") - skew = max(0, int(min_ttl_seconds)) - if isinstance(exp, (int, float)): - return float(exp) > (time.time() + skew) - return not _is_expiring(expires_at, skew) - - -def _nous_invoke_jwt_unavailable_reason( - token: Any, - *, - scope: Any = None, - expires_at: Any = None, - min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, -) -> str: +) -> Optional[str]: + """Return None when the token can be used for inference, else a reason.""" claims = _decode_jwt_claims(token) if not claims: return "access_token_not_jwt" @@ -1618,11 +1622,149 @@ def _nous_invoke_jwt_unavailable_reason( return "missing_inference_invoke_scope" exp = claims.get("exp") skew = max(0, int(min_ttl_seconds)) - if isinstance(exp, (int, float)) and float(exp) <= (time.time() + skew): - return "invoke_jwt_expiring" - if not isinstance(exp, (int, float)) and _is_expiring(expires_at, skew): + if isinstance(exp, (int, float)): + if float(exp) <= (time.time() + skew): + return "invoke_jwt_expiring" + return None + if _is_expiring(expires_at, skew): return "invoke_jwt_expiry_unknown_or_expiring" - return "invoke_jwt_unavailable" + return None + + +def _nous_invoke_jwt_is_usable( + token: Any, + *, + scope: Any = None, + expires_at: Any = None, + min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, +) -> bool: + return ( + _nous_invoke_jwt_status( + token, + scope=scope, + expires_at=expires_at, + min_ttl_seconds=min_ttl_seconds, + ) + is None + ) + + +def _nous_invoke_jwt_unavailable_reason( + token: Any, + *, + scope: Any = None, + expires_at: Any = None, + min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, +) -> str: + return ( + _nous_invoke_jwt_status( + token, + scope=scope, + expires_at=expires_at, + min_ttl_seconds=min_ttl_seconds, + ) + or "invoke_jwt_unavailable" + ) + + +def _nous_can_select_invoke_jwt(auth_mode: str = NOUS_INFERENCE_AUTH_AUTO) -> bool: + return ( + not _nous_legacy_session_keys_forced() + and _normalize_nous_auth_mode(auth_mode) != NOUS_INFERENCE_AUTH_LEGACY + ) + + +def _nous_legacy_session_key_reason( + token: Any, + *, + scope: Any = None, + expires_at: Any = None, + auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, +) -> str: + if _normalize_nous_auth_mode(auth_mode) == NOUS_INFERENCE_AUTH_LEGACY: + return "forced_legacy_session_key" + if _nous_legacy_session_keys_forced(): + return "forced_legacy_session_keys" + return _nous_invoke_jwt_unavailable_reason( + token, + scope=scope, + expires_at=expires_at, + ) + + +def _nous_cached_agent_key_is_usable( + state: Dict[str, Any], + min_ttl_seconds: int, +) -> bool: + return _agent_key_is_usable(state, min_ttl_seconds) + + +def _choose_nous_inference_auth_path( + state: Dict[str, Any], + *, + access_token: Any = None, + min_key_ttl_seconds: int = DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, + auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, +) -> Tuple[str, Optional[str]]: + auth_mode = _normalize_nous_auth_mode(auth_mode) + token = state.get("access_token") if access_token is None else access_token + if ( + _nous_can_select_invoke_jwt(auth_mode) + and _nous_invoke_jwt_is_usable( + token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) + ): + return "invoke_jwt", None + if ( + auth_mode == NOUS_INFERENCE_AUTH_AUTO + and _nous_cached_agent_key_is_usable( + state, + max(60, int(min_key_ttl_seconds)), + ) + ): + return "legacy_session_key_cache", None + return ( + "legacy_session_key_mint", + _nous_legacy_session_key_reason( + token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + auth_mode=auth_mode, + ), + ) + + +def _log_nous_invoke_jwt_selected( + *, + access_token: Any, + sequence_id: Optional[str] = None, +) -> None: + logger.info("Nous inference auth: using NAS invoke JWT") + _oauth_trace( + "nous_invoke_jwt_selected", + sequence_id=sequence_id, + access_token_fp=_token_fingerprint(access_token), + ) + + +def _log_nous_legacy_session_key_selected( + reason: str, + *, + access_token: Any, + sequence_id: Optional[str] = None, +) -> None: + logger.info( + "Nous inference auth: using legacy session key path (%s)", + reason, + ) + _oauth_trace( + "nous_legacy_session_key_selected", + sequence_id=sequence_id, + reason=reason, + access_token_fp=_token_fingerprint(access_token), + ) def _nous_jwt_expires_at(token: Any, fallback_expires_at: Any = None) -> Optional[str]: @@ -1645,7 +1787,17 @@ def _set_nous_agent_key_from_invoke_jwt( if not isinstance(access_token, str) or not access_token.strip(): return now = datetime.now(timezone.utc) - effective_obtained_at = obtained_at or now.isoformat() + existing_obtained_at = state.get("agent_key_obtained_at") + if obtained_at: + effective_obtained_at = obtained_at + elif ( + state.get("agent_key") == access_token + and isinstance(existing_obtained_at, str) + and existing_obtained_at.strip() + ): + effective_obtained_at = existing_obtained_at + else: + effective_obtained_at = now.isoformat() expires_at = _nous_jwt_expires_at(access_token, state.get("expires_at")) expires_epoch = _parse_iso_timestamp(expires_at) expires_in = ( @@ -1664,6 +1816,38 @@ def _set_nous_agent_key_from_invoke_jwt( state["agent_key_obtained_at"] = effective_obtained_at +def _select_nous_invoke_jwt( + state: Dict[str, Any], + *, + access_token: Any = None, + sequence_id: Optional[str] = None, +) -> None: + if isinstance(access_token, str) and access_token.strip(): + state["access_token"] = access_token + _set_nous_agent_key_from_invoke_jwt(state) + _log_nous_invoke_jwt_selected( + access_token=state.get("access_token"), + sequence_id=sequence_id, + ) + + +_NOUS_EFFECTIVE_STATE_IGNORED_KEYS = frozenset({ + # These are derived from expires_at/JWT exp and naturally tick down between + # reads. Persisting only these changes makes auth.json noisy and defeats + # the mtime-keyed auth-status cache. + "expires_in", + "agent_key_expires_in", +}) + + +def _nous_effective_provider_state(state: Dict[str, Any]) -> Dict[str, Any]: + return { + key: value + for key, value in state.items() + if key not in _NOUS_EFFECTIVE_STATE_IGNORED_KEYS + } + + def _codex_access_token_is_expiring(access_token: Any, skew_seconds: int) -> bool: claims = _decode_jwt_claims(access_token) exp = claims.get("exp") @@ -3476,6 +3660,57 @@ def _is_nous_invoke_scope_refusal(exc: Exception) -> bool: ) +def _nous_device_scope( + requested_scope: Optional[str], + *, + default_scope: str = DEFAULT_NOUS_SCOPE, +) -> Tuple[str, bool]: + explicit_scope = requested_scope is not None + scope = requested_scope or default_scope + if _nous_legacy_session_keys_forced(): + scope = NOUS_LEGACY_AGENT_KEY_SCOPE + return scope, explicit_scope + + +def _request_nous_device_code_with_scope_fallback( + *, + client: httpx.Client, + portal_base_url: str, + client_id: str, + scope: str, + allow_legacy_fallback: bool, +) -> Tuple[Dict[str, Any], str]: + try: + return ( + _request_device_code( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + scope=scope, + ), + scope, + ) + except Exception as exc: + if ( + allow_legacy_fallback + and _nous_scope_has_invoke(scope) + and _is_nous_invoke_scope_refusal(exc) + ): + logger.info("Nous inference auth: NAS refused invoke scope, retrying legacy scope") + _oauth_trace("nous_device_code_invoke_scope_refused") + retry_scope = NOUS_LEGACY_AGENT_KEY_SCOPE + return ( + _request_device_code( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + scope=retry_scope, + ), + retry_scope, + ) + raise + + def _poll_for_token( client: httpx.Client, portal_base_url: str, @@ -3817,6 +4052,39 @@ def _quarantine_nous_oauth_state( invalidate_nous_auth_status_cache() +def _quarantine_nous_pool_entries( + auth_store: Dict[str, Any], + error: AuthError, + *, + reason: str, +) -> bool: + """Remove singleton-seeded Nous pool entries that contain dead OAuth state.""" + pool = auth_store.get("credential_pool") + if not isinstance(pool, dict): + return False + entries = pool.get("nous") + if not isinstance(entries, list): + return False + + retained = [] + removed = False + singleton_sources = {NOUS_DEVICE_CODE_SOURCE, f"manual:{NOUS_DEVICE_CODE_SOURCE}"} + for entry in entries: + if isinstance(entry, dict) and entry.get("source") in singleton_sources: + removed = True + continue + retained.append(entry) + + if removed: + pool["nous"] = retained + _oauth_trace( + "nous_pool_device_code_quarantined", + reason=reason, + error_code=error.code, + ) + return removed + + def _try_import_shared_nous_state( *, timeout_seconds: float = 15.0, @@ -3842,7 +4110,7 @@ def _try_import_shared_nous_state( # Build a full state dict so refresh_nous_oauth_from_state has every # field it needs. force_refresh=True gets us a fresh access_token - # for this profile; force_mint=True gets us a fresh agent_key. + # for this profile; fresh auth mode avoids stale cached legacy keys. state: Dict[str, Any] = { "access_token": shared.get("access_token"), "refresh_token": shared.get("refresh_token"), @@ -3863,7 +4131,7 @@ def _try_import_shared_nous_state( min_key_ttl_seconds=min_key_ttl_seconds, timeout_seconds=timeout_seconds, force_refresh=True, - force_mint=True, + auth_mode=NOUS_INFERENCE_AUTH_FRESH, ) _write_shared_nous_state(refreshed) except AuthError as exc: @@ -4121,6 +4389,11 @@ def resolve_nous_access_token( exc, reason="managed_access_token_refresh_failure", ) + _quarantine_nous_pool_entries( + auth_store, + exc, + reason="managed_access_token_refresh_failure", + ) _save_provider_state(auth_store, "nous", state) _save_auth_store(auth_store) raise @@ -4167,9 +4440,10 @@ def refresh_nous_oauth_pure( insecure: Optional[bool] = None, ca_bundle: Optional[str] = None, force_refresh: bool = False, - force_mint: bool = False, + auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, ) -> Dict[str, Any]: """Refresh Nous OAuth state without mutating auth.json.""" + auth_mode = _normalize_nous_auth_mode(auth_mode) state: Dict[str, Any] = { "access_token": access_token, "refresh_token": refresh_token, @@ -4229,38 +4503,17 @@ def refresh_nous_oauth_pure( now.timestamp() + access_ttl, tz=timezone.utc ).isoformat() - if ( - not legacy_session_keys - and _nous_invoke_jwt_is_usable( - state.get("access_token"), - scope=state.get("scope"), - expires_at=state.get("expires_at"), - ) - ): - _set_nous_agent_key_from_invoke_jwt(state) - logger.info("Nous inference auth: using NAS invoke JWT") - _oauth_trace( - "nous_invoke_jwt_selected", - access_token_fp=_token_fingerprint(state.get("access_token")), - ) - elif force_mint or not _agent_key_is_usable(state, min_agent_key_ttl): - fallback_reason = ( - "forced_legacy_session_keys" - if legacy_session_keys - else _nous_invoke_jwt_unavailable_reason( - state.get("access_token"), - scope=state.get("scope"), - expires_at=state.get("expires_at"), - ) - ) - logger.info( - "Nous inference auth: using legacy session key path (%s)", - fallback_reason, - ) - _oauth_trace( - "nous_legacy_session_key_selected", - reason=fallback_reason, - access_token_fp=_token_fingerprint(state.get("access_token")), + selected_auth_path, fallback_reason = _choose_nous_inference_auth_path( + state, + min_key_ttl_seconds=min_agent_key_ttl, + auth_mode=auth_mode, + ) + if selected_auth_path == "invoke_jwt": + _select_nous_invoke_jwt(state) + elif selected_auth_path == "legacy_session_key_mint": + _log_nous_legacy_session_key_selected( + fallback_reason or "legacy_session_key_required", + access_token=state.get("access_token"), ) mint_payload = _mint_agent_key( client=client, @@ -4288,7 +4541,7 @@ def refresh_nous_oauth_from_state( min_key_ttl_seconds: int = DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, timeout_seconds: float = 15.0, force_refresh: bool = False, - force_mint: bool = False, + auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, ) -> Dict[str, Any]: """Refresh Nous OAuth from a state dict. Thin wrapper around refresh_nous_oauth_pure.""" tls = state.get("tls") or {} @@ -4309,13 +4562,10 @@ def refresh_nous_oauth_from_state( insecure=tls.get("insecure"), ca_bundle=tls.get("ca_bundle"), force_refresh=force_refresh, - force_mint=force_mint, + auth_mode=auth_mode, ) -NOUS_DEVICE_CODE_SOURCE = "device_code" - - def persist_nous_credentials( creds: Dict[str, Any], *, @@ -4390,7 +4640,7 @@ def resolve_nous_runtime_credentials( timeout_seconds: float = 15.0, insecure: Optional[bool] = None, ca_bundle: Optional[str] = None, - force_mint: bool = False, + auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, ) -> Dict[str, Any]: """ Resolve Nous inference credentials for runtime use. @@ -4402,6 +4652,7 @@ def resolve_nous_runtime_credentials( Returns dict with: provider, base_url, api_key, key_id, expires_at, expires_in, source ("invoke_jwt", "cache", or "portal"), and auth_path. """ + auth_mode = _normalize_nous_auth_mode(auth_mode) min_key_ttl_seconds = max(60, int(min_key_ttl_seconds)) sequence_id = uuid.uuid4().hex[:12] @@ -4413,6 +4664,9 @@ def resolve_nous_runtime_credentials( raise AuthError("Hermes is not logged into Nous Portal.", provider="nous", relogin_required=True) + persisted_state = dict(state) + state_persisted = False + portal_base_url = ( _optional_base_url(state.get("portal_base_url")) or os.getenv("HERMES_PORTAL_BASE_URL") @@ -4427,6 +4681,17 @@ def resolve_nous_runtime_credentials( client_id = str(state.get("client_id") or DEFAULT_NOUS_CLIENT_ID) def _persist_state(reason: str) -> None: + nonlocal persisted_state, state_persisted + if ( + _nous_effective_provider_state(state) + == _nous_effective_provider_state(persisted_state) + ): + _oauth_trace( + "nous_state_persist_skipped", + sequence_id=sequence_id, + reason=reason, + ) + return try: _save_provider_state(auth_store, "nous", state) _save_auth_store(auth_store) @@ -4445,6 +4710,8 @@ def resolve_nous_runtime_credentials( refresh_token_fp=_token_fingerprint(state.get("refresh_token")), access_token_fp=_token_fingerprint(state.get("access_token")), ) + persisted_state = dict(state) + state_persisted = True # Mirror post-refresh state to the shared store so sibling # profiles don't hold stale refresh_tokens after rotation. # Best-effort — any failure is logged and swallowed inside @@ -4456,7 +4723,7 @@ def resolve_nous_runtime_credentials( _oauth_trace( "nous_runtime_credentials_start", sequence_id=sequence_id, - force_mint=bool(force_mint), + auth_mode=auth_mode, min_key_ttl_seconds=min_key_ttl_seconds, refresh_token_fp=_token_fingerprint(state.get("refresh_token")), ) @@ -4520,6 +4787,11 @@ def resolve_nous_runtime_credentials( exc, reason="runtime_access_refresh_failure", ) + _quarantine_nous_pool_entries( + auth_store, + exc, + reason="runtime_access_refresh_failure", + ) _persist_state("terminal_runtime_access_refresh_failure") raise now = datetime.now(timezone.utc) @@ -4554,50 +4826,28 @@ def resolve_nous_runtime_credentials( # the opaque session key. used_cached_key = False mint_payload: Optional[Dict[str, Any]] = None - selected_auth_path = "legacy_session_key" - legacy_session_keys = _nous_legacy_session_keys_forced() + selected_auth_path, fallback_reason = _choose_nous_inference_auth_path( + state, + access_token=access_token, + min_key_ttl_seconds=min_key_ttl_seconds, + auth_mode=auth_mode, + ) - if ( - not legacy_session_keys - and _nous_invoke_jwt_is_usable( - access_token, - scope=state.get("scope"), - expires_at=state.get("expires_at"), - ) - ): - _set_nous_agent_key_from_invoke_jwt(state) - selected_auth_path = "invoke_jwt" - logger.info("Nous inference auth: using NAS invoke JWT") - _oauth_trace( - "nous_invoke_jwt_selected", + if selected_auth_path == "invoke_jwt": + _select_nous_invoke_jwt( + state, + access_token=access_token, sequence_id=sequence_id, - access_token_fp=_token_fingerprint(access_token), ) - elif not force_mint and _agent_key_is_usable(state, min_key_ttl_seconds): + elif selected_auth_path == "legacy_session_key_cache": used_cached_key = True - selected_auth_path = "legacy_session_key_cache" - logger.info("Nous inference auth: using cached legacy session key") + logger.info("Nous inference auth: using cached agent_key") _oauth_trace("agent_key_reuse", sequence_id=sequence_id) else: - fallback_reason = ( - "forced_legacy_session_keys" - if legacy_session_keys - else _nous_invoke_jwt_unavailable_reason( - access_token, - scope=state.get("scope"), - expires_at=state.get("expires_at"), - ) - ) - selected_auth_path = "legacy_session_key_mint" - logger.info( - "Nous inference auth: using legacy session key path (%s)", - fallback_reason, - ) - _oauth_trace( - "nous_legacy_session_key_selected", + _log_nous_legacy_session_key_selected( + fallback_reason or "legacy_session_key_required", + access_token=access_token, sequence_id=sequence_id, - reason=fallback_reason, - access_token_fp=_token_fingerprint(access_token), ) try: _oauth_trace( @@ -4646,6 +4896,11 @@ def resolve_nous_runtime_credentials( exc, reason="runtime_mint_retry_refresh_failure", ) + _quarantine_nous_pool_entries( + auth_store, + exc, + reason="runtime_mint_retry_refresh_failure", + ) _persist_state("terminal_runtime_mint_retry_refresh_failure") raise now = datetime.now(timezone.utc) @@ -4674,22 +4929,24 @@ def resolve_nous_runtime_credentials( # Persist retry refresh immediately for crash safety and cross-process visibility. _persist_state("post_refresh_mint_retry") - if ( - not legacy_session_keys - and _nous_invoke_jwt_is_usable( - access_token, - scope=state.get("scope"), - expires_at=state.get("expires_at"), - ) - ): - _set_nous_agent_key_from_invoke_jwt(state) + retry_auth_mode = ( + NOUS_INFERENCE_AUTH_LEGACY + if auth_mode == NOUS_INFERENCE_AUTH_LEGACY + else NOUS_INFERENCE_AUTH_FRESH + ) + retry_auth_path, _ = _choose_nous_inference_auth_path( + state, + access_token=access_token, + min_key_ttl_seconds=min_key_ttl_seconds, + auth_mode=retry_auth_mode, + ) + if retry_auth_path == "invoke_jwt": mint_payload = None selected_auth_path = "invoke_jwt" - logger.info("Nous inference auth: using NAS invoke JWT") - _oauth_trace( - "nous_invoke_jwt_selected", + _select_nous_invoke_jwt( + state, + access_token=access_token, sequence_id=sequence_id, - access_token_fp=_token_fingerprint(access_token), ) else: mint_payload = _mint_agent_key( @@ -4727,7 +4984,8 @@ def resolve_nous_runtime_credentials( _persist_state("resolve_nous_runtime_credentials_final") - _sync_nous_pool_from_auth_store() + if state_persisted: + _sync_nous_pool_from_auth_store() api_key = state.get("agent_key") if not isinstance(api_key, str) or not api_key: @@ -6433,10 +6691,7 @@ def _nous_device_code_login( or pconfig.inference_base_url ).rstrip("/") client_id = client_id or pconfig.client_id - explicit_scope = scope is not None - scope = scope or pconfig.scope - if _nous_legacy_session_keys_forced(): - scope = NOUS_LEGACY_AGENT_KEY_SCOPE + scope, explicit_scope = _nous_device_scope(scope, default_scope=pconfig.scope) timeout = httpx.Timeout(timeout_seconds) verify: bool | str = False if insecure else (ca_bundle if ca_bundle else True) @@ -6451,30 +6706,13 @@ def _nous_device_code_login( print(f"TLS verification: custom CA bundle ({ca_bundle})") with httpx.Client(timeout=timeout, headers={"Accept": "application/json"}, verify=verify) as client: - try: - device_data = _request_device_code( - client=client, - portal_base_url=portal_base_url, - client_id=client_id, - scope=scope, - ) - except Exception as exc: - if ( - not explicit_scope - and _nous_scope_has_invoke(scope) - and _is_nous_invoke_scope_refusal(exc) - ): - logger.info("Nous inference auth: NAS refused invoke scope, retrying legacy scope") - _oauth_trace("nous_device_code_invoke_scope_refused") - scope = NOUS_LEGACY_AGENT_KEY_SCOPE - device_data = _request_device_code( - client=client, - portal_base_url=portal_base_url, - client_id=client_id, - scope=scope, - ) - else: - raise + device_data, scope = _request_nous_device_code_with_scope_fallback( + client=client, + portal_base_url=portal_base_url, + client_id=client_id, + scope=scope, + allow_legacy_fallback=not explicit_scope, + ) verification_url = str(device_data["verification_uri_complete"]) user_code = str(device_data["user_code"]) @@ -6543,7 +6781,7 @@ def _nous_device_code_login( min_key_ttl_seconds=min_key_ttl_seconds, timeout_seconds=timeout_seconds, force_refresh=False, - force_mint=True, + auth_mode=NOUS_INFERENCE_AUTH_FRESH, ) except AuthError as exc: if exc.code == "subscription_required": diff --git a/hermes_cli/proxy/adapters/base.py b/hermes_cli/proxy/adapters/base.py index 5ac8a5dcedd..c7f36e25a2b 100644 --- a/hermes_cli/proxy/adapters/base.py +++ b/hermes_cli/proxy/adapters/base.py @@ -81,6 +81,21 @@ class UpstreamAdapter(ABC): refresh fails. The proxy will return 401 to the client. """ + def get_retry_credential( + self, + *, + failed_credential: UpstreamCredential, + status_code: int, + ) -> Optional[UpstreamCredential]: + """Return an alternate credential after an upstream auth failure. + + The default is no retry. Providers can override this for one-shot + fallback paths, such as switching from a preferred token type to a + legacy bearer after the upstream rejects the first request. + """ + del failed_credential, status_code + return None + def describe(self) -> str: """One-line status summary for ``proxy status``.""" try: diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index b69f9d52644..a8cfd4cbada 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -19,13 +19,16 @@ from typing import Any, Dict, FrozenSet, Optional from hermes_cli.auth import ( AuthError, DEFAULT_NOUS_INFERENCE_URL, + NOUS_INFERENCE_AUTH_AUTO, + NOUS_INFERENCE_AUTH_LEGACY, _load_auth_store, _is_terminal_nous_refresh_error, _quarantine_nous_oauth_state, + _quarantine_nous_pool_entries, _save_auth_store, _write_shared_nous_state, refresh_nous_oauth_from_state, -) + ) from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential logger = logging.getLogger(__name__) @@ -76,6 +79,21 @@ class NousPortalAdapter(UpstreamAdapter): ) def get_credential(self) -> UpstreamCredential: + return self._get_credential(auth_mode=NOUS_INFERENCE_AUTH_AUTO) + + def get_retry_credential( + self, + *, + failed_credential: UpstreamCredential, + status_code: int, + ) -> Optional[UpstreamCredential]: + del failed_credential + if status_code != 401: + return None + logger.info("proxy: Nous upstream rejected bearer; retrying with legacy session key") + return self._get_credential(auth_mode=NOUS_INFERENCE_AUTH_LEGACY) + + def _get_credential(self, *, auth_mode: str) -> UpstreamCredential: with self._lock: state = self._read_state() if state is None: @@ -84,7 +102,10 @@ class NousPortalAdapter(UpstreamAdapter): ) try: - refreshed = refresh_nous_oauth_from_state(state) + refreshed = refresh_nous_oauth_from_state( + state, + auth_mode=auth_mode, + ) except AuthError as exc: if _is_terminal_nous_refresh_error(exc): _quarantine_nous_oauth_state( @@ -92,7 +113,11 @@ class NousPortalAdapter(UpstreamAdapter): exc, reason="proxy_refresh_failure", ) - self._save_state(state) + self._save_state( + state, + quarantine_error=exc, + quarantine_reason="proxy_refresh_failure", + ) raise RuntimeError( f"Failed to refresh Nous Portal credentials: {exc}" ) from exc @@ -136,9 +161,21 @@ class NousPortalAdapter(UpstreamAdapter): return None return dict(state) # copy so the refresh helper can mutate freely - def _save_state(self, state: Dict[str, Any]) -> None: + def _save_state( + self, + state: Dict[str, Any], + *, + quarantine_error: Optional[AuthError] = None, + quarantine_reason: Optional[str] = None, + ) -> None: try: store = _load_auth_store() + if quarantine_error is not None and quarantine_reason: + _quarantine_nous_pool_entries( + store, + quarantine_error, + reason=quarantine_reason, + ) providers = store.setdefault("providers", {}) providers["nous"] = state _save_auth_store(store) diff --git a/hermes_cli/proxy/server.py b/hermes_cli/proxy/server.py index fa497f13291..a72f75d67ee 100644 --- a/hermes_cli/proxy/server.py +++ b/hermes_cli/proxy/server.py @@ -26,7 +26,7 @@ except ImportError: web = None # type: ignore[assignment] AIOHTTP_AVAILABLE = False -from hermes_cli.proxy.adapters.base import UpstreamAdapter +from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential logger = logging.getLogger(__name__) @@ -136,50 +136,93 @@ def create_app(adapter: UpstreamAdapter) -> "web.Application": logger.warning("proxy: credential resolution failed: %s", exc) return _json_error(401, str(exc), code="upstream_auth_failed") - upstream_url = f"{cred.base_url.rstrip('/')}{rel_path}" - # Preserve query string verbatim. - if request.query_string: - upstream_url = f"{upstream_url}?{request.query_string}" - # Forward body verbatim. Read into memory once — request bodies for # chat/completions/embeddings are small (<1MB typically). If we ever # need to forward large multipart uploads we'll switch to streaming # the request body too. body = await request.read() - fwd_headers = _filter_request_headers(request.headers) - fwd_headers["Authorization"] = f"{cred.token_type} {cred.bearer}" - - logger.debug( - "proxy: forwarding %s %s -> %s (body=%d bytes)", - request.method, rel_path, upstream_url, len(body), - ) - - # Use a per-request session so connection state doesn't leak between - # clients. Could be optimized to a shared session later. timeout = aiohttp.ClientTimeout(total=None, sock_connect=15, sock_read=300) - try: - session = aiohttp.ClientSession(timeout=timeout) - except Exception as exc: # pragma: no cover - aiohttp setup issue - return _json_error(500, f"proxy session init failed: {exc}") - try: - upstream_resp = await session.request( - request.method, - upstream_url, - data=body if body else None, - headers=fwd_headers, - allow_redirects=False, + async def _send_upstream(active_cred: UpstreamCredential): + upstream_url = f"{active_cred.base_url.rstrip('/')}{rel_path}" + # Preserve query string verbatim. + if request.query_string: + upstream_url = f"{upstream_url}?{request.query_string}" + + fwd_headers = _filter_request_headers(request.headers) + fwd_headers["Authorization"] = f"{active_cred.token_type} {active_cred.bearer}" + + logger.debug( + "proxy: forwarding %s %s -> %s (body=%d bytes)", + request.method, rel_path, upstream_url, len(body), ) - except aiohttp.ClientError as exc: - await session.close() - logger.warning("proxy: upstream connection failed: %s", exc) - return _json_error(502, f"upstream connection failed: {exc}", - code="upstream_unreachable") - except asyncio.TimeoutError: - await session.close() - return _json_error(504, "upstream request timed out", - code="upstream_timeout") + + try: + session = aiohttp.ClientSession(timeout=timeout) + except Exception as exc: # pragma: no cover - aiohttp setup issue + raise RuntimeError(f"proxy session init failed: {exc}") from exc + + try: + upstream_resp = await session.request( + request.method, + upstream_url, + data=body if body else None, + headers=fwd_headers, + allow_redirects=False, + ) + except Exception: + await session.close() + raise + return session, upstream_resp + + async def _open_upstream(active_cred: UpstreamCredential): + try: + return await _send_upstream(active_cred) + except RuntimeError as exc: + return _json_error(500, str(exc)), None + except aiohttp.ClientError as exc: + logger.warning("proxy: upstream connection failed: %s", exc) + return ( + _json_error( + 502, + f"upstream connection failed: {exc}", + code="upstream_unreachable", + ), + None, + ) + except asyncio.TimeoutError: + return ( + _json_error( + 504, + "upstream request timed out", + code="upstream_timeout", + ), + None, + ) + + session_or_response, upstream_resp = await _open_upstream(cred) + if upstream_resp is None: + return session_or_response + session = session_or_response + + if upstream_resp.status == 401: + try: + retry_cred = adapter.get_retry_credential( + failed_credential=cred, + status_code=upstream_resp.status, + ) + except Exception as exc: + logger.warning("proxy: retry credential resolution failed: %s", exc) + retry_cred = None + + if retry_cred is not None: + upstream_resp.release() + await session.close() + session_or_response, upstream_resp = await _open_upstream(retry_cred) + if upstream_resp is None: + return session_or_response + session = session_or_response # Stream response back. Headers first, then chunked body. resp = web.StreamResponse( diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 8a1e4aca2e1..bfd47e9cc24 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -1815,7 +1815,11 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: so the UI can render the verification page link + user code. """ if provider_id == "nous": - from hermes_cli.auth import _request_device_code, PROVIDER_REGISTRY + from hermes_cli.auth import ( + _nous_device_scope, + _request_nous_device_code_with_scope_fallback, + PROVIDER_REGISTRY, + ) import httpx pconfig = PROVIDER_REGISTRY["nous"] portal_base_url = ( @@ -1824,22 +1828,31 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: or pconfig.portal_base_url ).rstrip("/") client_id = pconfig.client_id - scope = pconfig.scope + scope, explicit_scope = _nous_device_scope(None, default_scope=pconfig.scope) + def _do_nous_device_request(): - with httpx.Client(timeout=httpx.Timeout(15.0), headers={"Accept": "application/json"}) as client: - return _request_device_code( + with httpx.Client( + timeout=httpx.Timeout(15.0), + headers={"Accept": "application/json"}, + ) as client: + return _request_nous_device_code_with_scope_fallback( client=client, portal_base_url=portal_base_url, client_id=client_id, scope=scope, + allow_legacy_fallback=not explicit_scope, ) - device_data = await asyncio.get_running_loop().run_in_executor(None, _do_nous_device_request) + + device_data, effective_scope = await asyncio.get_running_loop().run_in_executor( + None, _do_nous_device_request + ) sid, sess = _new_oauth_session("nous", "device_code") sess["device_code"] = str(device_data["device_code"]) sess["interval"] = int(device_data["interval"]) sess["expires_at"] = time.time() + int(device_data["expires_in"]) sess["portal_base_url"] = portal_base_url sess["client_id"] = client_id + sess["scope"] = effective_scope threading.Thread( target=_nous_poller, args=(sid,), daemon=True, name=f"oauth-poll-{sid[:6]}" ).start() @@ -1968,7 +1981,11 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: def _nous_poller(session_id: str) -> None: """Background poller that drives a Nous device-code flow to completion.""" - from hermes_cli.auth import _poll_for_token, refresh_nous_oauth_from_state + from hermes_cli.auth import ( + NOUS_INFERENCE_AUTH_FRESH, + _poll_for_token, + refresh_nous_oauth_from_state, + ) from datetime import datetime, timezone import httpx with _oauth_sessions_lock: @@ -1979,6 +1996,7 @@ def _nous_poller(session_id: str) -> None: client_id = sess["client_id"] device_code = sess["device_code"] interval = sess["interval"] + scope = sess.get("scope") expires_in = max(60, int(sess["expires_at"] - time.time())) try: with httpx.Client(timeout=httpx.Timeout(15.0), headers={"Accept": "application/json"}) as client: @@ -1997,7 +2015,7 @@ def _nous_poller(session_id: str) -> None: "portal_base_url": portal_base_url, "inference_base_url": token_data.get("inference_base_url"), "client_id": client_id, - "scope": token_data.get("scope"), + "scope": token_data.get("scope") or scope, "token_type": token_data.get("token_type", "Bearer"), "access_token": token_data["access_token"], "refresh_token": token_data.get("refresh_token"), @@ -2009,8 +2027,11 @@ def _nous_poller(session_id: str) -> None: "expires_in": token_ttl, } full_state = refresh_nous_oauth_from_state( - auth_state, min_key_ttl_seconds=300, timeout_seconds=15.0, - force_refresh=False, force_mint=True, + auth_state, + min_key_ttl_seconds=300, + timeout_seconds=15.0, + force_refresh=False, + auth_mode=NOUS_INFERENCE_AUTH_FRESH, ) from hermes_cli.auth import persist_nous_credentials persist_nous_credentials(full_state) diff --git a/run_agent.py b/run_agent.py index 6e9877a1182..1244d372fdf 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2628,12 +2628,20 @@ class AIAgent: return False try: - from hermes_cli.auth import resolve_nous_runtime_credentials + from hermes_cli.auth import ( + NOUS_INFERENCE_AUTH_AUTO, + NOUS_INFERENCE_AUTH_LEGACY, + resolve_nous_runtime_credentials, + ) creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - force_mint=force, + auth_mode=( + NOUS_INFERENCE_AUTH_LEGACY + if force + else NOUS_INFERENCE_AUTH_AUTO + ), ) except Exception as exc: logger.debug("Nous credential refresh failed: %s", exc) diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index f7eaf9fa273..875b08d91f0 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -566,7 +566,7 @@ def test_load_pool_mirrors_nous_invoke_jwt_agent_key_runtime_api_key(tmp_path, m assert pool_entry["agent_key_expires_at"] == expires_at -def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): +def test_nous_pool_terminal_refresh_removes_device_code_entry(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared")) _write_auth_store( @@ -591,7 +591,7 @@ def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): }, ) - from agent.credential_pool import load_pool + from agent.credential_pool import PooledCredential, load_pool from hermes_cli import auth as auth_mod from hermes_cli.auth import AuthError @@ -606,18 +606,30 @@ def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): relogin_required=True, ) + pool = load_pool("nous") + selected = pool.select() + assert selected is not None + assert selected.source == "device_code" + pool.add_entry(PooledCredential.from_dict("nous", { + "id": "legacy-seeded", + "source": "manual:device_code", + "auth_type": "oauth", + "access_token": "old-access-token", + "refresh_token": "old-refresh-token", + "agent_key": "old-agent-key", + })) + pool.add_entry(PooledCredential.from_dict("nous", { + "id": "manual-key", + "source": "manual", + "auth_type": "api_key", + "access_token": "manual-nous-key", + })) + monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _terminal_refresh_failure) - pool = load_pool("nous") - assert pool.select() is not None assert pool.try_refresh_current() is None - entry = pool.entries()[0] - assert entry.last_status == "exhausted" - assert entry.last_error_code == 401 - assert entry.refresh_token is None - assert entry.access_token is None - assert entry.agent_key is None + assert [entry.id for entry in pool.entries()] == ["manual-key"] auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text()) nous_state = auth_payload["providers"]["nous"] @@ -625,11 +637,63 @@ def test_nous_pool_terminal_refresh_clears_tokens(tmp_path, monkeypatch): assert not nous_state.get("access_token") assert not nous_state.get("agent_key") assert nous_state["last_auth_error"]["code"] == "invalid_grant" + assert [entry["id"] for entry in auth_payload["credential_pool"]["nous"]] == ["manual-key"] assert pool.try_refresh_current() is None assert refresh_calls["count"] == 1 +def test_load_pool_removes_nous_device_code_when_singleton_quarantined(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + _write_auth_store( + tmp_path, + { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "inference_base_url": "https://inference.example.com/v1", + "client_id": "hermes-cli", + "last_auth_error": {"code": "invalid_grant"}, + } + }, + "credential_pool": { + "nous": [ + { + "id": "seeded-current", + "source": "device_code", + "auth_type": "oauth", + "access_token": "stale-access", + "refresh_token": "stale-refresh", + "agent_key": "stale-agent", + }, + { + "id": "seeded-legacy", + "source": "manual:device_code", + "auth_type": "oauth", + "access_token": "older-stale-access", + }, + { + "id": "manual-key", + "source": "manual", + "auth_type": "api_key", + "access_token": "manual-nous-key", + }, + ] + }, + }, + ) + + from agent.credential_pool import load_pool + + pool = load_pool("nous") + + assert [entry.id for entry in pool.entries()] == ["manual-key"] + auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text()) + assert [entry["id"] for entry in auth_payload["credential_pool"]["nous"]] == ["manual-key"] + + def test_load_pool_removes_stale_file_backed_singleton_entry(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index 1d07737a857..0bdb1330a29 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -231,6 +231,83 @@ def test_resolve_nous_runtime_credentials_prefers_invoke_jwt_and_mirrors( assert pool_entries[0]["source"] == auth_mod.NOUS_DEVICE_CODE_SOURCE +def test_resolve_nous_runtime_credentials_invoke_jwt_is_idempotent( + tmp_path, + monkeypatch, +): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + hermes_home.mkdir(parents=True, exist_ok=True) + exp = int(time.time() + 3600) + expires_at = datetime.fromtimestamp(exp, tz=timezone.utc).isoformat() + token = _jwt_with_claims({ + "sub": "test-user", + "scope": auth_mod.DEFAULT_NOUS_SCOPE, + "exp": exp, + }) + original_obtained_at = "2026-04-17T22:00:10+00:00" + auth_store = { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "inference_base_url": "https://inference.example.com/v1", + "client_id": "hermes-cli", + "token_type": "Bearer", + "scope": auth_mod.DEFAULT_NOUS_SCOPE, + "access_token": token, + "refresh_token": "refresh-token", + "obtained_at": "2026-02-01T00:00:00+00:00", + "expires_in": 123, + "expires_at": expires_at, + "agent_key": token, + "agent_key_id": None, + "agent_key_expires_at": expires_at, + "agent_key_expires_in": 123, + "agent_key_reused": False, + "agent_key_obtained_at": original_obtained_at, + "tls": {"insecure": False, "ca_bundle": None}, + }, + }, + } + auth_path = hermes_home / "auth.json" + auth_path.write_text(json.dumps(auth_store, indent=2)) + before_content = auth_path.read_text() + before_mtime = auth_path.stat().st_mtime_ns + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + def _unexpected_mint(*args, **kwargs): + raise AssertionError("stable invoke JWT should not mint a legacy key") + + def _unexpected_shared_write(*args, **kwargs): + raise AssertionError("unchanged invoke JWT resolution should not sync shared store") + + sync_calls = [] + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _unexpected_mint) + monkeypatch.setattr(auth_mod, "_write_shared_nous_state", _unexpected_shared_write) + monkeypatch.setattr( + auth_mod, + "_sync_nous_pool_from_auth_store", + lambda: sync_calls.append(True), + ) + + creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) + + assert creds["api_key"] == token + assert creds["source"] == "invoke_jwt" + assert auth_path.read_text() == before_content + assert auth_path.stat().st_mtime_ns == before_mtime + assert sync_calls == [] + payload = json.loads(auth_path.read_text()) + assert ( + payload["providers"]["nous"]["agent_key_obtained_at"] + == original_obtained_at + ) + + def test_resolve_nous_runtime_credentials_trusts_invoke_jwt_exp_over_stale_metadata( tmp_path, monkeypatch, @@ -301,6 +378,41 @@ def test_resolve_nous_runtime_credentials_does_not_apply_legacy_ttl_to_invoke_jw assert payload["credential_pool"]["nous"][0]["agent_key"] == token +def test_legacy_auth_mode_bypasses_usable_invoke_jwt(tmp_path, monkeypatch): + import hermes_cli.auth as auth_mod + + hermes_home = tmp_path / "hermes" + token = _invoke_jwt(seconds=3600) + _setup_nous_auth( + hermes_home, + access_token=token, + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at=_future_iso(3600), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + mint_calls = [] + + def _fake_mint_agent_key(*, client, portal_base_url, access_token, min_ttl_seconds): + del client, portal_base_url, min_ttl_seconds + mint_calls.append(access_token) + return _mint_payload(api_key="legacy-after-jwt-401") + + monkeypatch.setattr(auth_mod, "_mint_agent_key", _fake_mint_agent_key) + + creds = auth_mod.resolve_nous_runtime_credentials( + min_key_ttl_seconds=300, + auth_mode=auth_mod.NOUS_INFERENCE_AUTH_LEGACY, + ) + + assert mint_calls == [token] + assert creds["api_key"] == "legacy-after-jwt-401" + assert creds["auth_path"] == "legacy_session_key_mint" + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload["providers"]["nous"]["agent_key"] == "legacy-after-jwt-401" + + def test_resolve_nous_runtime_credentials_falls_back_when_invoke_scope_missing( tmp_path, monkeypatch, @@ -735,6 +847,9 @@ def test_terminal_refresh_failure_quarantines_tokens( hermes_home = tmp_path / "hermes" _setup_nous_auth(hermes_home, refresh_token="refresh-old") monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + from agent.credential_pool import load_pool + + assert load_pool("nous").select() is not None shared_state = _full_state_fixture() shared_state["access_token"] = "access-old" @@ -765,6 +880,8 @@ def test_terminal_refresh_failure_quarantines_tokens( assert not state_after_failure.get("agent_key") assert state_after_failure["last_auth_error"]["code"] == "invalid_grant" assert auth_mod._read_shared_nous_state() is None + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload.get("credential_pool", {}).get("nous") == [] with pytest.raises(AuthError, match="No access token found"): auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) @@ -780,6 +897,9 @@ def test_managed_access_token_refresh_failure_quarantines_tokens( hermes_home = tmp_path / "hermes" _setup_nous_auth(hermes_home, refresh_token="refresh-old") monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + from agent.credential_pool import load_pool + + assert load_pool("nous").select() is not None refresh_calls: list[str] = [] @@ -802,6 +922,8 @@ def test_managed_access_token_refresh_failure_quarantines_tokens( assert not state_after_failure.get("refresh_token") assert not state_after_failure.get("access_token") assert state_after_failure["last_auth_error"]["message"] == "Invalid refresh token" + payload = json.loads((hermes_home / "auth.json").read_text()) + assert payload.get("credential_pool", {}).get("nous") == [] with pytest.raises(AuthError, match="No access token found"): auth_mod.resolve_nous_access_token() @@ -1076,7 +1198,11 @@ def test_persist_nous_credentials_allows_recovery_from_401(tmp_path, monkeypatch calls after a Nous 401 — before the fix it would raise AuthError because providers.nous was empty. """ - from hermes_cli.auth import persist_nous_credentials, resolve_nous_runtime_credentials + from hermes_cli.auth import ( + NOUS_INFERENCE_AUTH_FRESH, + persist_nous_credentials, + resolve_nous_runtime_credentials, + ) hermes_home = tmp_path / "hermes" hermes_home.mkdir(parents=True, exist_ok=True) @@ -1104,7 +1230,10 @@ def test_persist_nous_credentials_allows_recovery_from_401(tmp_path, monkeypatch monkeypatch.setattr("hermes_cli.auth._refresh_access_token", _fake_refresh_access_token) monkeypatch.setattr("hermes_cli.auth._mint_agent_key", _fake_mint_agent_key) - creds = resolve_nous_runtime_credentials(min_key_ttl_seconds=300, force_mint=True) + creds = resolve_nous_runtime_credentials( + min_key_ttl_seconds=300, + auth_mode=NOUS_INFERENCE_AUTH_FRESH, + ) assert creds["api_key"] == "new-agent-key" @@ -1569,7 +1698,7 @@ def test_try_import_shared_rehydrates_on_success(shared_store_env, monkeypatch): def _fake_refresh(state, **kwargs): # Simulate portal returning fresh tokens + a new agent_key assert kwargs.get("force_refresh") is True - assert kwargs.get("force_mint") is True + assert kwargs.get("auth_mode") == auth_mod.NOUS_INFERENCE_AUTH_FRESH return { **state, "access_token": "fresh-access-tok", @@ -1697,7 +1826,7 @@ def test_runtime_refresh_uses_newer_shared_token_before_local_stale_token( creds = auth_mod.resolve_nous_runtime_credentials( min_key_ttl_seconds=300, - force_mint=True, + auth_mode=auth_mod.NOUS_INFERENCE_AUTH_FRESH, ) assert creds["api_key"] == "agent-key-from-shared-token" diff --git a/tests/hermes_cli/test_proxy.py b/tests/hermes_cli/test_proxy.py index 3ab06eeb92f..9303fb1c702 100644 --- a/tests/hermes_cli/test_proxy.py +++ b/tests/hermes_cli/test_proxy.py @@ -141,6 +141,45 @@ def test_nous_adapter_get_credential_refreshes_and_persists(tmp_path, monkeypatc assert stored["providers"]["nous"]["agent_key"] == "minted-bearer" +def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + _write_auth_store(tmp_path, { + "access_token": "jwt-access", + "refresh_token": "refresh-tok", + "client_id": "hermes-cli", + "portal_base_url": "https://portal.nousresearch.com", + "inference_base_url": "https://inference-api.nousresearch.com/v1", + "agent_key": "jwt-access", + }) + + refreshed_state = { + "access_token": "jwt-access", + "refresh_token": "refresh-tok", + "client_id": "hermes-cli", + "portal_base_url": "https://portal.nousresearch.com", + "inference_base_url": "https://inference-api.nousresearch.com/v1", + "agent_key": "legacy-bearer", + "agent_key_expires_at": "2099-01-01T00:00:00Z", + } + + with patch( + "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + return_value=refreshed_state, + ) as mock_refresh: + adapter = NousPortalAdapter() + cred = adapter.get_retry_credential( + failed_credential=UpstreamCredential( + bearer="jwt-access", + base_url="https://inference-api.nousresearch.com/v1", + ), + status_code=401, + ) + + assert cred is not None + assert cred.bearer == "legacy-bearer" + assert mock_refresh.call_args.kwargs["auth_mode"] == "legacy" + + def test_nous_adapter_get_credential_raises_when_not_logged_in(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) adapter = NousPortalAdapter() @@ -166,6 +205,7 @@ def test_nous_adapter_get_credential_raises_on_refresh_failure(tmp_path, monkeyp def test_nous_adapter_quarantines_terminal_refresh_failure(tmp_path, monkeypatch): from hermes_cli.auth import AuthError + from agent.credential_pool import load_pool monkeypatch.setenv("HERMES_HOME", str(tmp_path)) _write_auth_store(tmp_path, { @@ -173,6 +213,7 @@ def test_nous_adapter_quarantines_terminal_refresh_failure(tmp_path, monkeypatch "refresh_token": "refresh-tok", "agent_key": "stale-agent-key", }) + assert load_pool("nous").select() is not None with patch( "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", @@ -193,6 +234,7 @@ def test_nous_adapter_quarantines_terminal_refresh_failure(tmp_path, monkeypatch assert not nous_state.get("access_token") assert not nous_state.get("agent_key") assert nous_state["last_auth_error"]["code"] == "invalid_grant" + assert stored.get("credential_pool", {}).get("nous") == [] def test_nous_adapter_get_credential_raises_when_no_agent_key_returned(tmp_path, monkeypatch): @@ -291,12 +333,15 @@ class FakeAdapter(UpstreamAdapter): """A test adapter that returns a fixed credential without touching disk.""" def __init__(self, base_url: str, bearer: str = "test-bearer", - allowed=None, raise_on_credential=False): + allowed=None, raise_on_credential=False, + retry_bearer: str | None = None): self._base_url = base_url self._bearer = bearer self._allowed = frozenset(allowed or ["/chat/completions"]) self._raise = raise_on_credential + self._retry_bearer = retry_bearer self.calls = 0 + self.retry_calls = 0 @property def name(self): return "fake" @@ -318,6 +363,17 @@ class FakeAdapter(UpstreamAdapter): expires_at="2099-01-01T00:00:00Z", ) + def get_retry_credential(self, *, failed_credential, status_code): + del failed_credential + self.retry_calls += 1 + if status_code != 401 or not self._retry_bearer: + return None + return UpstreamCredential( + bearer=self._retry_bearer, + base_url=self._base_url, + expires_at="2099-01-01T00:00:00Z", + ) + async def _start_runner(app: "web.Application"): """Spin up an aiohttp app on an ephemeral localhost port. Returns (runner, base_url).""" @@ -358,6 +414,25 @@ def _build_fake_upstream(captured: Dict[str, Any]) -> "web.Application": return app +def _build_retrying_fake_upstream(captured: Dict[str, Any]) -> "web.Application": + async def maybe_unauthorized(request): + body = await request.read() + auth = request.headers.get("Authorization") + captured["requests"].append({ + "method": request.method, + "path": request.path, + "auth": auth, + "body": body.decode("utf-8") if body else "", + }) + if auth == "Bearer jwt-bearer": + return web.json_response({"error": "bad token"}, status=401) + return web.json_response({"ok": True}) + + app = web.Application() + app.router.add_route("*", "/v1/chat/completions", maybe_unauthorized) + return app + + def test_server_forwards_chat_completions(): async def run(): captured: Dict[str, Any] = {"requests": []} @@ -388,6 +463,41 @@ def test_server_forwards_chat_completions(): asyncio.run(run()) +def test_server_retries_once_with_adapter_retry_credential_on_401(): + async def run(): + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner( + _build_retrying_fake_upstream(captured) + ) + adapter = FakeAdapter( + f"{upstream_base}/v1", + bearer="jwt-bearer", + retry_bearer="legacy-bearer", + ) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", + json={"model": "Hermes-4-70B"}, + ) as resp: + assert resp.status == 200 + data = await resp.json() + assert data["ok"] is True + + assert adapter.retry_calls == 1 + assert [req["auth"] for req in captured["requests"]] == [ + "Bearer jwt-bearer", + "Bearer legacy-bearer", + ] + finally: + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + def test_server_rejects_disallowed_path(): async def run(): adapter = FakeAdapter("http://unused.example/v1", allowed=["/chat/completions"]) diff --git a/tests/hermes_cli/test_web_oauth_dispatch.py b/tests/hermes_cli/test_web_oauth_dispatch.py index 23b72a303cf..b9ee20ccae8 100644 --- a/tests/hermes_cli/test_web_oauth_dispatch.py +++ b/tests/hermes_cli/test_web_oauth_dispatch.py @@ -19,11 +19,12 @@ The fix: These tests pin the corrected behavior. """ +import asyncio import time from datetime import datetime, timezone from unittest.mock import patch -import pytest +import httpx from fastapi.testclient import TestClient from hermes_cli.web_server import _SESSION_TOKEN, app @@ -32,6 +33,32 @@ client = TestClient(app) HEADERS = {"X-Hermes-Session-Token": _SESSION_TOKEN} +def _fake_nous_device_data(): + return { + "device_code": "device-code", + "user_code": "NOUS-1234", + "verification_uri": "https://portal.nousresearch.com/device", + "verification_uri_complete": ( + "https://portal.nousresearch.com/device?user_code=NOUS-1234" + ), + "expires_in": 600, + "interval": 5, + } + + +def _invoke_scope_refusal(): + request = httpx.Request("POST", "https://portal.nousresearch.com/oauth/device/code") + response = httpx.Response( + 400, + json={ + "error": "invalid_scope", + "error_description": "unsupported scope inference:invoke", + }, + request=request, + ) + return httpx.HTTPStatusError("invalid scope", request=request, response=response) + + def test_minimax_login_does_not_launch_anthropic_flow(): """Click 'Login' on MiniMax → MUST NOT return claude.ai auth_url.""" fake_user_code_resp = { @@ -48,6 +75,9 @@ def test_minimax_login_does_not_launch_anthropic_flow(): ), patch( "hermes_cli.auth._minimax_pkce_pair", return_value=("verifier-stub", "challenge-stub", "stub-state"), + ), patch( + "hermes_cli.web_server._minimax_poller", + return_value=None, ): resp = client.post( "/api/providers/oauth/minimax-oauth/start", @@ -69,6 +99,113 @@ def test_minimax_login_does_not_launch_anthropic_flow(): assert body["expires_in"] == 600 +def test_nous_dashboard_device_flow_honors_legacy_scope_override(monkeypatch): + from hermes_cli import auth as auth_mod + from hermes_cli import web_server as ws + + requested_scopes = [] + + def fake_request_device_code(**kwargs): + requested_scopes.append(kwargs["scope"]) + return _fake_nous_device_data() + + monkeypatch.setenv(auth_mod.NOUS_LEGACY_SESSION_KEYS_ENV, "true") + monkeypatch.setattr(auth_mod, "_request_device_code", fake_request_device_code) + monkeypatch.setattr(ws, "_nous_poller", lambda sid: None) + + result = asyncio.run(ws._start_device_code_flow("nous")) + try: + assert requested_scopes == [auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE] + assert result["flow"] == "device_code" + assert result["user_code"] == "NOUS-1234" + assert ( + ws._oauth_sessions[result["session_id"]]["scope"] + == auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE + ) + finally: + ws._oauth_sessions.pop(result["session_id"], None) + + +def test_nous_dashboard_device_flow_retries_legacy_scope_on_invoke_refusal(monkeypatch): + from hermes_cli import auth as auth_mod + from hermes_cli import web_server as ws + + requested_scopes = [] + + def fake_request_device_code(**kwargs): + requested_scopes.append(kwargs["scope"]) + if len(requested_scopes) == 1: + raise _invoke_scope_refusal() + return _fake_nous_device_data() + + monkeypatch.delenv(auth_mod.NOUS_LEGACY_SESSION_KEYS_ENV, raising=False) + monkeypatch.setattr(auth_mod, "_request_device_code", fake_request_device_code) + monkeypatch.setattr(ws, "_nous_poller", lambda sid: None) + + result = asyncio.run(ws._start_device_code_flow("nous")) + try: + assert requested_scopes == [ + auth_mod.DEFAULT_NOUS_SCOPE, + auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + ] + assert ( + ws._oauth_sessions[result["session_id"]]["scope"] + == auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE + ) + finally: + ws._oauth_sessions.pop(result["session_id"], None) + + +def test_nous_dashboard_poller_preserves_effective_scope_when_token_omits_scope(monkeypatch): + from hermes_cli import auth as auth_mod + from hermes_cli import web_server as ws + + session_id = "nous-effective-scope-test" + ws._oauth_sessions[session_id] = { + "session_id": session_id, + "provider": "nous", + "flow": "device_code", + "created_at": time.time(), + "status": "pending", + "error_message": None, + "portal_base_url": "https://portal.nousresearch.com", + "client_id": "hermes-cli", + "device_code": "device-code", + "interval": 5, + "expires_at": time.time() + 600, + "scope": auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE, + } + captured_state = {} + + def fake_refresh_nous_oauth_from_state(state, **kwargs): + captured_state.update(state) + return {**state, "agent_key": "legacy-agent-key"} + + monkeypatch.setattr( + auth_mod, + "_poll_for_token", + lambda **kwargs: { + "access_token": "access-token", + "refresh_token": "refresh-token", + "expires_in": 3600, + "token_type": "Bearer", + }, + ) + monkeypatch.setattr( + auth_mod, + "refresh_nous_oauth_from_state", + fake_refresh_nous_oauth_from_state, + ) + monkeypatch.setattr(auth_mod, "persist_nous_credentials", lambda state: None) + + try: + ws._nous_poller(session_id) + assert captured_state["scope"] == auth_mod.NOUS_LEGACY_AGENT_KEY_SCOPE + assert ws._oauth_sessions[session_id]["status"] == "approved" + finally: + ws._oauth_sessions.pop(session_id, None) + + def test_minimax_dashboard_poller_accepts_absolute_ms_expired_in(): """Dashboard MiniMax completion must accept unix-ms token expiry values.""" from hermes_cli import web_server as ws diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index a72359227a6..e569da31666 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -3667,7 +3667,7 @@ class TestNousCredentialRefresh: assert ok is True assert closed["value"] is True - assert captured["force_mint"] is True + assert captured["auth_mode"] == "legacy" assert rebuilt["kwargs"]["api_key"] == "new-nous-key" assert ( rebuilt["kwargs"]["base_url"] == "https://inference-api.nousresearch.com/v1" From 20bffa5b37ce121f6adc1c68b4759440a79473ec Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Sun, 17 May 2026 21:18:53 +1000 Subject: [PATCH 135/142] refactor(auth): mostly cleanups and style changes --- agent/auxiliary_client.py | 17 ++- agent/credential_pool.py | 6 +- hermes_cli/auth.py | 146 +++++++++----------- hermes_cli/proxy/adapters/base.py | 2 +- hermes_cli/proxy/adapters/nous_portal.py | 21 +-- hermes_cli/web_server.py | 11 +- run_agent.py | 10 +- tests/hermes_cli/test_auth_nous_provider.py | 25 ++-- tests/hermes_cli/test_proxy.py | 30 +++- tests/run_agent/test_run_agent.py | 2 +- 10 files changed, 145 insertions(+), 125 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index e67b37b00da..4d11804f4cb 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -1253,18 +1253,18 @@ def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[ """ try: from hermes_cli.auth import ( - NOUS_INFERENCE_AUTH_AUTO, - NOUS_INFERENCE_AUTH_LEGACY, + NOUS_INFERENCE_AUTH_MODE_AUTO, + NOUS_INFERENCE_AUTH_MODE_LEGACY, resolve_nous_runtime_credentials, ) creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - auth_mode=( - NOUS_INFERENCE_AUTH_LEGACY + inference_auth_mode=( + NOUS_INFERENCE_AUTH_MODE_LEGACY if force_refresh - else NOUS_INFERENCE_AUTH_AUTO + else NOUS_INFERENCE_AUTH_MODE_AUTO ), ) except Exception as exc: @@ -2509,12 +2509,15 @@ def _refresh_provider_credentials(provider: str) -> bool: _evict_cached_clients(normalized) return True if normalized == "nous": - from hermes_cli.auth import NOUS_INFERENCE_AUTH_LEGACY, resolve_nous_runtime_credentials + from hermes_cli.auth import ( + NOUS_INFERENCE_AUTH_MODE_LEGACY, + resolve_nous_runtime_credentials, + ) creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - auth_mode=NOUS_INFERENCE_AUTH_LEGACY, + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_LEGACY, ) if not str(creds.get("api_key", "") or "").strip(): return False diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 7c91a08d2aa..7bdfe1c2973 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -831,10 +831,10 @@ class CredentialPool: nous_state, min_key_ttl_seconds=DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, force_refresh=force, - auth_mode=( - auth_mod.NOUS_INFERENCE_AUTH_LEGACY + inference_auth_mode=( + auth_mod.NOUS_INFERENCE_AUTH_MODE_LEGACY if force - else auth_mod.NOUS_INFERENCE_AUTH_AUTO + else auth_mod.NOUS_INFERENCE_AUTH_MODE_AUTO ), ) # Apply returned fields: dataclass fields via replace, extras via dict update diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 783f2c0c655..e65d9da20c8 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -78,13 +78,21 @@ NOUS_INFERENCE_INVOKE_SCOPE = "inference:invoke" DEFAULT_NOUS_SCOPE = f"{NOUS_INFERENCE_INVOKE_SCOPE} {NOUS_LEGACY_AGENT_KEY_SCOPE}" NOUS_LEGACY_SESSION_KEYS_ENV = "HERMES_AGENT_USE_LEGACY_SESSION_KEYS" NOUS_DEVICE_CODE_SOURCE = "device_code" -NOUS_INFERENCE_AUTH_AUTO = "auto" -NOUS_INFERENCE_AUTH_FRESH = "fresh" -NOUS_INFERENCE_AUTH_LEGACY = "legacy" +NOUS_INFERENCE_AUTH_MODE_AUTO = "auto" +NOUS_INFERENCE_AUTH_MODE_FRESH = "fresh" +NOUS_INFERENCE_AUTH_MODE_LEGACY = "legacy" NOUS_INFERENCE_AUTH_MODES = frozenset({ - NOUS_INFERENCE_AUTH_AUTO, - NOUS_INFERENCE_AUTH_FRESH, - NOUS_INFERENCE_AUTH_LEGACY, + NOUS_INFERENCE_AUTH_MODE_AUTO, + NOUS_INFERENCE_AUTH_MODE_FRESH, + NOUS_INFERENCE_AUTH_MODE_LEGACY, +}) +NOUS_AUTH_PATH_INVOKE_JWT = "invoke_jwt" +NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE = "legacy_session_key_cache" +NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT = "legacy_session_key_mint" +NOUS_AUTH_PATHS = frozenset({ + NOUS_AUTH_PATH_INVOKE_JWT, + NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE, + NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT, }) DEFAULT_AGENT_KEY_MIN_TTL_SECONDS = 30 * 60 # 30 minutes ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 # refresh 2 min before expiry @@ -1592,12 +1600,13 @@ def _nous_scope_has_invoke(raw_scope: Any) -> bool: return NOUS_INFERENCE_INVOKE_SCOPE in _scope_values(raw_scope) -def _normalize_nous_auth_mode(auth_mode: Optional[str]) -> str: - mode = str(auth_mode or NOUS_INFERENCE_AUTH_AUTO).strip().lower() +def _normalize_nous_inference_auth_mode(inference_auth_mode: Optional[str]) -> str: + mode = str(inference_auth_mode or NOUS_INFERENCE_AUTH_MODE_AUTO).strip().lower() if mode not in NOUS_INFERENCE_AUTH_MODES: allowed = ", ".join(sorted(NOUS_INFERENCE_AUTH_MODES)) raise ValueError( - f"Invalid Nous inference auth mode {auth_mode!r}; expected one of: {allowed}" + "Invalid Nous inference auth mode " + f"{inference_auth_mode!r}; expected one of: {allowed}" ) return mode @@ -1649,89 +1658,57 @@ def _nous_invoke_jwt_is_usable( ) -def _nous_invoke_jwt_unavailable_reason( - token: Any, - *, - scope: Any = None, - expires_at: Any = None, - min_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, -) -> str: - return ( - _nous_invoke_jwt_status( - token, - scope=scope, - expires_at=expires_at, - min_ttl_seconds=min_ttl_seconds, - ) - or "invoke_jwt_unavailable" - ) - - -def _nous_can_select_invoke_jwt(auth_mode: str = NOUS_INFERENCE_AUTH_AUTO) -> bool: - return ( - not _nous_legacy_session_keys_forced() - and _normalize_nous_auth_mode(auth_mode) != NOUS_INFERENCE_AUTH_LEGACY - ) - - def _nous_legacy_session_key_reason( token: Any, *, scope: Any = None, expires_at: Any = None, - auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, + inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, ) -> str: - if _normalize_nous_auth_mode(auth_mode) == NOUS_INFERENCE_AUTH_LEGACY: + if inference_auth_mode == NOUS_INFERENCE_AUTH_MODE_LEGACY: return "forced_legacy_session_key" if _nous_legacy_session_keys_forced(): return "forced_legacy_session_keys" - return _nous_invoke_jwt_unavailable_reason( - token, - scope=scope, - expires_at=expires_at, + return ( + _nous_invoke_jwt_status(token, scope=scope, expires_at=expires_at) + or "invoke_jwt_unavailable" ) -def _nous_cached_agent_key_is_usable( - state: Dict[str, Any], - min_ttl_seconds: int, -) -> bool: - return _agent_key_is_usable(state, min_ttl_seconds) - - def _choose_nous_inference_auth_path( state: Dict[str, Any], *, access_token: Any = None, min_key_ttl_seconds: int = DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, - auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, + inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, ) -> Tuple[str, Optional[str]]: - auth_mode = _normalize_nous_auth_mode(auth_mode) + inference_auth_mode = _normalize_nous_inference_auth_mode(inference_auth_mode) token = state.get("access_token") if access_token is None else access_token if ( - _nous_can_select_invoke_jwt(auth_mode) + not _nous_legacy_session_keys_forced() + and inference_auth_mode != NOUS_INFERENCE_AUTH_MODE_LEGACY and _nous_invoke_jwt_is_usable( token, scope=state.get("scope"), expires_at=state.get("expires_at"), ) ): - return "invoke_jwt", None + return NOUS_AUTH_PATH_INVOKE_JWT, None if ( - auth_mode == NOUS_INFERENCE_AUTH_AUTO - and _nous_cached_agent_key_is_usable( + inference_auth_mode == NOUS_INFERENCE_AUTH_MODE_AUTO + and _agent_key_is_usable( state, max(60, int(min_key_ttl_seconds)), ) ): - return "legacy_session_key_cache", None + return NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE, None return ( - "legacy_session_key_mint", + NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT, _nous_legacy_session_key_reason( token, scope=state.get("scope"), expires_at=state.get("expires_at"), - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, ), ) @@ -3660,7 +3637,7 @@ def _is_nous_invoke_scope_refusal(exc: Exception) -> bool: ) -def _nous_device_scope( +def _nous_device_scope_with_env_override( requested_scope: Optional[str], *, default_scope: str = DEFAULT_NOUS_SCOPE, @@ -4131,7 +4108,7 @@ def _try_import_shared_nous_state( min_key_ttl_seconds=min_key_ttl_seconds, timeout_seconds=timeout_seconds, force_refresh=True, - auth_mode=NOUS_INFERENCE_AUTH_FRESH, + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_FRESH, ) _write_shared_nous_state(refreshed) except AuthError as exc: @@ -4440,10 +4417,10 @@ def refresh_nous_oauth_pure( insecure: Optional[bool] = None, ca_bundle: Optional[str] = None, force_refresh: bool = False, - auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, + inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, ) -> Dict[str, Any]: """Refresh Nous OAuth state without mutating auth.json.""" - auth_mode = _normalize_nous_auth_mode(auth_mode) + inference_auth_mode = _normalize_nous_inference_auth_mode(inference_auth_mode) state: Dict[str, Any] = { "access_token": access_token, "refresh_token": refresh_token, @@ -4506,11 +4483,11 @@ def refresh_nous_oauth_pure( selected_auth_path, fallback_reason = _choose_nous_inference_auth_path( state, min_key_ttl_seconds=min_agent_key_ttl, - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, ) - if selected_auth_path == "invoke_jwt": + if selected_auth_path == NOUS_AUTH_PATH_INVOKE_JWT: _select_nous_invoke_jwt(state) - elif selected_auth_path == "legacy_session_key_mint": + elif selected_auth_path == NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT: _log_nous_legacy_session_key_selected( fallback_reason or "legacy_session_key_required", access_token=state.get("access_token"), @@ -4541,7 +4518,7 @@ def refresh_nous_oauth_from_state( min_key_ttl_seconds: int = DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, timeout_seconds: float = 15.0, force_refresh: bool = False, - auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, + inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, ) -> Dict[str, Any]: """Refresh Nous OAuth from a state dict. Thin wrapper around refresh_nous_oauth_pure.""" tls = state.get("tls") or {} @@ -4562,7 +4539,7 @@ def refresh_nous_oauth_from_state( insecure=tls.get("insecure"), ca_bundle=tls.get("ca_bundle"), force_refresh=force_refresh, - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, ) @@ -4640,7 +4617,7 @@ def resolve_nous_runtime_credentials( timeout_seconds: float = 15.0, insecure: Optional[bool] = None, ca_bundle: Optional[str] = None, - auth_mode: str = NOUS_INFERENCE_AUTH_AUTO, + inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, ) -> Dict[str, Any]: """ Resolve Nous inference credentials for runtime use. @@ -4652,7 +4629,7 @@ def resolve_nous_runtime_credentials( Returns dict with: provider, base_url, api_key, key_id, expires_at, expires_in, source ("invoke_jwt", "cache", or "portal"), and auth_path. """ - auth_mode = _normalize_nous_auth_mode(auth_mode) + inference_auth_mode = _normalize_nous_inference_auth_mode(inference_auth_mode) min_key_ttl_seconds = max(60, int(min_key_ttl_seconds)) sequence_id = uuid.uuid4().hex[:12] @@ -4682,6 +4659,8 @@ def resolve_nous_runtime_credentials( def _persist_state(reason: str) -> None: nonlocal persisted_state, state_persisted + # Skip writes where only derived TTL countdowns changed; this keeps + # the mtime-keyed Nous auth-status cache warm during read paths. if ( _nous_effective_provider_state(state) == _nous_effective_provider_state(persisted_state) @@ -4723,7 +4702,7 @@ def resolve_nous_runtime_credentials( _oauth_trace( "nous_runtime_credentials_start", sequence_id=sequence_id, - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, min_key_ttl_seconds=min_key_ttl_seconds, refresh_token_fp=_token_fingerprint(state.get("refresh_token")), ) @@ -4830,16 +4809,16 @@ def resolve_nous_runtime_credentials( state, access_token=access_token, min_key_ttl_seconds=min_key_ttl_seconds, - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, ) - if selected_auth_path == "invoke_jwt": + if selected_auth_path == NOUS_AUTH_PATH_INVOKE_JWT: _select_nous_invoke_jwt( state, access_token=access_token, sequence_id=sequence_id, ) - elif selected_auth_path == "legacy_session_key_cache": + elif selected_auth_path == NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE: used_cached_key = True logger.info("Nous inference auth: using cached agent_key") _oauth_trace("agent_key_reuse", sequence_id=sequence_id) @@ -4929,20 +4908,20 @@ def resolve_nous_runtime_credentials( # Persist retry refresh immediately for crash safety and cross-process visibility. _persist_state("post_refresh_mint_retry") - retry_auth_mode = ( - NOUS_INFERENCE_AUTH_LEGACY - if auth_mode == NOUS_INFERENCE_AUTH_LEGACY - else NOUS_INFERENCE_AUTH_FRESH + retry_inference_auth_mode = ( + NOUS_INFERENCE_AUTH_MODE_LEGACY + if inference_auth_mode == NOUS_INFERENCE_AUTH_MODE_LEGACY + else NOUS_INFERENCE_AUTH_MODE_FRESH ) retry_auth_path, _ = _choose_nous_inference_auth_path( state, access_token=access_token, min_key_ttl_seconds=min_key_ttl_seconds, - auth_mode=retry_auth_mode, + inference_auth_mode=retry_inference_auth_mode, ) - if retry_auth_path == "invoke_jwt": + if retry_auth_path == NOUS_AUTH_PATH_INVOKE_JWT: mint_payload = None - selected_auth_path = "invoke_jwt" + selected_auth_path = NOUS_AUTH_PATH_INVOKE_JWT _select_nous_invoke_jwt( state, access_token=access_token, @@ -5008,8 +4987,8 @@ def resolve_nous_runtime_credentials( "expires_at": expires_at, "expires_in": expires_in, "source": ( - "invoke_jwt" - if selected_auth_path == "invoke_jwt" + NOUS_AUTH_PATH_INVOKE_JWT + if selected_auth_path == NOUS_AUTH_PATH_INVOKE_JWT else ("cache" if used_cached_key else "portal") ), "auth_path": selected_auth_path, @@ -6691,7 +6670,10 @@ def _nous_device_code_login( or pconfig.inference_base_url ).rstrip("/") client_id = client_id or pconfig.client_id - scope, explicit_scope = _nous_device_scope(scope, default_scope=pconfig.scope) + scope, explicit_scope = _nous_device_scope_with_env_override( + scope, + default_scope=pconfig.scope, + ) timeout = httpx.Timeout(timeout_seconds) verify: bool | str = False if insecure else (ca_bundle if ca_bundle else True) @@ -6781,7 +6763,7 @@ def _nous_device_code_login( min_key_ttl_seconds=min_key_ttl_seconds, timeout_seconds=timeout_seconds, force_refresh=False, - auth_mode=NOUS_INFERENCE_AUTH_FRESH, + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_FRESH, ) except AuthError as exc: if exc.code == "subscription_required": diff --git a/hermes_cli/proxy/adapters/base.py b/hermes_cli/proxy/adapters/base.py index c7f36e25a2b..db778e18fa9 100644 --- a/hermes_cli/proxy/adapters/base.py +++ b/hermes_cli/proxy/adapters/base.py @@ -93,7 +93,7 @@ class UpstreamAdapter(ABC): fallback paths, such as switching from a preferred token type to a legacy bearer after the upstream rejects the first request. """ - del failed_credential, status_code + _ = failed_credential, status_code return None def describe(self) -> str: diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index a8cfd4cbada..eda8f831773 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -19,8 +19,8 @@ from typing import Any, Dict, FrozenSet, Optional from hermes_cli.auth import ( AuthError, DEFAULT_NOUS_INFERENCE_URL, - NOUS_INFERENCE_AUTH_AUTO, - NOUS_INFERENCE_AUTH_LEGACY, + NOUS_INFERENCE_AUTH_MODE_AUTO, + NOUS_INFERENCE_AUTH_MODE_LEGACY, _load_auth_store, _is_terminal_nous_refresh_error, _quarantine_nous_oauth_state, @@ -28,7 +28,7 @@ from hermes_cli.auth import ( _save_auth_store, _write_shared_nous_state, refresh_nous_oauth_from_state, - ) +) from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential logger = logging.getLogger(__name__) @@ -79,7 +79,9 @@ class NousPortalAdapter(UpstreamAdapter): ) def get_credential(self) -> UpstreamCredential: - return self._get_credential(auth_mode=NOUS_INFERENCE_AUTH_AUTO) + return self._get_credential( + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_AUTO, + ) def get_retry_credential( self, @@ -87,13 +89,16 @@ class NousPortalAdapter(UpstreamAdapter): failed_credential: UpstreamCredential, status_code: int, ) -> Optional[UpstreamCredential]: - del failed_credential if status_code != 401: return None + if failed_credential.bearer.count(".") != 2: + return None logger.info("proxy: Nous upstream rejected bearer; retrying with legacy session key") - return self._get_credential(auth_mode=NOUS_INFERENCE_AUTH_LEGACY) + return self._get_credential( + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_LEGACY, + ) - def _get_credential(self, *, auth_mode: str) -> UpstreamCredential: + def _get_credential(self, *, inference_auth_mode: str) -> UpstreamCredential: with self._lock: state = self._read_state() if state is None: @@ -104,7 +109,7 @@ class NousPortalAdapter(UpstreamAdapter): try: refreshed = refresh_nous_oauth_from_state( state, - auth_mode=auth_mode, + inference_auth_mode=inference_auth_mode, ) except AuthError as exc: if _is_terminal_nous_refresh_error(exc): diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index bfd47e9cc24..ebf053a6257 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -1816,7 +1816,7 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: """ if provider_id == "nous": from hermes_cli.auth import ( - _nous_device_scope, + _nous_device_scope_with_env_override, _request_nous_device_code_with_scope_fallback, PROVIDER_REGISTRY, ) @@ -1828,7 +1828,10 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: or pconfig.portal_base_url ).rstrip("/") client_id = pconfig.client_id - scope, explicit_scope = _nous_device_scope(None, default_scope=pconfig.scope) + scope, explicit_scope = _nous_device_scope_with_env_override( + None, + default_scope=pconfig.scope, + ) def _do_nous_device_request(): with httpx.Client( @@ -1982,7 +1985,7 @@ async def _start_device_code_flow(provider_id: str) -> Dict[str, Any]: def _nous_poller(session_id: str) -> None: """Background poller that drives a Nous device-code flow to completion.""" from hermes_cli.auth import ( - NOUS_INFERENCE_AUTH_FRESH, + NOUS_INFERENCE_AUTH_MODE_FRESH, _poll_for_token, refresh_nous_oauth_from_state, ) @@ -2031,7 +2034,7 @@ def _nous_poller(session_id: str) -> None: min_key_ttl_seconds=300, timeout_seconds=15.0, force_refresh=False, - auth_mode=NOUS_INFERENCE_AUTH_FRESH, + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_FRESH, ) from hermes_cli.auth import persist_nous_credentials persist_nous_credentials(full_state) diff --git a/run_agent.py b/run_agent.py index 1244d372fdf..484f9f84fd9 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2629,18 +2629,18 @@ class AIAgent: try: from hermes_cli.auth import ( - NOUS_INFERENCE_AUTH_AUTO, - NOUS_INFERENCE_AUTH_LEGACY, + NOUS_INFERENCE_AUTH_MODE_AUTO, + NOUS_INFERENCE_AUTH_MODE_LEGACY, resolve_nous_runtime_credentials, ) creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))), timeout_seconds=float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")), - auth_mode=( - NOUS_INFERENCE_AUTH_LEGACY + inference_auth_mode=( + NOUS_INFERENCE_AUTH_MODE_LEGACY if force - else NOUS_INFERENCE_AUTH_AUTO + else NOUS_INFERENCE_AUTH_MODE_AUTO ), ) except Exception as exc: diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index 0bdb1330a29..93c86ebe8f2 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -217,8 +217,8 @@ def test_resolve_nous_runtime_credentials_prefers_invoke_jwt_and_mirrors( creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) assert creds["api_key"] == token - assert creds["source"] == "invoke_jwt" - assert creds["auth_path"] == "invoke_jwt" + assert creds["source"] == auth_mod.NOUS_AUTH_PATH_INVOKE_JWT + assert creds["auth_path"] == auth_mod.NOUS_AUTH_PATH_INVOKE_JWT payload = json.loads((hermes_home / "auth.json").read_text()) singleton = payload["providers"]["nous"] @@ -297,7 +297,7 @@ def test_resolve_nous_runtime_credentials_invoke_jwt_is_idempotent( creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) assert creds["api_key"] == token - assert creds["source"] == "invoke_jwt" + assert creds["source"] == auth_mod.NOUS_AUTH_PATH_INVOKE_JWT assert auth_path.read_text() == before_content assert auth_path.stat().st_mtime_ns == before_mtime assert sync_calls == [] @@ -339,7 +339,7 @@ def test_resolve_nous_runtime_credentials_trusts_invoke_jwt_exp_over_stale_metad creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=300) assert creds["api_key"] == token - assert creds["source"] == "invoke_jwt" + assert creds["source"] == auth_mod.NOUS_AUTH_PATH_INVOKE_JWT payload = json.loads((hermes_home / "auth.json").read_text()) singleton = payload["providers"]["nous"] assert singleton["agent_key"] == token @@ -372,7 +372,7 @@ def test_resolve_nous_runtime_credentials_does_not_apply_legacy_ttl_to_invoke_jw creds = auth_mod.resolve_nous_runtime_credentials(min_key_ttl_seconds=1800) assert creds["api_key"] == token - assert creds["source"] == "invoke_jwt" + assert creds["source"] == auth_mod.NOUS_AUTH_PATH_INVOKE_JWT payload = json.loads((hermes_home / "auth.json").read_text()) assert payload["providers"]["nous"]["agent_key"] == token assert payload["credential_pool"]["nous"][0]["agent_key"] == token @@ -403,12 +403,12 @@ def test_legacy_auth_mode_bypasses_usable_invoke_jwt(tmp_path, monkeypatch): creds = auth_mod.resolve_nous_runtime_credentials( min_key_ttl_seconds=300, - auth_mode=auth_mod.NOUS_INFERENCE_AUTH_LEGACY, + inference_auth_mode=auth_mod.NOUS_INFERENCE_AUTH_MODE_LEGACY, ) assert mint_calls == [token] assert creds["api_key"] == "legacy-after-jwt-401" - assert creds["auth_path"] == "legacy_session_key_mint" + assert creds["auth_path"] == auth_mod.NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT payload = json.loads((hermes_home / "auth.json").read_text()) assert payload["providers"]["nous"]["agent_key"] == "legacy-after-jwt-401" @@ -1199,7 +1199,7 @@ def test_persist_nous_credentials_allows_recovery_from_401(tmp_path, monkeypatch providers.nous was empty. """ from hermes_cli.auth import ( - NOUS_INFERENCE_AUTH_FRESH, + NOUS_INFERENCE_AUTH_MODE_FRESH, persist_nous_credentials, resolve_nous_runtime_credentials, ) @@ -1232,7 +1232,7 @@ def test_persist_nous_credentials_allows_recovery_from_401(tmp_path, monkeypatch creds = resolve_nous_runtime_credentials( min_key_ttl_seconds=300, - auth_mode=NOUS_INFERENCE_AUTH_FRESH, + inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_FRESH, ) assert creds["api_key"] == "new-agent-key" @@ -1698,7 +1698,10 @@ def test_try_import_shared_rehydrates_on_success(shared_store_env, monkeypatch): def _fake_refresh(state, **kwargs): # Simulate portal returning fresh tokens + a new agent_key assert kwargs.get("force_refresh") is True - assert kwargs.get("auth_mode") == auth_mod.NOUS_INFERENCE_AUTH_FRESH + assert ( + kwargs.get("inference_auth_mode") + == auth_mod.NOUS_INFERENCE_AUTH_MODE_FRESH + ) return { **state, "access_token": "fresh-access-tok", @@ -1826,7 +1829,7 @@ def test_runtime_refresh_uses_newer_shared_token_before_local_stale_token( creds = auth_mod.resolve_nous_runtime_credentials( min_key_ttl_seconds=300, - auth_mode=auth_mod.NOUS_INFERENCE_AUTH_FRESH, + inference_auth_mode=auth_mod.NOUS_INFERENCE_AUTH_MODE_FRESH, ) assert creds["api_key"] == "agent-key-from-shared-token" diff --git a/tests/hermes_cli/test_proxy.py b/tests/hermes_cli/test_proxy.py index 9303fb1c702..45a098443f9 100644 --- a/tests/hermes_cli/test_proxy.py +++ b/tests/hermes_cli/test_proxy.py @@ -169,7 +169,7 @@ def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch) adapter = NousPortalAdapter() cred = adapter.get_retry_credential( failed_credential=UpstreamCredential( - bearer="jwt-access", + bearer="header.jwt.signature", base_url="https://inference-api.nousresearch.com/v1", ), status_code=401, @@ -177,7 +177,31 @@ def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch) assert cred is not None assert cred.bearer == "legacy-bearer" - assert mock_refresh.call_args.kwargs["auth_mode"] == "legacy" + assert mock_refresh.call_args.kwargs["inference_auth_mode"] == "legacy" + + +def test_nous_adapter_retry_credential_skips_opaque_bearer(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + _write_auth_store(tmp_path, { + "access_token": "jwt-access", + "refresh_token": "refresh-tok", + "agent_key": "opaque-bearer", + }) + + with patch( + "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + ) as mock_refresh: + adapter = NousPortalAdapter() + cred = adapter.get_retry_credential( + failed_credential=UpstreamCredential( + bearer="opaque-bearer", + base_url="https://inference-api.nousresearch.com/v1", + ), + status_code=401, + ) + + assert cred is None + mock_refresh.assert_not_called() def test_nous_adapter_get_credential_raises_when_not_logged_in(tmp_path, monkeypatch): @@ -364,7 +388,7 @@ class FakeAdapter(UpstreamAdapter): ) def get_retry_credential(self, *, failed_credential, status_code): - del failed_credential + _ = failed_credential self.retry_calls += 1 if status_code != 401 or not self._retry_bearer: return None diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index e569da31666..bc8a044e3ad 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -3667,7 +3667,7 @@ class TestNousCredentialRefresh: assert ok is True assert closed["value"] is True - assert captured["auth_mode"] == "legacy" + assert captured["inference_auth_mode"] == "legacy" assert rebuilt["kwargs"]["api_key"] == "new-nous-key" assert ( rebuilt["kwargs"]["base_url"] == "https://inference-api.nousresearch.com/v1" From 569bc94b59b687b5b6efa013f521756156f6e778 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Sun, 17 May 2026 22:29:40 +1000 Subject: [PATCH 136/142] fix(auth) fix a few cases where refresh tokens were not rotated. --- agent/credential_pool.py | 54 +++++++--------- hermes_cli/auth.py | 31 ++++++--- hermes_cli/proxy/adapters/nous_portal.py | 57 ++++++++--------- tests/agent/test_credential_pool.py | 2 +- tests/hermes_cli/test_auth_nous_provider.py | 70 +++++++++++++++++++++ tests/hermes_cli/test_proxy.py | 61 +++++++----------- 6 files changed, 166 insertions(+), 109 deletions(-) diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 7bdfe1c2973..98dbaf30839 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -623,18 +623,35 @@ class CredentialPool: return entry store_refresh = state.get("refresh_token", "") store_access = state.get("access_token", "") - if store_refresh and store_refresh != entry.refresh_token: + comparable_updates = { + "access_token": store_access, + "refresh_token": store_refresh, + "expires_at": state.get("expires_at"), + "agent_key": state.get("agent_key"), + "agent_key_expires_at": state.get("agent_key_expires_at"), + "inference_base_url": state.get("inference_base_url"), + } + should_sync = any( + value not in (None, "") and getattr(entry, key, None) != value + for key, value in comparable_updates.items() + ) + if should_sync: logger.debug( - "Pool entry %s: syncing tokens from auth.json (Nous refresh token changed)", + "Pool entry %s: syncing Nous state from auth.json", entry.id, ) field_updates: Dict[str, Any] = { - "access_token": store_access, - "refresh_token": store_refresh, "last_status": None, "last_status_at": None, "last_error_code": None, + "last_error_reason": None, + "last_error_message": None, + "last_error_reset_at": None, } + if store_access: + field_updates["access_token"] = store_access + if store_refresh: + field_updates["refresh_token"] = store_refresh if state.get("expires_at"): field_updates["expires_at"] = state["expires_at"] if state.get("agent_key"): @@ -813,40 +830,15 @@ class CredentialPool: synced = self._sync_nous_entry_from_auth_store(entry) if synced is not entry: entry = synced - nous_state = { - "access_token": entry.access_token, - "refresh_token": entry.refresh_token, - "client_id": entry.client_id, - "portal_base_url": entry.portal_base_url, - "inference_base_url": entry.inference_base_url, - "token_type": entry.token_type, - "scope": entry.scope, - "obtained_at": entry.obtained_at, - "expires_at": entry.expires_at, - "agent_key": entry.agent_key, - "agent_key_expires_at": entry.agent_key_expires_at, - "tls": entry.tls, - } - refreshed = auth_mod.refresh_nous_oauth_from_state( - nous_state, + auth_mod.resolve_nous_runtime_credentials( min_key_ttl_seconds=DEFAULT_AGENT_KEY_MIN_TTL_SECONDS, - force_refresh=force, inference_auth_mode=( auth_mod.NOUS_INFERENCE_AUTH_MODE_LEGACY if force else auth_mod.NOUS_INFERENCE_AUTH_MODE_AUTO ), ) - # Apply returned fields: dataclass fields via replace, extras via dict update - field_updates = {} - extra_updates = dict(entry.extra) - _field_names = {f.name for f in fields(entry)} - for k, v in refreshed.items(): - if k in _field_names: - field_updates[k] = v - elif k in _EXTRA_KEYS: - extra_updates[k] = v - updated = replace(entry, extra=extra_updates, **field_updates) + updated = self._sync_nous_entry_from_auth_store(entry) else: return entry except Exception as exc: diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index e65d9da20c8..cb97a4c2300 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -41,7 +41,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from urllib.parse import parse_qs, urlencode, urlparse import httpx @@ -89,11 +89,6 @@ NOUS_INFERENCE_AUTH_MODES = frozenset({ NOUS_AUTH_PATH_INVOKE_JWT = "invoke_jwt" NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE = "legacy_session_key_cache" NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT = "legacy_session_key_mint" -NOUS_AUTH_PATHS = frozenset({ - NOUS_AUTH_PATH_INVOKE_JWT, - NOUS_AUTH_PATH_LEGACY_SESSION_KEY_CACHE, - NOUS_AUTH_PATH_LEGACY_SESSION_KEY_MINT, -}) DEFAULT_AGENT_KEY_MIN_TTL_SECONDS = 30 * 60 # 30 minutes ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 # refresh 2 min before expiry NOUS_INVOKE_JWT_MIN_TTL_SECONDS = ACCESS_TOKEN_REFRESH_SKEW_SECONDS @@ -3991,7 +3986,7 @@ def _is_terminal_nous_refresh_error(exc: Exception) -> bool: return ( isinstance(exc, AuthError) and exc.provider == "nous" - and exc.code in {"invalid_grant", "invalid_token"} + and exc.code in {"invalid_grant", "invalid_token", "refresh_token_reused"} and bool(exc.relogin_required) ) @@ -4103,12 +4098,16 @@ def _try_import_shared_nous_state( "tls": {"insecure": False, "ca_bundle": None}, } + def _persist_shared_refresh(updated_state: Dict[str, Any], _reason: str) -> None: + _write_shared_nous_state(updated_state) + refreshed = refresh_nous_oauth_from_state( state, min_key_ttl_seconds=min_key_ttl_seconds, timeout_seconds=timeout_seconds, force_refresh=True, inference_auth_mode=NOUS_INFERENCE_AUTH_MODE_FRESH, + on_state_update=_persist_shared_refresh, ) _write_shared_nous_state(refreshed) except AuthError as exc: @@ -4163,7 +4162,7 @@ def _refresh_access_token( code = str(error_payload.get("error", "invalid_grant")) description = str(error_payload.get("error_description") or "Refresh token exchange failed") - relogin = code in {"invalid_grant", "invalid_token"} + relogin = code in {"invalid_grant", "invalid_token", "refresh_token_reused"} # Detect the OAuth 2.1 "refresh token reuse" signal from the Nous portal # server and surface an actionable message. This fires when an external @@ -4173,7 +4172,7 @@ def _refresh_access_token( # retires the original RT, Hermes's next refresh uses it, and the whole # session chain gets revoked as a token-theft signal (#15099). lowered = description.lower() - if "reuse" in lowered or "reuse detected" in lowered: + if code == "refresh_token_reused" or "reuse" in lowered or "reuse detected" in lowered: description = ( "Nous Portal detected refresh-token reuse and revoked this session.\n" "This usually means an external process (monitoring script, " @@ -4185,6 +4184,7 @@ def _refresh_access_token( "instead.\n" "Re-authenticate with: hermes auth add nous" ) + relogin = True raise AuthError(description, provider="nous", code=code, relogin_required=relogin) @@ -4418,8 +4418,14 @@ def refresh_nous_oauth_pure( ca_bundle: Optional[str] = None, force_refresh: bool = False, inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, + on_state_update: Optional[Callable[[Dict[str, Any], str], None]] = None, ) -> Dict[str, Any]: - """Refresh Nous OAuth state without mutating auth.json.""" + """Refresh Nous OAuth state without mutating auth.json directly. + + ``on_state_update`` is called after a successful access-token refresh and + before any subsequent agent-key mint. Callers that own persistent state can + use it to save the newly rotated refresh token before later work can fail. + """ inference_auth_mode = _normalize_nous_inference_auth_mode(inference_auth_mode) state: Dict[str, Any] = { "access_token": access_token, @@ -4479,6 +4485,8 @@ def refresh_nous_oauth_pure( state["expires_at"] = datetime.fromtimestamp( now.timestamp() + access_ttl, tz=timezone.utc ).isoformat() + if on_state_update is not None: + on_state_update(dict(state), "post_refresh_access_token") selected_auth_path, fallback_reason = _choose_nous_inference_auth_path( state, @@ -4519,6 +4527,7 @@ def refresh_nous_oauth_from_state( timeout_seconds: float = 15.0, force_refresh: bool = False, inference_auth_mode: str = NOUS_INFERENCE_AUTH_MODE_AUTO, + on_state_update: Optional[Callable[[Dict[str, Any], str], None]] = None, ) -> Dict[str, Any]: """Refresh Nous OAuth from a state dict. Thin wrapper around refresh_nous_oauth_pure.""" tls = state.get("tls") or {} @@ -4540,6 +4549,7 @@ def refresh_nous_oauth_from_state( ca_bundle=tls.get("ca_bundle"), force_refresh=force_refresh, inference_auth_mode=inference_auth_mode, + on_state_update=on_state_update, ) @@ -4603,6 +4613,7 @@ def persist_nous_credentials( def _sync_nous_pool_from_auth_store() -> None: + """Best-effort pool reseed after providers.nous changes; never fail login.""" try: from agent.credential_pool import load_pool diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index eda8f831773..9fb07a9c053 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -1,13 +1,13 @@ """Nous Portal upstream adapter. -Reads the user's Nous OAuth state from ``~/.hermes/auth.json``, refreshes -the access token and resolves the ``agent_key`` compatibility credential -when needed, then exposes the upstream base URL plus bearer for the proxy -server to forward to. +Reads the user's Nous OAuth state from ``~/.hermes/auth.json`` through the +shared runtime resolver, refreshes the access token and resolves the +``agent_key`` compatibility credential when needed, then exposes the upstream +base URL plus bearer for the proxy server to forward to. The ``agent_key`` field may hold either a NAS invoke JWT or the legacy opaque session key. The refresh helper handles both — see -:func:`hermes_cli.auth.refresh_nous_oauth_from_state`. +:func:`hermes_cli.auth.resolve_nous_runtime_credentials`. """ from __future__ import annotations @@ -22,12 +22,13 @@ from hermes_cli.auth import ( NOUS_INFERENCE_AUTH_MODE_AUTO, NOUS_INFERENCE_AUTH_MODE_LEGACY, _load_auth_store, + _auth_store_lock, _is_terminal_nous_refresh_error, _quarantine_nous_oauth_state, _quarantine_nous_pool_entries, _save_auth_store, _write_shared_nous_state, - refresh_nous_oauth_from_state, + resolve_nous_runtime_credentials, ) from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential @@ -50,9 +51,8 @@ class NousPortalAdapter(UpstreamAdapter): """Proxy upstream for the Nous Portal inference API.""" def __init__(self) -> None: - # Lock guards _load → refresh → _save against parallel proxy requests - # racing to refresh expired tokens. Refresh itself is HTTP, so we - # hold the lock across the network call (brief; OAuth refresh is fast). + # Serialize proxy requests in this process; cross-process token refresh + # and persistence are handled by resolve_nous_runtime_credentials(). self._lock = threading.Lock() @property @@ -107,8 +107,7 @@ class NousPortalAdapter(UpstreamAdapter): ) try: - refreshed = refresh_nous_oauth_from_state( - state, + refreshed = resolve_nous_runtime_credentials( inference_auth_mode=inference_auth_mode, ) except AuthError as exc: @@ -131,22 +130,20 @@ class NousPortalAdapter(UpstreamAdapter): f"Failed to refresh Nous Portal credentials: {exc}" ) from exc - self._save_state(refreshed) - - agent_key = refreshed.get("agent_key") + agent_key = refreshed.get("api_key") if not agent_key: raise RuntimeError( "Nous Portal refresh did not return a usable agent_key. " "Try `hermes login nous` to re-authenticate." ) - base_url = refreshed.get("inference_base_url") or DEFAULT_NOUS_INFERENCE_URL + base_url = refreshed.get("base_url") or DEFAULT_NOUS_INFERENCE_URL base_url = base_url.rstrip("/") return UpstreamCredential( bearer=agent_key, base_url=base_url, - expires_at=refreshed.get("agent_key_expires_at"), + expires_at=refreshed.get("expires_at"), ) # ------------------------------------------------------------------ @@ -156,7 +153,8 @@ class NousPortalAdapter(UpstreamAdapter): def _read_state(self) -> Optional[Dict[str, Any]]: try: - store = _load_auth_store() + with _auth_store_lock(): + store = _load_auth_store() except Exception as exc: logger.warning("proxy: failed to load auth store: %s", exc) return None @@ -174,21 +172,20 @@ class NousPortalAdapter(UpstreamAdapter): quarantine_reason: Optional[str] = None, ) -> None: try: - store = _load_auth_store() - if quarantine_error is not None and quarantine_reason: - _quarantine_nous_pool_entries( - store, - quarantine_error, - reason=quarantine_reason, - ) - providers = store.setdefault("providers", {}) - providers["nous"] = state - _save_auth_store(store) + with _auth_store_lock(): + store = _load_auth_store() + if quarantine_error is not None and quarantine_reason: + _quarantine_nous_pool_entries( + store, + quarantine_error, + reason=quarantine_reason, + ) + providers = store.setdefault("providers", {}) + providers["nous"] = state + _save_auth_store(store) _write_shared_nous_state(state) except Exception as exc: - # Best effort — we still return the fresh credential. The next - # request just won't see cached state, which means another refresh. - logger.warning("proxy: failed to persist refreshed Nous state: %s", exc) + logger.warning("proxy: failed to persist Nous quarantine state: %s", exc) __all__ = ["NousPortalAdapter"] diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index 875b08d91f0..c288619aedf 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -625,7 +625,7 @@ def test_nous_pool_terminal_refresh_removes_device_code_entry(tmp_path, monkeypa "access_token": "manual-nous-key", })) - monkeypatch.setattr(auth_mod, "refresh_nous_oauth_from_state", _terminal_refresh_failure) + monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _terminal_refresh_failure) assert pool.try_refresh_current() is None diff --git a/tests/hermes_cli/test_auth_nous_provider.py b/tests/hermes_cli/test_auth_nous_provider.py index 93c86ebe8f2..55903b11816 100644 --- a/tests/hermes_cli/test_auth_nous_provider.py +++ b/tests/hermes_cli/test_auth_nous_provider.py @@ -1426,6 +1426,36 @@ def test_refresh_token_reuse_detection_surfaces_actionable_message(): assert exc_info.value.relogin_required is True +def test_refresh_token_reuse_error_code_is_terminal(): + """Nous may return refresh_token_reused as the OAuth error code itself.""" + from hermes_cli import auth as auth_mod + + class _FakeResponse: + status_code = 400 + + def json(self): + return { + "error": "refresh_token_reused", + "error_description": "Refresh token reuse detected", + } + + class _FakeClient: + def post(self, *args, **kwargs): + return _FakeResponse() + + with pytest.raises(AuthError) as exc_info: + auth_mod._refresh_access_token( + client=_FakeClient(), + portal_base_url="https://portal.nousresearch.com", + client_id="hermes-cli", + refresh_token="rt_consumed_elsewhere", + ) + + assert exc_info.value.code == "refresh_token_reused" + assert exc_info.value.relogin_required is True + assert auth_mod._is_terminal_nous_refresh_error(exc_info.value) is True + + def test_refresh_token_exchange_sends_refresh_token_header(): """Nous refresh tokens must be sent in a header so sandbox proxies can substitute placeholder credentials without parsing form bodies. @@ -1686,6 +1716,46 @@ def test_try_import_shared_returns_none_on_refresh_failure( assert auth_mod._read_shared_nous_state() is None +def test_try_import_shared_persists_rotated_token_when_mint_fails( + shared_store_env, monkeypatch, +): + """A forced shared import refresh rotates the single-use token before minting. + + If the later agent-key mint fails, the shared store must still keep the + rotated refresh token; otherwise the next import attempt replays the + consumed token and trips refresh-token reuse. + """ + from hermes_cli import auth as auth_mod + + shared_state = _full_state_fixture() + shared_state["refresh_token"] = "refresh-old" + shared_state["access_token"] = "access-old" + auth_mod._write_shared_nous_state(shared_state) + + def _fake_refresh_access_token(*, client, portal_base_url, client_id, refresh_token): + assert refresh_token == "refresh-old" + return { + "access_token": "access-new", + "refresh_token": "refresh-new", + "expires_in": 900, + "token_type": "Bearer", + } + + def _fake_mint_agent_key(*, client, portal_base_url, access_token, min_ttl_seconds): + assert access_token == "access-new" + raise AuthError("credits exhausted", provider="nous", code="insufficient_credits") + + monkeypatch.setattr(auth_mod, "_refresh_access_token", _fake_refresh_access_token) + monkeypatch.setattr(auth_mod, "_mint_agent_key", _fake_mint_agent_key) + + assert auth_mod._try_import_shared_nous_state() is None + + shared_after = auth_mod._read_shared_nous_state() + assert shared_after is not None + assert shared_after["refresh_token"] == "refresh-new" + assert shared_after["access_token"] == "access-new" + + def test_try_import_shared_rehydrates_on_success(shared_store_env, monkeypatch): """Happy path: stored refresh_token is accepted, forced refresh+mint returns a fresh access_token + agent_key, and the returned dict has diff --git a/tests/hermes_cli/test_proxy.py b/tests/hermes_cli/test_proxy.py index 45a098443f9..34a10bfa5ff 100644 --- a/tests/hermes_cli/test_proxy.py +++ b/tests/hermes_cli/test_proxy.py @@ -103,7 +103,7 @@ def test_nous_adapter_authenticated_with_refresh_token_only(tmp_path, monkeypatc assert NousPortalAdapter().is_authenticated() -def test_nous_adapter_get_credential_refreshes_and_persists(tmp_path, monkeypatch): +def test_nous_adapter_get_credential_uses_runtime_resolver(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) _write_auth_store(tmp_path, { "access_token": "access-tok", @@ -114,32 +114,24 @@ def test_nous_adapter_get_credential_refreshes_and_persists(tmp_path, monkeypatc }) refreshed_state = { - "access_token": "access-tok", - "refresh_token": "refresh-tok", - "client_id": "hermes-cli", - "portal_base_url": "https://portal.nousresearch.com", - "inference_base_url": "https://inference-api.nousresearch.com/v1", - "agent_key": "minted-bearer", - "agent_key_expires_at": "2099-01-01T00:00:00Z", + "api_key": "minted-bearer", + "base_url": "https://inference-api.nousresearch.com/v1", + "expires_at": "2099-01-01T00:00:00Z", } with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", return_value=refreshed_state, - ) as mock_refresh: + ) as mock_resolve: adapter = NousPortalAdapter() cred = adapter.get_credential() - mock_refresh.assert_called_once() + mock_resolve.assert_called_once() assert cred.bearer == "minted-bearer" assert cred.base_url == "https://inference-api.nousresearch.com/v1" assert cred.expires_at == "2099-01-01T00:00:00Z" assert cred.token_type == "Bearer" - # Verify state was persisted back - stored = json.loads((tmp_path / "auth.json").read_text()) - assert stored["providers"]["nous"]["agent_key"] == "minted-bearer" - def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) @@ -153,19 +145,15 @@ def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch) }) refreshed_state = { - "access_token": "jwt-access", - "refresh_token": "refresh-tok", - "client_id": "hermes-cli", - "portal_base_url": "https://portal.nousresearch.com", - "inference_base_url": "https://inference-api.nousresearch.com/v1", - "agent_key": "legacy-bearer", - "agent_key_expires_at": "2099-01-01T00:00:00Z", + "api_key": "legacy-bearer", + "base_url": "https://inference-api.nousresearch.com/v1", + "expires_at": "2099-01-01T00:00:00Z", } with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", return_value=refreshed_state, - ) as mock_refresh: + ) as mock_resolve: adapter = NousPortalAdapter() cred = adapter.get_retry_credential( failed_credential=UpstreamCredential( @@ -177,7 +165,7 @@ def test_nous_adapter_retry_credential_forces_legacy_mint(tmp_path, monkeypatch) assert cred is not None assert cred.bearer == "legacy-bearer" - assert mock_refresh.call_args.kwargs["inference_auth_mode"] == "legacy" + assert mock_resolve.call_args.kwargs["inference_auth_mode"] == "legacy" def test_nous_adapter_retry_credential_skips_opaque_bearer(tmp_path, monkeypatch): @@ -189,8 +177,8 @@ def test_nous_adapter_retry_credential_skips_opaque_bearer(tmp_path, monkeypatch }) with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", - ) as mock_refresh: + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", + ) as mock_resolve: adapter = NousPortalAdapter() cred = adapter.get_retry_credential( failed_credential=UpstreamCredential( @@ -201,7 +189,7 @@ def test_nous_adapter_retry_credential_skips_opaque_bearer(tmp_path, monkeypatch ) assert cred is None - mock_refresh.assert_not_called() + mock_resolve.assert_not_called() def test_nous_adapter_get_credential_raises_when_not_logged_in(tmp_path, monkeypatch): @@ -219,7 +207,7 @@ def test_nous_adapter_get_credential_raises_on_refresh_failure(tmp_path, monkeyp }) with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", side_effect=RuntimeError("Refresh session has been revoked"), ): adapter = NousPortalAdapter() @@ -240,7 +228,7 @@ def test_nous_adapter_quarantines_terminal_refresh_failure(tmp_path, monkeypatch assert load_pool("nous").select() is not None with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", side_effect=AuthError( "Refresh session has been revoked", provider="nous", @@ -270,7 +258,7 @@ def test_nous_adapter_get_credential_raises_when_no_agent_key_returned(tmp_path, }) with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", return_value={"access_token": "a", "refresh_token": "r"}, ): adapter = NousPortalAdapter() @@ -291,7 +279,7 @@ def test_nous_adapter_concurrent_refresh_serialized(tmp_path, monkeypatch): counter = [0] counter_lock = threading.Lock() - def serializing_refresh(state, **kwargs): + def serializing_refresh(**kwargs): # If another thread is already inside refresh, the lock is broken. if in_flight.is_set(): overlap_detected.set() @@ -305,10 +293,9 @@ def test_nous_adapter_concurrent_refresh_serialized(tmp_path, monkeypatch): counter[0] += 1 idx = counter[0] return { - **state, - "agent_key": f"key-{idx}", - "agent_key_expires_at": "2099-01-01T00:00:00Z", - "inference_base_url": "https://inference-api.nousresearch.com/v1", + "api_key": f"key-{idx}", + "expires_at": "2099-01-01T00:00:00Z", + "base_url": "https://inference-api.nousresearch.com/v1", } finally: in_flight.clear() @@ -324,7 +311,7 @@ def test_nous_adapter_concurrent_refresh_serialized(tmp_path, monkeypatch): errors.append(exc) with patch( - "hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state", + "hermes_cli.proxy.adapters.nous_portal.resolve_nous_runtime_credentials", side_effect=serializing_refresh, ): threads = [threading.Thread(target=worker) for _ in range(3)] From 24c209f1129a0f1f540c049a7b2e7ad7e032385b Mon Sep 17 00:00:00 2001 From: Bartok9 Date: Sat, 16 May 2026 03:36:36 -0400 Subject: [PATCH 137/142] fix(auxiliary): detect quota exhaustion as payment error; allow capacity-error fallback for explicit providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #26803 Root causes: 1. _is_payment_error() checked for billing keywords (credits, insufficient funds, billing, payment required) but missed daily token quota exhaustion phrases used by Bedrock, Vertex AI, and LiteLLM proxies — e.g. 'Too many tokens per day', 'quota exceeded', 'resource exhausted', 'daily limit'. These are functionally identical to credit exhaustion (provider cannot serve the request) but don't trigger fallback. 2. The call_llm() fallback chain was gated on resolved_provider == 'auto'. When a task resolves to a specific provider (e.g. 'custom' for a LiteLLM proxy, or 'openrouter'), capacity failures (payment/quota/connection) silently raise instead of trying alternatives. This is overly conservative: capacity errors mean the provider *cannot* serve the request regardless of user intent, so alternatives should always be tried. Fixes: - Add quota-related keywords to _is_payment_error(): quota_exceeded, too many tokens per day, daily limit, tokens per day, daily quota, resource exhausted (Vertex AI gRPC code). - Allow fallback for capacity errors (payment + connection) even when resolved_provider is not 'auto'. Rate-limit fallback stays gated on is_auto to honour explicit provider constraints for transient limits. - Apply both fixes to sync call_llm() and async acall_llm() paths. - Add 6 targeted tests for the new quota-error detection cases. --- agent/auxiliary_client.py | 43 ++++++++++++++++++++++------ tests/agent/test_auxiliary_client.py | 38 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 4d11804f4cb..39fa378a914 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -2096,7 +2096,13 @@ def _is_payment_error(exc: Exception) -> bool: """Detect payment/credit/quota exhaustion errors. Returns True for HTTP 402 (Payment Required) and for 429/other errors - whose message indicates billing exhaustion rather than rate limiting. + whose message indicates billing exhaustion or daily quota exhaustion + rather than transient rate limiting. + + Daily token quota errors (e.g. Bedrock "Too many tokens per day", + Vertex AI "quota exceeded") are functionally equivalent to credit + exhaustion — the provider cannot serve the request until the quota + resets — and should trigger the same provider-fallback logic. """ status = getattr(exc, "status_code", None) if status == 402: @@ -2104,10 +2110,19 @@ def _is_payment_error(exc: Exception) -> bool: err_lower = str(exc).lower() # OpenRouter and other providers include "credits" or "afford" in 402 bodies, # but sometimes wrap them in 429 or other codes. + # Daily quota exhaustion from Bedrock, Vertex AI, and similar providers + # uses different language but is semantically identical to credit exhaustion. if status in {402, 429, None}: - if any(kw in err_lower for kw in ("credits", "insufficient funds", - "can only afford", "billing", - "payment required")): + if any(kw in err_lower for kw in ( + "credits", "insufficient funds", + "can only afford", "billing", + "payment required", + # Daily / monthly quota exhaustion keywords + "quota exceeded", "quota_exceeded", + "too many tokens per day", "daily limit", + "tokens per day", "daily quota", + "resource exhausted", # Vertex AI / gRPC quota errors + )): return True return False @@ -4538,11 +4553,17 @@ def call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) ) - # Only try alternative providers when the user didn't explicitly - # configure this task's provider. Explicit provider = hard constraint; - # auto (the default) = best-effort fallback chain. (#7559) + # Respect explicit provider choice for transient errors (auth, request + # validation, etc.) but allow fallback when the provider clearly cannot + # serve the request due to capacity: payment/quota exhaustion and + # connection failures are capacity problems, not request constraints. + # See #26803: daily token quota (429 + "too many tokens per day") must + # fall back just like a 402 credit error. is_auto = resolved_provider in {"auto", "", None} - if should_fallback and is_auto: + # Capacity errors bypass the explicit-provider gate: the provider + # literally cannot serve this request regardless of user intent. + is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) + if should_fallback and (is_auto or is_capacity_error): if _is_payment_error(first_err): reason = "payment error" # Resolve the actual provider label (resolved_provider may be @@ -4870,8 +4891,12 @@ async def async_call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) ) + # Capacity errors (payment/quota/connection) bypass the explicit-provider + # gate — the provider cannot serve the request regardless of user intent. + # See #26803: daily token quota must fall back like a 402 credit error. is_auto = resolved_provider in {"auto", "", None} - if should_fallback and is_auto: + is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) + if should_fallback and (is_auto or is_capacity_error): if _is_payment_error(first_err): reason = "payment error" _mark_provider_unhealthy( diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 61af7585a21..6194d586928 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -925,6 +925,44 @@ class TestIsPaymentError: exc = Exception("connection reset") assert _is_payment_error(exc) is False + # ── Daily / monthly quota exhaustion (#26803) ──────────────────────────── + + def test_429_quota_exceeded(self): + """Cloud provider quota exhaustion (e.g. Vertex AI) is a payment error.""" + exc = Exception("RESOURCE_EXHAUSTED: quota exceeded for project") + exc.status_code = 429 + assert _is_payment_error(exc) is True + + def test_429_too_many_tokens_per_day(self): + """Bedrock / LiteLLM daily token limit is a payment error.""" + exc = Exception("Too many tokens per day: 1000000 used, 1000000 limit") + exc.status_code = 429 + assert _is_payment_error(exc) is True + + def test_429_daily_limit_phrase(self): + """Generic 'daily limit' phrasing is a payment error.""" + exc = Exception("You have exceeded your daily limit.") + exc.status_code = 429 + assert _is_payment_error(exc) is True + + def test_429_resource_exhausted_grpc(self): + """Vertex AI gRPC RESOURCE_EXHAUSTED maps to payment error.""" + exc = Exception("resource exhausted") + exc.status_code = 429 + assert _is_payment_error(exc) is True + + def test_429_daily_quota_phrase(self): + """'daily quota' phrasing is a payment error.""" + exc = Exception("Daily quota of 500 requests reached.") + exc.status_code = 429 + assert _is_payment_error(exc) is True + + def test_429_transient_rate_limit_not_quota(self): + """Transient 429 rate limit without quota keywords is NOT a payment error.""" + exc = Exception("Rate limit exceeded. Retry after 10s.") + exc.status_code = 429 + assert _is_payment_error(exc) is False + class TestIsRateLimitError: """_is_rate_limit_error detects 429 rate-limit errors warranting fallback.""" From ec096cfbd8e0049aac360fd289a21a2759748410 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 14:45:28 -0700 Subject: [PATCH 138/142] test(auxiliary): adapt eviction tests to capacity-error fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two TestAuxiliaryClientPoisonedCacheEviction tests were written when explicit-provider users got no fallback at all on connection errors — they asserted ConnectionError propagated after eviction because the fallback gate blocked the auto chain. After the #26803 fix in the previous commit, capacity errors (payment/quota/connection) now DO trigger fallback even on explicit providers. The tests still verify cache eviction (their actual contract) but now stub _try_payment_fallback so the fallback machinery does not attempt a real network call. --- tests/agent/test_auxiliary_client.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 6194d586928..49d26825dde 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -2389,10 +2389,13 @@ class TestAuxiliaryClientPoisonedCacheEviction: def test_call_llm_evicts_on_connection_error_with_explicit_provider(self): """Connection error on an explicit provider must drop the cached client. - This is the exact reporter scenario: ``auxiliary.compression.provider: - main`` (resolves to ``openai-codex``) → no fallback chain runs (not - auto), but the cached client was poisoned by a prior timeout and must - be evicted so the next call rebuilds. + Reporter scenario: ``auxiliary.compression.provider: main`` (resolves + to ``openai-codex``). After #26803, capacity errors (payment/quota/ + connection) DO trigger fallback even on explicit providers — so we + also stub ``_try_payment_fallback`` to ``(None, None, "")`` so the + connection error re-raises after eviction instead of escaping into + a real network call. The contract under test is cache eviction, + not the fallback gate. """ from agent.auxiliary_client import _client_cache, _client_cache_lock @@ -2412,6 +2415,9 @@ class TestAuxiliaryClientPoisonedCacheEviction: ), patch( "agent.auxiliary_client._get_cached_client", return_value=(poisoned, "gpt-5.5"), + ), patch( + "agent.auxiliary_client._try_payment_fallback", + return_value=(None, None, ""), ): with pytest.raises(ConnectionError): call_llm( @@ -2445,6 +2451,9 @@ class TestAuxiliaryClientPoisonedCacheEviction: ), patch( "agent.auxiliary_client._get_cached_client", return_value=(poisoned, "gpt-5.5"), + ), patch( + "agent.auxiliary_client._try_payment_fallback", + return_value=(None, None, ""), ): with pytest.raises(ConnectionError): await async_call_llm( From a57424683759617040dd82082d85128deb236de4 Mon Sep 17 00:00:00 2001 From: zccyman Date: Sat, 16 May 2026 16:07:41 +0000 Subject: [PATCH 139/142] feat(auxiliary): add configurable fallback chains + main-agent safety net MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layered fallback for auxiliary tasks (compression, vision, tts, web_extract, session_search, etc.): 1. Primary aux provider (existing) 2. User-configured auxiliary..fallback_chain (new) 3. Main agent provider + model (new — last-resort safety net) 4. Warn user + re-raise original error (new) For users on 'auto' (no explicit aux provider), the existing _try_payment_fallback auto-detection chain runs instead — its Step 1 already IS the main agent model, so they get the same behaviour without configuration. The configured fallback_chain config schema comes from #26882 / @zccyman; the main-agent safety net + exhaustion warning were added on top. Closes #26882. Builds on the capacity-error gate fix in the previous commit (#26803 / @Bartok9). --- agent/auxiliary_client.py | 180 +++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 4 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 39fa378a914..ba78833248e 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -2606,6 +2606,133 @@ def _try_payment_fallback( return None, None, "" +def _try_main_agent_model_fallback( + failed_provider: str, + task: str = None, + reason: str = "error", +) -> Tuple[Optional[Any], Optional[str], str]: + """Last-resort fallback to the user's main agent provider + model. + + Used after the configured fallback_chain is exhausted (or empty) for + users with an explicit auxiliary provider. This is the "safety net" + layer: if nothing the user asked for can serve the request, try the + main chat model before giving up. + + Skips when the failed provider already IS the main provider (no point + retrying the same backend that just failed). + + Returns: + (client, model, provider_label) or (None, None, "") if no fallback. + """ + main_provider = (_read_main_provider() or "").strip() + main_model = (_read_main_model() or "").strip() + if not main_provider or not main_model or main_provider.lower() in {"auto", ""}: + return None, None, "" + + skip = (failed_provider or "").lower().strip() + if main_provider.lower() == skip: + # The thing that failed IS the main model — nothing to fall back to. + return None, None, "" + if _is_provider_unhealthy(main_provider): + _log_skip_unhealthy(main_provider, task) + return None, None, "" + + try: + client, resolved_model = resolve_provider_client( + provider=main_provider, model=main_model, + ) + except Exception: + client, resolved_model = None, None + + if client is None: + return None, None, "" + + label = f"main-agent({main_provider})" + logger.info( + "Auxiliary %s: %s on %s — falling back to main agent model %s (%s)", + task or "call", reason, failed_provider, label, resolved_model or main_model, + ) + return client, resolved_model or main_model, label + + +def _try_configured_fallback_chain( + task: str, + failed_provider: str, + reason: str = "error", +) -> Tuple[Optional[Any], Optional[str], str]: + """Try user-configured fallback_chain for a specific auxiliary task. + + Reads auxiliary..fallback_chain from config.yaml and tries each + entry in order. Each entry must have at least ``provider``; ``model``, + ``base_url``, and ``api_key`` are optional. + + Returns: + (client, model, provider_label) or (None, None, "") if no fallback. + """ + if not task: + return None, None, "" + + task_config = _get_auxiliary_task_config(task) + chain = task_config.get("fallback_chain") + if not chain or not isinstance(chain, list): + return None, None, "" + + skip = failed_provider.lower().strip() + tried = [] + + for i, entry in enumerate(chain): + if not isinstance(entry, dict): + continue + fb_provider = str(entry.get("provider", "")).strip() + if not fb_provider or fb_provider.lower() == skip: + continue + fb_model = str(entry.get("model", "")).strip() or None + fb_base_url = str(entry.get("base_url", "")).strip() or None + fb_api_key = str(entry.get("api_key", "")).strip() or None + + label = f"fallback_chain[{i}]({fb_provider})" + + try: + fb_client = _resolve_single_provider( + fb_provider, fb_model, fb_base_url, fb_api_key) + except Exception: + fb_client = None + + if fb_client is not None: + logger.info( + "Auxiliary %s: %s on %s — configured fallback to %s (%s)", + task, reason, failed_provider, label, fb_model or "default", + ) + return fb_client, fb_model, label + tried.append(label) + + if tried: + logger.debug( + "Auxiliary %s: configured fallback_chain exhausted (tried: %s)", + task, ", ".join(tried), + ) + return None, None, "" + + +def _resolve_single_provider( + provider: str, + model: Optional[str] = None, + base_url: Optional[str] = None, + api_key: Optional[str] = None, +) -> Optional[Any]: + """Resolve a single provider entry from fallback_chain to an OpenAI client. + + Uses the existing provider resolution infrastructure where possible. + """ + # Reuse resolve_provider_client which handles provider→client mapping + client, resolved_model = resolve_provider_client( + provider=provider, + model=model, + base_url=base_url, + api_key=api_key, + ) + return client + def _resolve_auto(main_runtime: Optional[Dict[str, Any]] = None) -> Tuple[Optional[OpenAI], Optional[str]]: """Full auto-detection chain. @@ -4579,8 +4706,24 @@ def call_llm( reason = "connection error" logger.info("Auxiliary %s: %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) - fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason=reason) + + # Fallback order (#26882, #26803): + # 1. User-configured fallback_chain (per-task) if set + # 2. Main agent model (last-resort safety net) + # For auto users (no explicit aux provider), use the full + # auto-detection chain instead — its Step 1 IS the main agent + # model, so users on `auto` already get main-model fallback. + fb_client, fb_model, fb_label = (None, None, "") + if is_auto: + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason=reason) + else: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason) + if fb_client is None: + fb_client, fb_model, fb_label = _try_main_agent_model_fallback( + resolved_provider, task, reason=reason) + if fb_client is not None: fb_kwargs = _build_call_kwargs( fb_label, fb_model, messages, @@ -4590,6 +4733,14 @@ def call_llm( base_url=str(getattr(fb_client, "base_url", "") or "")) return _validate_llm_response( fb_client.chat.completions.create(**fb_kwargs), task) + # All fallback layers exhausted — emit a single user-visible + # warning so the operator knows aux task is about to fail. + # (#26882) The error itself is re-raised below. + logger.warning( + "Auxiliary %s: %s on %s and all fallbacks exhausted " + "(fallback_chain + main agent model). Raising original error.", + task or "call", reason, resolved_provider, + ) # Connection/timeout errors leave the cached client poisoned (closed # httpx transport, half-read stream, dead async loop). Drop it from # the cache regardless of whether we found a fallback above so the @@ -4908,8 +5059,23 @@ async def async_call_llm( reason = "connection error" logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) - fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason=reason) + + # Fallback order (#26882, #26803): + # 1. User-configured fallback_chain (per-task) if set + # 2. Main agent model (last-resort safety net) + # Auto users get the full auto-detection chain instead — its + # Step 1 IS the main agent model. + fb_client, fb_model, fb_label = (None, None, "") + if is_auto: + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason=reason) + else: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason) + if fb_client is None: + fb_client, fb_model, fb_label = _try_main_agent_model_fallback( + resolved_provider, task, reason=reason) + if fb_client is not None: fb_kwargs = _build_call_kwargs( fb_label, fb_model, messages, @@ -4925,6 +5091,12 @@ async def async_call_llm( fb_kwargs["model"] = async_fb_model return _validate_llm_response( await async_fb.chat.completions.create(**fb_kwargs), task) + # All fallback layers exhausted — warn before re-raising. (#26882) + logger.warning( + "Auxiliary %s (async): %s on %s and all fallbacks exhausted " + "(fallback_chain + main agent model). Raising original error.", + task or "call", reason, resolved_provider, + ) # Mirror the sync path: drop poisoned clients on connection/timeout # so the next aux call rebuilds. See issue #23432. if _is_connection_error(first_err): From 034110e7ac08e01b077e23f30530c0791d06baee Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 16:26:00 -0700 Subject: [PATCH 140/142] chore(release): map zccyman noreply email for #26998 --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 6bb3d200583..d554e474fe6 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -184,6 +184,7 @@ AUTHOR_MAP = { "santoshhumagain1887@gmail.com": "npmisantosh", "39641663+luarss@users.noreply.github.com": "luarss", "16263913+zccyman@users.noreply.github.com": "zccyman", + "zccyman@users.noreply.github.com": "zccyman", # PR #26998 (auxiliary fallback chain) "ahmetosrak@Ahmet-MacBook-Air.local": "Osraka", "98612432+Osraka@users.noreply.github.com": "Osraka", "112634774+ryptotalent@users.noreply.github.com": "ryptotalent", From 766f263bd2453838bb34e98fbe048e09f9fefa25 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 16:27:06 -0700 Subject: [PATCH 141/142] =?UTF-8?q?test(auxiliary):=20cover=20layered=20fa?= =?UTF-8?q?llback=20(chain=20=E2=86=92=20main=20agent=20=E2=86=92=20warn)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7 new tests: TestAuxiliaryFallbackLayering (3): - configured_chain succeeds → main agent fallback NOT consulted - chain returns nothing → main agent fallback runs and succeeds - both exhausted → user-visible 'all fallbacks exhausted' warning fires before the original error is re-raised TestTryMainAgentModelFallback (4): - returns (None, None, "") when main provider is 'auto' - returns (None, None, "") when failed provider == main provider (no point retrying the same backend) - resolves the main provider's client when configured correctly - skips when main provider is marked unhealthy --- tests/agent/test_auxiliary_client.py | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 49d26825dde..2522fa16197 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -1151,6 +1151,140 @@ class TestCallLlmPaymentFallback: # Fallback client should have been used assert fallback_client.chat.completions.create.called + +class TestAuxiliaryFallbackLayering: + """Explicit-provider users get layered fallback: configured_chain → main agent → warn.""" + + def _make_payment_err(self): + exc = Exception("Payment Required: insufficient credits") + exc.status_code = 402 + return exc + + def test_explicit_provider_uses_configured_chain_first(self, monkeypatch, caplog): + """When a user has fallback_chain configured, it's tried BEFORE the main agent model.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + chain_client = MagicMock() + chain_client.chat.completions.create.return_value = MagicMock(choices=[ + MagicMock(message=MagicMock(content="from configured chain")) + ]) + + main_called = MagicMock() + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(chain_client, "gpt-4o-mini", "fallback_chain[0](openai)")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + side_effect=main_called): + result = call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert chain_client.chat.completions.create.called + # Main agent fallback should NOT have been consulted — chain succeeded first + main_called.assert_not_called() + + def test_explicit_provider_falls_back_to_main_when_chain_exhausted(self, monkeypatch): + """If configured fallback_chain returns nothing, main agent model is tried next.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + main_client = MagicMock() + main_client.chat.completions.create.return_value = MagicMock(choices=[ + MagicMock(message=MagicMock(content="from main agent")) + ]) + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + return_value=(main_client, "claude-sonnet-4", "main-agent(openrouter)")): + result = call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert main_client.chat.completions.create.called + + def test_warning_emitted_when_all_fallbacks_exhausted(self, monkeypatch, caplog): + """When chain AND main model both fail, a user-visible warning fires before re-raise.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + return_value=(None, None, "")), \ + caplog.at_level("WARNING", logger="agent.auxiliary_client"): + with pytest.raises(Exception, match="Payment Required"): + call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert any( + "all fallbacks exhausted" in r.message for r in caplog.records + ), f"Expected exhaustion warning, got: {[r.message for r in caplog.records]}" + + +class TestTryMainAgentModelFallback: + """_try_main_agent_model_fallback resolves the user's main provider+model as a safety net.""" + + def test_returns_none_when_main_provider_is_auto(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="auto"), \ + patch("agent.auxiliary_client._read_main_model", return_value="some-model"): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is None and model is None and label == "" + + def test_returns_none_when_failed_provider_equals_main(self): + """If the thing that failed IS the main model, no point retrying it.""" + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"): + client, model, label = _try_main_agent_model_fallback("openrouter", task="vision") + assert client is None and label == "" + + def test_resolves_main_provider_client(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + fake_client = MagicMock() + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"), \ + patch("agent.auxiliary_client._is_provider_unhealthy", return_value=False), \ + patch("agent.auxiliary_client.resolve_provider_client", + return_value=(fake_client, "anthropic/claude-sonnet-4")): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is fake_client + assert model == "anthropic/claude-sonnet-4" + assert label == "main-agent(openrouter)" + + def test_skips_when_main_provider_is_unhealthy(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"), \ + patch("agent.auxiliary_client._is_provider_unhealthy", return_value=True): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is None + + # --------------------------------------------------------------------------- # Gate: _resolve_api_key_provider must skip anthropic when not configured # --------------------------------------------------------------------------- From 43e566f77eaf01293086eb7cb99a21e240d60634 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 16:53:11 -0700 Subject: [PATCH 142/142] docs(fallback): document layered auxiliary fallback ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new 'Auxiliary Capacity-Error Fallback' section to website/docs/user-guide/features/fallback-providers.md covering: - The 4-step ladder (primary → fallback_chain → main agent → warn) - Which errors trigger fallback (402, 429 quota, connection) vs which respect explicit provider choice (transient 429 rate limits) - Optional fallback_chain config schema with vision + compression examples - Recognized quota-error phrases (Bedrock, Vertex AI, generic) Updates the bottom summary table — every auxiliary task now shows 'Layered (see above)' instead of 'Auto-detection chain' since explicit-provider users also get the main-agent safety net. --- .../user-guide/features/fallback-providers.md | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/website/docs/user-guide/features/fallback-providers.md b/website/docs/user-guide/features/fallback-providers.md index 72528796d55..b17102cb82e 100644 --- a/website/docs/user-guide/features/fallback-providers.md +++ b/website/docs/user-guide/features/fallback-providers.md @@ -320,6 +320,55 @@ auxiliary: --- +## Auxiliary Capacity-Error Fallback + +When you set an explicit auxiliary provider (e.g. `auxiliary.vision.provider: glm`), Hermes treats that as your preferred choice — but if the provider literally cannot serve the request because of a **capacity error** (HTTP 402 payment required, HTTP 429 daily-quota exhaustion, connection failure), Hermes falls back through a layered chain instead of failing silently: + +1. **Primary aux provider** — the one you configured (tried first, always) +2. **`auxiliary..fallback_chain`** — your per-task override list, if you wrote one +3. **Main agent provider + model** — last-resort safety net (always tried, even if you didn't write a chain) +4. **Warn + re-raise** — if every layer fails, Hermes logs `Auxiliary : ... all fallbacks exhausted` at WARNING level and re-raises the original error + +Transient HTTP 429 rate limits (`Retry-After: ...`) are treated as request constraints, not capacity problems — they respect your explicit provider choice and do **not** trigger the fallback ladder. Only daily/monthly quota exhaustion, payment errors, and connection failures bypass the explicit-provider gate. + +For users on `provider: auto` (no explicit aux provider), the existing auto-detection chain runs in place of steps 2–3. Its first step is already the main agent model, so `auto` users get the same outcome with zero config. + +### Optional: per-task fallback chain + +If you want a different fallback ordering than "main agent model first", configure `fallback_chain` explicitly. Each entry needs at least `provider`; `model`, `base_url`, and `api_key` are optional. + +```yaml +auxiliary: + vision: + provider: glm + model: glm-4v-flash + fallback_chain: + - provider: openrouter + model: google/gemini-3-flash-preview + - provider: nous + model: anthropic/claude-sonnet-4 + + compression: + provider: openrouter + fallback_chain: + - provider: openai + model: gpt-4o-mini +``` + +You do **not** need to configure `fallback_chain` to get fallback — the main-agent safety net runs regardless. Use it only when you specifically want a different order than the default. + +### Provider quota errors that trigger fallback + +Hermes recognizes these as capacity-equivalent to 402 credit exhaustion (not transient rate limits): + +- Bedrock / LiteLLM: `Too many tokens per day`, `daily limit`, `tokens per day` +- Vertex AI / GCP: `quota exceeded`, `resource exhausted`, `RESOURCE_EXHAUSTED` +- Generic: `daily quota`, `quota_exceeded` + +If your provider returns a different phrase for daily-quota exhaustion and Hermes doesn't trigger fallback, that's a bug — open an issue with the exact error string. + +--- + ## Context Compression Fallback Context compression uses the `auxiliary.compression` config block to control which model and provider handles summarization: @@ -378,14 +427,16 @@ See [Scheduled Tasks (Cron)](/docs/user-guide/features/cron) for full configurat | Feature | Fallback Mechanism | Config Location | |---------|-------------------|----------------| | Main agent model | `fallback_model` in config.yaml — per-turn failover on errors (primary restored each turn) | `fallback_model:` (top-level) | -| Vision | Auto-detection chain + internal OpenRouter retry | `auxiliary.vision` | -| Web extraction | Auto-detection chain + internal OpenRouter retry | `auxiliary.web_extract` | -| Context compression | Auto-detection chain, degrades to no-summary if unavailable | `auxiliary.compression` | -| Session search | Auto-detection chain | `auxiliary.session_search` | -| Skills hub | Auto-detection chain | `auxiliary.skills_hub` | -| MCP helpers | Auto-detection chain | `auxiliary.mcp` | -| Approval classification | Auto-detection chain | `auxiliary.approval` | -| Title generation | Auto-detection chain | `auxiliary.title_generation` | -| Triage specifier | Auto-detection chain | `auxiliary.triage_specifier` | +| Auxiliary tasks (any) — auto users | Full auto-detection chain (main agent model first, then provider chain) on capacity errors | `auxiliary..provider: auto` | +| Auxiliary tasks (any) — explicit provider | `fallback_chain` (if set) → main agent model → warn + raise, on capacity errors only | `auxiliary..fallback_chain` | +| Vision | Layered (see above) + internal OpenRouter retry | `auxiliary.vision` | +| Web extraction | Layered (see above) + internal OpenRouter retry | `auxiliary.web_extract` | +| Context compression | Layered (see above); degrades to no-summary if all layers unavailable | `auxiliary.compression` | +| Session search | Layered (see above) | `auxiliary.session_search` | +| Skills hub | Layered (see above) | `auxiliary.skills_hub` | +| MCP helpers | Layered (see above) | `auxiliary.mcp` | +| Approval classification | Layered (see above) | `auxiliary.approval` | +| Title generation | Layered (see above) | `auxiliary.title_generation` | +| Triage specifier | Layered (see above) | `auxiliary.triage_specifier` | | Delegation | Provider override only (no automatic fallback) | `delegation.provider` / `delegation.model` | | Cron jobs | Per-job provider override only (no automatic fallback) | Per-job `provider` / `model` |